WebSVN – HelenOS – Diff – /trunk/kernel/generic/src/mm/tlb.c


/*
 * Copyright (c) 2001-2004 Jakub Jermar
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * - Redistributions of source code must retain the above copyright
 *   notice, this list of conditions and the following disclaimer.
 * - Redistributions in binary form must reproduce the above copyright
 *   notice, this list of conditions and the following disclaimer in the
 *   documentation and/or other materials provided with the distribution.
 * - The name of the author may not be used to endorse or promote products
 *   derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
 
/** @addtogroup genericmm
 * @{
 */
 
/**
 * @file
 * @brief   Generic TLB shootdown algorithm.
 *
 * The algorithm implemented here is based on the CMU TLB shootdown
 * algorithm and is further simplified (e.g. all CPUs receive all TLB
 * shootdown messages).
 */
 
#include <mm/tlb.h>
#include <mm/asid.h>
#include <arch/mm/tlb.h>
#include <smp/ipi.h>
#include <synch/spinlock.h>
#include <atomic.h>
#include <arch/interrupt.h>
#include <config.h>
#include <arch.h>
#include <panic.h>
#include <debug.h>
#include <cpu.h>
 
/**
 * This lock is used for synchronisation between sender and
 * recipients of TLB shootdown message. It must be acquired
 * before CPU structure lock.
 */
SPINLOCK_INITIALIZE(tlblock);
 
void tlb_init(void)
{
    tlb_arch_init();
}
 
#ifdef CONFIG_SMP
 
/** Send TLB shootdown message.
 *
 * This function attempts to deliver TLB shootdown message
 * to all other processors.
 *
 * This function must be called with interrupts disabled.
 *
 * @param type Type describing scope of shootdown.
 * @param asid Address space, if required by type.
 * @param page Virtual page address, if required by type.
 * @param count Number of pages, if required by type.
 */
void tlb_shootdown_start(tlb_invalidate_type_t type, asid_t asid,
    uintptr_t page, count_t count)
{
    unsigned int i;
 
    CPU->tlb_active = 0;
    spinlock_lock(&tlblock);
   
    for (i = 0; i < config.cpu_count; i++) {
        cpu_t *cpu;
       
        if (i == CPU->id)
            continue;
 
        cpu = &cpus[i];
        spinlock_lock(&cpu->lock);
        if (cpu->tlb_messages_count == TLB_MESSAGE_QUEUE_LEN) {
            /*
             * The message queue is full.
             * Erase the queue and store one TLB_INVL_ALL message.
             */
            cpu->tlb_messages_count = 1;
            cpu->tlb_messages[0].type = TLB_INVL_ALL;
            cpu->tlb_messages[0].asid = ASID_INVALID;
            cpu->tlb_messages[0].page = 0;
            cpu->tlb_messages[0].count = 0;
        } else {
            /*
             * Enqueue the message.
             */
            index_t idx = cpu->tlb_messages_count++;
            cpu->tlb_messages[idx].type = type;
            cpu->tlb_messages[idx].asid = asid;
            cpu->tlb_messages[idx].page = page;
            cpu->tlb_messages[idx].count = count;
        }
        spinlock_unlock(&cpu->lock);
    }
   
    tlb_shootdown_ipi_send();
 
busy_wait: 
    for (i = 0; i < config.cpu_count; i++)
        if (cpus[i].tlb_active)
            goto busy_wait;
}
 
/** Finish TLB shootdown sequence. */
void tlb_shootdown_finalize(void)
{
    spinlock_unlock(&tlblock);
    CPU->tlb_active = 1;
}
 
void tlb_shootdown_ipi_send(void)
{
    ipi_broadcast(VECTOR_TLB_SHOOTDOWN_IPI);
}
 
/** Receive TLB shootdown message. */
void tlb_shootdown_ipi_recv(void)
{
    tlb_invalidate_type_t type;
    asid_t asid;
    uintptr_t page;
    count_t count;
    unsigned int i;
   
    ASSERT(CPU);
   
    CPU->tlb_active = 0;
    spinlock_lock(&tlblock);
    spinlock_unlock(&tlblock);
   
    spinlock_lock(&CPU->lock);
    ASSERT(CPU->tlb_messages_count <= TLB_MESSAGE_QUEUE_LEN);
 
    for (i = 0; i < CPU->tlb_messages_count; CPU->tlb_messages_count--) {
        type = CPU->tlb_messages[i].type;
        asid = CPU->tlb_messages[i].asid;
        page = CPU->tlb_messages[i].page;
        count = CPU->tlb_messages[i].count;
 
        switch (type) {
        case TLB_INVL_ALL:
            tlb_invalidate_all();
            break;
        case TLB_INVL_ASID:
            tlb_invalidate_asid(asid);
            break;
        case TLB_INVL_PAGES:
                ASSERT(count);
            tlb_invalidate_pages(asid, page, count);
            break;
        default:
            panic("Unknown type (%d).", type);
            break;
        }
        if (type == TLB_INVL_ALL)
            break;
    }
   
    spinlock_unlock(&CPU->lock);
    CPU->tlb_active = 1;
}
 
#endif /* CONFIG_SMP */
 
/** @}
 */
 

Rev 3626	Rev 3790
1	/*	1	/*
2	* Copyright (c) 2001-2004 Jakub Jermar	2	* Copyright (c) 2001-2004 Jakub Jermar
3	* All rights reserved.	3	* All rights reserved.
4	*	4	*
5	* Redistribution and use in source and binary forms, with or without	5	* Redistribution and use in source and binary forms, with or without
6	* modification, are permitted provided that the following conditions	6	* modification, are permitted provided that the following conditions
7	* are met:	7	* are met:
8	*	8	*
9	* - Redistributions of source code must retain the above copyright	9	* - Redistributions of source code must retain the above copyright
10	* notice, this list of conditions and the following disclaimer.	10	* notice, this list of conditions and the following disclaimer.
11	* - Redistributions in binary form must reproduce the above copyright	11	* - Redistributions in binary form must reproduce the above copyright
12	* notice, this list of conditions and the following disclaimer in the	12	* notice, this list of conditions and the following disclaimer in the
13	* documentation and/or other materials provided with the distribution.	13	* documentation and/or other materials provided with the distribution.
14	* - The name of the author may not be used to endorse or promote products	14	* - The name of the author may not be used to endorse or promote products
15	* derived from this software without specific prior written permission.	15	* derived from this software without specific prior written permission.
16	*	16	*
17	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR	17	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES	18	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.	19	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,	20	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT	21	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,	22	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY	23	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT	24	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF	25	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.	26	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27	*/	27	*/
28		28
29	/** @addtogroup genericmm	29	/** @addtogroup genericmm
30	* @{	30	* @{
31	*/	31	*/
32		32
33	/**	33	/**
34	* @file	34	* @file
35	* @brief Generic TLB shootdown algorithm.	35	* @brief Generic TLB shootdown algorithm.
36	*	36	*
37	* The algorithm implemented here is based on the CMU TLB shootdown	37	* The algorithm implemented here is based on the CMU TLB shootdown
38	* algorithm and is further simplified (e.g. all CPUs receive all TLB	38	* algorithm and is further simplified (e.g. all CPUs receive all TLB
39	* shootdown messages).	39	* shootdown messages).
40	*/	40	*/
41		41
42	#include <mm/tlb.h>	42	#include <mm/tlb.h>
43	#include <mm/asid.h>	43	#include <mm/asid.h>
44	#include <arch/mm/tlb.h>	44	#include <arch/mm/tlb.h>
45	#include <smp/ipi.h>	45	#include <smp/ipi.h>
46	#include <synch/spinlock.h>	46	#include <synch/spinlock.h>
47	#include <atomic.h>	47	#include <atomic.h>
48	#include <arch/interrupt.h>	48	#include <arch/interrupt.h>
49	#include <config.h>	49	#include <config.h>
50	#include <arch.h>	50	#include <arch.h>
51	#include <panic.h>	51	#include <panic.h>
52	#include <debug.h>	52	#include <debug.h>
53	#include <cpu.h>	53	#include <cpu.h>
54		54
55	/**	55	/**
56	* This lock is used for synchronisation between sender and	56	* This lock is used for synchronisation between sender and
57	* recipients of TLB shootdown message. It must be acquired	57	* recipients of TLB shootdown message. It must be acquired
58	* before CPU structure lock.	58	* before CPU structure lock.
59	*/	59	*/
60	SPINLOCK_INITIALIZE(tlblock);	60	SPINLOCK_INITIALIZE(tlblock);
61		61
62	void tlb_init(void)	62	void tlb_init(void)
63	{	63	{
64	tlb_arch_init();	64	tlb_arch_init();
65	}	65	}
66		66
67	#ifdef CONFIG_SMP	67	#ifdef CONFIG_SMP
68		68
69	/** Send TLB shootdown message.	69	/** Send TLB shootdown message.
70	*	70	*
71	* This function attempts to deliver TLB shootdown message	71	* This function attempts to deliver TLB shootdown message
72	* to all other processors.	72	* to all other processors.
73	*	73	*
74	* This function must be called with interrupts disabled.	74	* This function must be called with interrupts disabled.
75	*	75	*
76	* @param type Type describing scope of shootdown.	76	* @param type Type describing scope of shootdown.
77	* @param asid Address space, if required by type.	77	* @param asid Address space, if required by type.
78	* @param page Virtual page address, if required by type.	78	* @param page Virtual page address, if required by type.
79	* @param count Number of pages, if required by type.	79	* @param count Number of pages, if required by type.
80	*/	80	*/
81	void tlb_shootdown_start(tlb_invalidate_type_t type, asid_t asid,	81	void tlb_shootdown_start(tlb_invalidate_type_t type, asid_t asid,
82	uintptr_t page, count_t count)	82	uintptr_t page, count_t count)
83	{	83	{
84	unsigned int i;	84	unsigned int i;
85		85
86	CPU->tlb_active = 0;	86	CPU->tlb_active = 0;
87	spinlock_lock(&tlblock);	87	spinlock_lock(&tlblock);
88		88
89	for (i = 0; i < config.cpu_count; i++) {	89	for (i = 0; i < config.cpu_count; i++) {
90	cpu_t *cpu;	90	cpu_t *cpu;
91		91
92	if (i == CPU->id)	92	if (i == CPU->id)
93	continue;	93	continue;
94		94
95	cpu = &cpus[i];	95	cpu = &cpus[i];
96	spinlock_lock(&cpu->lock);	96	spinlock_lock(&cpu->lock);
97	if (cpu->tlb_messages_count == TLB_MESSAGE_QUEUE_LEN) {	97	if (cpu->tlb_messages_count == TLB_MESSAGE_QUEUE_LEN) {
98	/*	98	/*
99	* The message queue is full.	99	* The message queue is full.
100	* Erase the queue and store one TLB_INVL_ALL message.	100	* Erase the queue and store one TLB_INVL_ALL message.
101	*/	101	*/
102	cpu->tlb_messages_count = 1;	102	cpu->tlb_messages_count = 1;
103	cpu->tlb_messages[0].type = TLB_INVL_ALL;	103	cpu->tlb_messages[0].type = TLB_INVL_ALL;
104	cpu->tlb_messages[0].asid = ASID_INVALID;	104	cpu->tlb_messages[0].asid = ASID_INVALID;
105	cpu->tlb_messages[0].page = 0;	105	cpu->tlb_messages[0].page = 0;
106	cpu->tlb_messages[0].count = 0;	106	cpu->tlb_messages[0].count = 0;
107	} else {	107	} else {
108	/*	108	/*
109	* Enqueue the message.	109	* Enqueue the message.
110	*/	110	*/
111	index_t idx = cpu->tlb_messages_count++;	111	index_t idx = cpu->tlb_messages_count++;
112	cpu->tlb_messages[idx].type = type;	112	cpu->tlb_messages[idx].type = type;
113	cpu->tlb_messages[idx].asid = asid;	113	cpu->tlb_messages[idx].asid = asid;
114	cpu->tlb_messages[idx].page = page;	114	cpu->tlb_messages[idx].page = page;
115	cpu->tlb_messages[idx].count = count;	115	cpu->tlb_messages[idx].count = count;
116	}	116	}
117	spinlock_unlock(&cpu->lock);	117	spinlock_unlock(&cpu->lock);
118	}	118	}
119		119
120	tlb_shootdown_ipi_send();	120	tlb_shootdown_ipi_send();
121		121
122	busy_wait:	122	busy_wait:
123	for (i = 0; i < config.cpu_count; i++)	123	for (i = 0; i < config.cpu_count; i++)
124	if (cpus[i].tlb_active)	124	if (cpus[i].tlb_active)
125	goto busy_wait;	125	goto busy_wait;
126	}	126	}
127		127
128	/** Finish TLB shootdown sequence. */	128	/** Finish TLB shootdown sequence. */
129	void tlb_shootdown_finalize(void)	129	void tlb_shootdown_finalize(void)
130	{	130	{
131	spinlock_unlock(&tlblock);	131	spinlock_unlock(&tlblock);
132	CPU->tlb_active = 1;	132	CPU->tlb_active = 1;
133	}	133	}
134		134
135	void tlb_shootdown_ipi_send(void)	135	void tlb_shootdown_ipi_send(void)
136	{	136	{
137	ipi_broadcast(VECTOR_TLB_SHOOTDOWN_IPI);	137	ipi_broadcast(VECTOR_TLB_SHOOTDOWN_IPI);
138	}	138	}
139		139
140	/** Receive TLB shootdown message. */	140	/** Receive TLB shootdown message. */
141	void tlb_shootdown_ipi_recv(void)	141	void tlb_shootdown_ipi_recv(void)
142	{	142	{
143	tlb_invalidate_type_t type;	143	tlb_invalidate_type_t type;
144	asid_t asid;	144	asid_t asid;
145	uintptr_t page;	145	uintptr_t page;
146	count_t count;	146	count_t count;
147	unsigned int i;	147	unsigned int i;
148		148
149	ASSERT(CPU);	149	ASSERT(CPU);
150		150
151	CPU->tlb_active = 0;	151	CPU->tlb_active = 0;
152	spinlock_lock(&tlblock);	152	spinlock_lock(&tlblock);
153	spinlock_unlock(&tlblock);	153	spinlock_unlock(&tlblock);
154		154
155	spinlock_lock(&CPU->lock);	155	spinlock_lock(&CPU->lock);
156	ASSERT(CPU->tlb_messages_count <= TLB_MESSAGE_QUEUE_LEN);	156	ASSERT(CPU->tlb_messages_count <= TLB_MESSAGE_QUEUE_LEN);
157		157
158	for (i = 0; i < CPU->tlb_messages_count; CPU->tlb_messages_count--) {	158	for (i = 0; i < CPU->tlb_messages_count; CPU->tlb_messages_count--) {
159	type = CPU->tlb_messages[i].type;	159	type = CPU->tlb_messages[i].type;
160	asid = CPU->tlb_messages[i].asid;	160	asid = CPU->tlb_messages[i].asid;
161	page = CPU->tlb_messages[i].page;	161	page = CPU->tlb_messages[i].page;
162	count = CPU->tlb_messages[i].count;	162	count = CPU->tlb_messages[i].count;
163		163
164	switch (type) {	164	switch (type) {
165	case TLB_INVL_ALL:	165	case TLB_INVL_ALL:
166	tlb_invalidate_all();	166	tlb_invalidate_all();
167	break;	167	break;
168	case TLB_INVL_ASID:	168	case TLB_INVL_ASID:
169	tlb_invalidate_asid(asid);	169	tlb_invalidate_asid(asid);
170	break;	170	break;
171	case TLB_INVL_PAGES:	171	case TLB_INVL_PAGES:
172	ASSERT(count);	172	ASSERT(count);
173	tlb_invalidate_pages(asid, page, count);	173	tlb_invalidate_pages(asid, page, count);
174	break;	174	break;
175	default:	175	default:
176	panic("unknown type (%d)\n", type);	176	panic("Unknown type (%d).", type);
177	break;	177	break;
178	}	178	}
179	if (type == TLB_INVL_ALL)	179	if (type == TLB_INVL_ALL)
180	break;	180	break;
181	}	181	}
182		182
183	spinlock_unlock(&CPU->lock);	183	spinlock_unlock(&CPU->lock);
184	CPU->tlb_active = 1;	184	CPU->tlb_active = 1;
185	}	185	}
186		186
187	#endif /* CONFIG_SMP */	187	#endif /* CONFIG_SMP */
188		188
189	/** @}	189	/** @}
190	*/	190	*/
191		191

Subversion Repositories HelenOS

(root)/trunk/kernel/generic/src/mm/tlb.c @ 2797 – Rev 3626 → 3790