WebSVN – HelenOS-historic – Diff – /kernel/trunk/generic/src/proc/scheduler.c


/*
 * Copyright (C) 2001-2004 Jakub Jermar
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * - Redistributions of source code must retain the above copyright
 *   notice, this list of conditions and the following disclaimer.
 * - Redistributions in binary form must reproduce the above copyright
 *   notice, this list of conditions and the following disclaimer in the
 *   documentation and/or other materials provided with the distribution.
 * - The name of the author may not be used to endorse or promote products
 *   derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
 
#include <proc/scheduler.h>
#include <proc/thread.h>
#include <proc/task.h>
#include <cpu.h>
#include <mm/vm.h>
#include <config.h>
#include <context.h>
#include <func.h>
#include <arch.h>
#include <arch/asm.h>
#include <list.h>
#include <panic.h>
#include <typedefs.h>
#include <mm/page.h>
#include <synch/spinlock.h>
#include <arch/faddr.h>
#include <arch/atomic.h>
 
volatile int nrdy;
 
 
/** Take actions before new thread runs
 *
 * Perform actions that need to be
 * taken before the newly selected
 * tread is passed control.
 *
 */
void before_thread_runs(void)
{
    before_thread_runs_arch();
    fpu_context_restore(&(THREAD->saved_fpu_context));
}
 
 
/** Initialize scheduler
 *
 * Initialize kernel scheduler.
 *
 */
void scheduler_init(void)
{
}
 
 
/** Get thread to be scheduled
 *
 * Get the optimal thread to be scheduled
 * according to thread accounting and scheduler
 * policy.
 *
 * @return Thread to be scheduled.
 *
 */
struct thread *find_best_thread(void)
{
    thread_t *t;
    runq_t *r;
    int i, n;
 
loop:
    cpu_priority_high();
 
    spinlock_lock(&CPU->lock);
    n = CPU->nrdy;
    spinlock_unlock(&CPU->lock);
 
    cpu_priority_low();
   
    if (n == 0) {
        #ifdef __SMP__
        /*
         * If the load balancing thread is not running, wake it up and
         * set CPU-private flag that the kcpulb has been started.
         */
        if (test_and_set(&CPU->kcpulbstarted) == 0) {
            waitq_wakeup(&CPU->kcpulb_wq, 0);
            goto loop;
        }
        #endif /* __SMP__ */
       
        /*
         * For there was nothing to run, the CPU goes to sleep
         * until a hardware interrupt or an IPI comes.
         * This improves energy saving and hyperthreading.
         * On the other hand, several hardware interrupts can be ignored.
         */
         cpu_sleep();
         goto loop;
    }
 
    cpu_priority_high();
   
    i = 0;
retry:
    for (; i<RQ_COUNT; i++) {
        r = &CPU->rq[i];
        spinlock_lock(&r->lock);
        if (r->n == 0) {
            /*
             * If this queue is empty, try a lower-priority queue.
             */
            spinlock_unlock(&r->lock);
            continue;
        }
   
        /* avoid deadlock with relink_rq() */
        if (!spinlock_trylock(&CPU->lock)) {
            /*
             * Unlock r and try again.
             */
            spinlock_unlock(&r->lock);
            goto retry;
        }
        CPU->nrdy--;
        spinlock_unlock(&CPU->lock);
 
        atomic_dec(&nrdy);
        r->n--;
 
        /*
         * Take the first thread from the queue.
         */
        t = list_get_instance(r->rq_head.next, thread_t, rq_link);
        list_remove(&t->rq_link);
 
        spinlock_unlock(&r->lock);
 
        spinlock_lock(&t->lock);
        t->cpu = CPU;
 
        t->ticks = us2ticks((i+1)*10000);
        t->pri = i; /* eventually correct rq index */
 
        /*
         * Clear the X_STOLEN flag so that t can be migrated when load balancing needs emerge.
         */
        t->flags &= ~X_STOLEN;
        spinlock_unlock(&t->lock);
 
        return t;
    }
    goto loop;
 
}
 
 
/** Prevent rq starvation
 *
 * Prevent low priority threads from starving in rq's.
 *
 * When the function decides to relink rq's, it reconnects
 * respective pointers so that in result threads with 'pri'
 * greater or equal 'start' are moved to a higher-priority queue.
 *
 * @param start Threshold priority.
 *
 */
void relink_rq(int start)
{
    link_t head;
    runq_t *r;
    int i, n;
 
    list_initialize(&head);
    spinlock_lock(&CPU->lock);
    if (CPU->needs_relink > NEEDS_RELINK_MAX) {
        for (i = start; i<RQ_COUNT-1; i++) {
            /* remember and empty rq[i + 1] */
            r = &CPU->rq[i + 1];
            spinlock_lock(&r->lock);
            list_concat(&head, &r->rq_head);
            n = r->n;
            r->n = 0;
            spinlock_unlock(&r->lock);
       
            /* append rq[i + 1] to rq[i] */
            r = &CPU->rq[i];
            spinlock_lock(&r->lock);
            list_concat(&r->rq_head, &head);
            r->n += n;
            spinlock_unlock(&r->lock);
        }
        CPU->needs_relink = 0;
    }
    spinlock_unlock(&CPU->lock);               
 
}
 
 
/** The scheduler
 *
 * The thread scheduling procedure.
 *
 */
void scheduler(void)
{
    volatile pri_t pri;
 
    pri = cpu_priority_high();
 
    if (haltstate)
        halt();
 
    if (THREAD) {
        spinlock_lock(&THREAD->lock);
        fpu_context_save(&(THREAD->saved_fpu_context));
        if (!context_save(&THREAD->saved_context)) {
            /*
             * This is the place where threads leave scheduler();
             */
            before_thread_runs();
            spinlock_unlock(&THREAD->lock);
            cpu_priority_restore(THREAD->saved_context.pri);
            return;
        }
 
        /*
         * CPU priority of preempted thread is recorded here
         * to facilitate scheduler() invocations from
         * cpu_priority_high()'ed code (e.g. waitq_sleep_timeout()).
         */
        THREAD->saved_context.pri = pri;
    }
 
    /*
     * Through the 'THE' structure, we keep track of THREAD, TASK, CPU
     * and preemption counter. At this point THE could be coming either
     * from THREAD's or CPU's stack.
     */
    the_copy(THE, (the_t *) CPU->stack);
 
    /*
     * We may not keep the old stack.
     * Reason: If we kept the old stack and got blocked, for instance, in
     * find_best_thread(), the old thread could get rescheduled by another
     * CPU and overwrite the part of its own stack that was also used by
     * the scheduler on this CPU.
     *
     * Moreover, we have to bypass the compiler-generated POP sequence
     * which is fooled by SP being set to the very top of the stack.
     * Therefore the scheduler() function continues in
     * scheduler_separated_stack().
     */
    context_save(&CPU->saved_context);
    context_set(&CPU->saved_context, FADDR(scheduler_separated_stack), CPU->stack, CPU_STACK_SIZE);
    context_restore(&CPU->saved_context);
    /* not reached */
}
 
 
/** Scheduler stack switch wrapper
 *
 * Second part of the scheduler() function
 * using new stack. Handling the actual context
 * switch to a new thread.
 *
 */
void scheduler_separated_stack(void)
{
    int priority;
 
    if (THREAD) {
        switch (THREAD->state) {
            case Running:
            THREAD->state = Ready;
            spinlock_unlock(&THREAD->lock);
            thread_ready(THREAD);
            break;
 
            case Exiting:
            frame_free((__address) THREAD->kstack);
            if (THREAD->ustack) {
                frame_free((__address) THREAD->ustack);
            }
 
            /*
             * Detach from the containing task.
             */
            spinlock_lock(&TASK->lock);
            list_remove(&THREAD->th_link);
            spinlock_unlock(&TASK->lock);
 
            spinlock_unlock(&THREAD->lock);
   
            spinlock_lock(&threads_lock);
            list_remove(&THREAD->threads_link);
            spinlock_unlock(&threads_lock);
 
            spinlock_lock(&CPU->lock);
            if(CPU->fpu_owner==THREAD) CPU->fpu_owner=NULL;
            spinlock_unlock(&CPU->lock);
 
            free(THREAD);
 
            break;
   
            case Sleeping:
            /*
             * Prefer the thread after it's woken up.
             */
            THREAD->pri = -1;
 
            /*
             * We need to release wq->lock which we locked in waitq_sleep().
             * Address of wq->lock is kept in THREAD->sleep_queue.
             */
            spinlock_unlock(&THREAD->sleep_queue->lock);
 
            /*
             * Check for possible requests for out-of-context invocation.
             */
            if (THREAD->call_me) {
                THREAD->call_me(THREAD->call_me_with);
                THREAD->call_me = NULL;
                THREAD->call_me_with = NULL;
            }
 
            spinlock_unlock(&THREAD->lock);
 
            break;
 
            default:
            /*
             * Entering state is unexpected.
             */
            panic("tid%d: unexpected state %s\n", THREAD->tid, thread_states[THREAD->state]);
            break;
        }
        THREAD = NULL;
    }
 
    THREAD = find_best_thread();
   
    spinlock_lock(&THREAD->lock);
    priority = THREAD->pri;
    spinlock_unlock(&THREAD->lock);
   
    relink_rq(priority);       
 
    spinlock_lock(&THREAD->lock);  
 
    /*
     * If both the old and the new task are the same, lots of work is avoided.
     */
    if (TASK != THREAD->task) {
        vm_t *m1 = NULL;
        vm_t *m2;
 
        if (TASK) {
            spinlock_lock(&TASK->lock);
            m1 = TASK->vm;
            spinlock_unlock(&TASK->lock);
        }
 
        spinlock_lock(&THREAD->task->lock);
        m2 = THREAD->task->vm;
        spinlock_unlock(&THREAD->task->lock);
       
        /*
         * Note that it is possible for two tasks to share one vm mapping.
         */
        if (m1 != m2) {
            /*
             * Both tasks and vm mappings are different.
             * Replace the old one with the new one.
             */
            vm_install(m2);
        }
        TASK = THREAD->task;   
    }
 
    THREAD->state = Running;
 
    #ifdef SCHEDULER_VERBOSE
    printf("cpu%d: tid %d (pri=%d,ticks=%d,nrdy=%d)\n", CPU->id, THREAD->tid, THREAD->pri, THREAD->ticks, CPU->nrdy);
    #endif  
 
    the_copy(THE, (the_t *) THREAD->kstack);
   
    context_restore(&THREAD->saved_context);
    /* not reached */
}
 
 
#ifdef __SMP__
/** Load balancing thread
 *
 * SMP load balancing thread, supervising thread supplies
 * for the CPU it's wired to.
 *
 * @param arg Generic thread argument (unused).
 *
 */
void kcpulb(void *arg)
{
    thread_t *t;
    int count, i, j, k = 0;
    pri_t pri;
 
loop:
    /*
     * Sleep until there's some work to do.
     */
    waitq_sleep(&CPU->kcpulb_wq);
 
not_satisfied:
    /*
     * Calculate the number of threads that will be migrated/stolen from
     * other CPU's. Note that situation can have changed between two
     * passes. Each time get the most up to date counts.
     */
    pri = cpu_priority_high();
    spinlock_lock(&CPU->lock);
    count = nrdy / config.cpu_active;
    count -= CPU->nrdy;
    spinlock_unlock(&CPU->lock);
    cpu_priority_restore(pri);
 
    if (count <= 0)
        goto satisfied;
 
    /*
     * Searching least priority queues on all CPU's first and most priority queues on all CPU's last.
     */
    for (j=RQ_COUNT-1; j >= 0; j--) {
        for (i=0; i < config.cpu_active; i++) {
            link_t *l;
            runq_t *r;
            cpu_t *cpu;
 
            cpu = &cpus[(i + k) % config.cpu_active];
 
            /*
             * Not interested in ourselves.
             * Doesn't require interrupt disabling for kcpulb is X_WIRED.
             */
            if (CPU == cpu)
                continue;              
 
restart:        pri = cpu_priority_high();
            r = &cpu->rq[j];
            spinlock_lock(&r->lock);
            if (r->n == 0) {
                spinlock_unlock(&r->lock);
                cpu_priority_restore(pri);
                continue;
            }
       
            t = NULL;
            l = r->rq_head.prev;    /* search rq from the back */
            while (l != &r->rq_head) {
                t = list_get_instance(l, thread_t, rq_link);
                /*
                 * We don't want to steal CPU-wired threads neither threads already stolen.
                 * The latter prevents threads from migrating between CPU's without ever being run.
                 * We don't want to steal threads whose FPU context is still in CPU.
                 */
                spinlock_lock(&t->lock);
                if ( (!(t->flags & (X_WIRED | X_STOLEN))) && (!(t->fpu_context_engaged)) ) {
               
                    /*
                     * Remove t from r.
                     */
 
                    spinlock_unlock(&t->lock);
                   
                    /*
                     * Here we have to avoid deadlock with relink_rq(),
                     * because it locks cpu and r in a different order than we do.
                     */
                    if (!spinlock_trylock(&cpu->lock)) {
                        /* Release all locks and try again. */
                        spinlock_unlock(&r->lock);
                        cpu_priority_restore(pri);
                        goto restart;
                    }
                    cpu->nrdy--;
                    spinlock_unlock(&cpu->lock);
 
                    atomic_dec(&nrdy);
 
                    r->n--;
                    list_remove(&t->rq_link);
 
                    break;
                }
                spinlock_unlock(&t->lock);
                l = l->prev;
                t = NULL;
            }
            spinlock_unlock(&r->lock);
 
            if (t) {
                /*
                 * Ready t on local CPU
                 */
                spinlock_lock(&t->lock);
                #ifdef KCPULB_VERBOSE
                printf("kcpulb%d: TID %d -> cpu%d, nrdy=%d, avg=%d\n", CPU->id, t->tid, CPU->id, CPU->nrdy, nrdy / config.cpu_active);
                #endif
                t->flags |= X_STOLEN;
                spinlock_unlock(&t->lock);
   
                thread_ready(t);
 
                cpu_priority_restore(pri);
   
                if (--count == 0)
                    goto satisfied;
                   
                /*
                 * We are not satisfied yet, focus on another CPU next time.
                 */
                k++;
               
                continue;
            }
            cpu_priority_restore(pri);
        }
    }
 
    if (CPU->nrdy) {
        /*
         * Be a little bit light-weight and let migrated threads run.
         */
        scheduler();
    }
    else {
        /*
         * We failed to migrate a single thread.
         * Something more sophisticated should be done.
         */
        scheduler();
    }
       
    goto not_satisfied;
 
satisfied:
    /*
     * Tell find_best_thread() to wake us up later again.
     */
    CPU->kcpulbstarted = 0;
    goto loop;
}
 
#endif /* __SMP__ */
 

Rev 170	Rev 184
1	/*	1	/*
2	* Copyright (C) 2001-2004 Jakub Jermar	2	* Copyright (C) 2001-2004 Jakub Jermar
3	* All rights reserved.	3	* All rights reserved.
4	*	4	*
5	* Redistribution and use in source and binary forms, with or without	5	* Redistribution and use in source and binary forms, with or without
6	* modification, are permitted provided that the following conditions	6	* modification, are permitted provided that the following conditions
7	* are met:	7	* are met:
8	*	8	*
9	* - Redistributions of source code must retain the above copyright	9	* - Redistributions of source code must retain the above copyright
10	* notice, this list of conditions and the following disclaimer.	10	* notice, this list of conditions and the following disclaimer.
11	* - Redistributions in binary form must reproduce the above copyright	11	* - Redistributions in binary form must reproduce the above copyright
12	* notice, this list of conditions and the following disclaimer in the	12	* notice, this list of conditions and the following disclaimer in the
13	* documentation and/or other materials provided with the distribution.	13	* documentation and/or other materials provided with the distribution.
14	* - The name of the author may not be used to endorse or promote products	14	* - The name of the author may not be used to endorse or promote products
15	* derived from this software without specific prior written permission.	15	* derived from this software without specific prior written permission.
16	*	16	*
17	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR	17	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES	18	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.	19	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,	20	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT	21	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,	22	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY	23	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT	24	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF	25	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.	26	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27	*/	27	*/
28		28
29	#include <proc/scheduler.h>	29	#include <proc/scheduler.h>
30	#include <proc/thread.h>	30	#include <proc/thread.h>
31	#include <proc/task.h>	31	#include <proc/task.h>
32	#include <cpu.h>	32	#include <cpu.h>
33	#include <mm/vm.h>	33	#include <mm/vm.h>
34	#include <config.h>	34	#include <config.h>
35	#include <context.h>	35	#include <context.h>
36	#include <func.h>	36	#include <func.h>
37	#include <arch.h>	37	#include <arch.h>
38	#include <arch/asm.h>	38	#include <arch/asm.h>
39	#include <list.h>	39	#include <list.h>
40	#include <panic.h>	40	#include <panic.h>
41	#include <typedefs.h>	41	#include <typedefs.h>
42	#include <mm/page.h>	42	#include <mm/page.h>
43	#include <synch/spinlock.h>	43	#include <synch/spinlock.h>
44	#include <arch/faddr.h>	44	#include <arch/faddr.h>
45	#include <arch/atomic.h>	45	#include <arch/atomic.h>
46		46
47	volatile int nrdy;	47	volatile int nrdy;
48		48
49		49
50	/** Take actions before new thread runs	50	/** Take actions before new thread runs
51	*	51	*
52	* Perform actions that need to be	52	* Perform actions that need to be
53	* taken before the newly selected	53	* taken before the newly selected
54	* tread is passed control.	54	* tread is passed control.
55	*	55	*
56	*/	56	*/
57	void before_thread_runs(void)	57	void before_thread_runs(void)
58	{	58	{
59	before_thread_runs_arch();	59	before_thread_runs_arch();
60	fpu_context_restore(&(THREAD->saved_fpu_context));	60	fpu_context_restore(&(THREAD->saved_fpu_context));
61	}	61	}
62		62
63		63
64	/** Initialize scheduler	64	/** Initialize scheduler
65	*	65	*
66	* Initialize kernel scheduler.	66	* Initialize kernel scheduler.
67	*	67	*
68	*/	68	*/
69	void scheduler_init(void)	69	void scheduler_init(void)
70	{	70	{
71	}	71	}
72		72
73		73
74	/** Get thread to be scheduled	74	/** Get thread to be scheduled
75	*	75	*
76	* Get the optimal thread to be scheduled	76	* Get the optimal thread to be scheduled
77	* according to thread accounting and scheduler	77	* according to thread accounting and scheduler
78	* policy.	78	* policy.
79	*	79	*
80	* @return Thread to be scheduled.	80	* @return Thread to be scheduled.
81	*	81	*
82	*/	82	*/
83	struct thread *find_best_thread(void)	83	struct thread *find_best_thread(void)
84	{	84	{
85	thread_t *t;	85	thread_t *t;
86	runq_t *r;	86	runq_t *r;
87	int i, n;	87	int i, n;
88		88
89	loop:	89	loop:
90	cpu_priority_high();	90	cpu_priority_high();
91		91
92	spinlock_lock(&CPU->lock);	92	spinlock_lock(&CPU->lock);
93	n = CPU->nrdy;	93	n = CPU->nrdy;
94	spinlock_unlock(&CPU->lock);	94	spinlock_unlock(&CPU->lock);
95		95
96	cpu_priority_low();	96	cpu_priority_low();
97		97
98	if (n == 0) {	98	if (n == 0) {
99	#ifdef __SMP__	99	#ifdef __SMP__
100	/*	100	/*
101	* If the load balancing thread is not running, wake it up and	101	* If the load balancing thread is not running, wake it up and
102	* set CPU-private flag that the kcpulb has been started.	102	* set CPU-private flag that the kcpulb has been started.
103	*/	103	*/
104	if (test_and_set(&CPU->kcpulbstarted) == 0) {	104	if (test_and_set(&CPU->kcpulbstarted) == 0) {
105	waitq_wakeup(&CPU->kcpulb_wq, 0);	105	waitq_wakeup(&CPU->kcpulb_wq, 0);
106	goto loop;	106	goto loop;
107	}	107	}
108	#endif /* __SMP__ */	108	#endif /* __SMP__ */
109		109
110	/*	110	/*
111	* For there was nothing to run, the CPU goes to sleep	111	* For there was nothing to run, the CPU goes to sleep
112	* until a hardware interrupt or an IPI comes.	112	* until a hardware interrupt or an IPI comes.
113	* This improves energy saving and hyperthreading.	113	* This improves energy saving and hyperthreading.
114	* On the other hand, several hardware interrupts can be ignored.	114	* On the other hand, several hardware interrupts can be ignored.
115	*/	115	*/
116	cpu_sleep();	116	cpu_sleep();
117	goto loop;	117	goto loop;
118	}	118	}
119		119
120	cpu_priority_high();	120	cpu_priority_high();
121		121
122	i = 0;	122	i = 0;
123	retry:	123	retry:
124	for (; i<RQ_COUNT; i++) {	124	for (; i<RQ_COUNT; i++) {
125	r = &CPU->rq[i];	125	r = &CPU->rq[i];
126	spinlock_lock(&r->lock);	126	spinlock_lock(&r->lock);
127	if (r->n == 0) {	127	if (r->n == 0) {
128	/*	128	/*
129	* If this queue is empty, try a lower-priority queue.	129	* If this queue is empty, try a lower-priority queue.
130	*/	130	*/
131	spinlock_unlock(&r->lock);	131	spinlock_unlock(&r->lock);
132	continue;	132	continue;
133	}	133	}
134		134
135	/* avoid deadlock with relink_rq() */	135	/* avoid deadlock with relink_rq() */
136	if (!spinlock_trylock(&CPU->lock)) {	136	if (!spinlock_trylock(&CPU->lock)) {
137	/*	137	/*
138	* Unlock r and try again.	138	* Unlock r and try again.
139	*/	139	*/
140	spinlock_unlock(&r->lock);	140	spinlock_unlock(&r->lock);
141	goto retry;	141	goto retry;
142	}	142	}
143	CPU->nrdy--;	143	CPU->nrdy--;
144	spinlock_unlock(&CPU->lock);	144	spinlock_unlock(&CPU->lock);
145		145
146	atomic_dec(&nrdy);	146	atomic_dec(&nrdy);
147	r->n--;	147	r->n--;
148		148
149	/*	149	/*
150	* Take the first thread from the queue.	150	* Take the first thread from the queue.
151	*/	151	*/
152	t = list_get_instance(r->rq_head.next, thread_t, rq_link);	152	t = list_get_instance(r->rq_head.next, thread_t, rq_link);
153	list_remove(&t->rq_link);	153	list_remove(&t->rq_link);
154		154
155	spinlock_unlock(&r->lock);	155	spinlock_unlock(&r->lock);
156		156
157	spinlock_lock(&t->lock);	157	spinlock_lock(&t->lock);
158	t->cpu = CPU;	158	t->cpu = CPU;
159		159
160	t->ticks = us2ticks((i+1)*10000);	160	t->ticks = us2ticks((i+1)*10000);
161	t->pri = i; /* eventually correct rq index */	161	t->pri = i; /* eventually correct rq index */
162		162
163	/*	163	/*
164	* Clear the X_STOLEN flag so that t can be migrated when load balancing needs emerge.	164	* Clear the X_STOLEN flag so that t can be migrated when load balancing needs emerge.
165	*/	165	*/
166	t->flags &= ~X_STOLEN;	166	t->flags &= ~X_STOLEN;
167	spinlock_unlock(&t->lock);	167	spinlock_unlock(&t->lock);
168		168
169	return t;	169	return t;
170	}	170	}
171	goto loop;	171	goto loop;
172		172
173	}	173	}
174		174
175		175
176	/** Prevent rq starvation	176	/** Prevent rq starvation
177	*	177	*
178	* Prevent low priority threads from starving in rq's.	178	* Prevent low priority threads from starving in rq's.
179	*	179	*
180	* When the function decides to relink rq's, it reconnects	180	* When the function decides to relink rq's, it reconnects
181	* respective pointers so that in result threads with 'pri'	181	* respective pointers so that in result threads with 'pri'
182	* greater or equal 'start' are moved to a higher-priority queue.	182	* greater or equal 'start' are moved to a higher-priority queue.
183	*	183	*
184	* @param start Threshold priority.	184	* @param start Threshold priority.
185	*	185	*
186	*/	186	*/
187	void relink_rq(int start)	187	void relink_rq(int start)
188	{	188	{
189	link_t head;	189	link_t head;
190	runq_t *r;	190	runq_t *r;
191	int i, n;	191	int i, n;
192		192
193	list_initialize(&head);	193	list_initialize(&head);
194	spinlock_lock(&CPU->lock);	194	spinlock_lock(&CPU->lock);
195	if (CPU->needs_relink > NEEDS_RELINK_MAX) {	195	if (CPU->needs_relink > NEEDS_RELINK_MAX) {
196	for (i = start; i<RQ_COUNT-1; i++) {	196	for (i = start; i<RQ_COUNT-1; i++) {
197	/* remember and empty rq[i + 1] */	197	/* remember and empty rq[i + 1] */
198	r = &CPU->rq[i + 1];	198	r = &CPU->rq[i + 1];
199	spinlock_lock(&r->lock);	199	spinlock_lock(&r->lock);
200	list_concat(&head, &r->rq_head);	200	list_concat(&head, &r->rq_head);
201	n = r->n;	201	n = r->n;
202	r->n = 0;	202	r->n = 0;
203	spinlock_unlock(&r->lock);	203	spinlock_unlock(&r->lock);
204		204
205	/* append rq[i + 1] to rq[i] */	205	/* append rq[i + 1] to rq[i] */
206	r = &CPU->rq[i];	206	r = &CPU->rq[i];
207	spinlock_lock(&r->lock);	207	spinlock_lock(&r->lock);
208	list_concat(&r->rq_head, &head);	208	list_concat(&r->rq_head, &head);
209	r->n += n;	209	r->n += n;
210	spinlock_unlock(&r->lock);	210	spinlock_unlock(&r->lock);
211	}	211	}
212	CPU->needs_relink = 0;	212	CPU->needs_relink = 0;
213	}	213	}
214	spinlock_unlock(&CPU->lock);	214	spinlock_unlock(&CPU->lock);
215		215
216	}	216	}
217		217
218		218
219	/** The scheduler	219	/** The scheduler
220	*	220	*
221	* The thread scheduling procedure.	221	* The thread scheduling procedure.
222	*	222	*
223	*/	223	*/
224	void scheduler(void)	224	void scheduler(void)
225	{	225	{
226	volatile pri_t pri;	226	volatile pri_t pri;
227		227
228	pri = cpu_priority_high();	228	pri = cpu_priority_high();
229		229
230	if (haltstate)	230	if (haltstate)
231	halt();	231	halt();
232		232
233	if (THREAD) {	233	if (THREAD) {
234	spinlock_lock(&THREAD->lock);	234	spinlock_lock(&THREAD->lock);
235	fpu_context_save(&(THREAD->saved_fpu_context));	235	fpu_context_save(&(THREAD->saved_fpu_context));
236	if (!context_save(&THREAD->saved_context)) {	236	if (!context_save(&THREAD->saved_context)) {
237	/*	237	/*
238	* This is the place where threads leave scheduler();	238	* This is the place where threads leave scheduler();
239	*/	239	*/
240	before_thread_runs();	240	before_thread_runs();
241	spinlock_unlock(&THREAD->lock);	241	spinlock_unlock(&THREAD->lock);
242	cpu_priority_restore(THREAD->saved_context.pri);	242	cpu_priority_restore(THREAD->saved_context.pri);
243	return;	243	return;
244	}	244	}
245		245
246	/*	246	/*
247	* CPU priority of preempted thread is recorded here	247	* CPU priority of preempted thread is recorded here
248	* to facilitate scheduler() invocations from	248	* to facilitate scheduler() invocations from
249	* cpu_priority_high()'ed code (e.g. waitq_sleep_timeout()).	249	* cpu_priority_high()'ed code (e.g. waitq_sleep_timeout()).
250	*/	250	*/
251	THREAD->saved_context.pri = pri;	251	THREAD->saved_context.pri = pri;
252	}	252	}
253		253
254	/*	254	/*
-		255	* Through the 'THE' structure, we keep track of THREAD, TASK, CPU
-		256	* and preemption counter. At this point THE could be coming either
-		257	* from THREAD's or CPU's stack.
-		258	*/
-		259	the_copy(THE, (the_t *) CPU->stack);
-		260
-		261	/*
255	* We may not keep the old stack.	262	* We may not keep the old stack.
256	* Reason: If we kept the old stack and got blocked, for instance, in	263	* Reason: If we kept the old stack and got blocked, for instance, in
257	* find_best_thread(), the old thread could get rescheduled by another	264	* find_best_thread(), the old thread could get rescheduled by another
258	* CPU and overwrite the part of its own stack that was also used by	265	* CPU and overwrite the part of its own stack that was also used by
259	* the scheduler on this CPU.	266	* the scheduler on this CPU.
260	*	267	*
261	* Moreover, we have to bypass the compiler-generated POP sequence	268	* Moreover, we have to bypass the compiler-generated POP sequence
262	* which is fooled by SP being set to the very top of the stack.	269	* which is fooled by SP being set to the very top of the stack.
263	* Therefore the scheduler() function continues in	270	* Therefore the scheduler() function continues in
264	* scheduler_separated_stack().	271	* scheduler_separated_stack().
265	*/	272	*/
266	context_save(&CPU->saved_context);	273	context_save(&CPU->saved_context);
267	context_set(&CPU->saved_context, FADDR(scheduler_separated_stack), CPU->stack, CPU_STACK_SIZE);	274	context_set(&CPU->saved_context, FADDR(scheduler_separated_stack), CPU->stack, CPU_STACK_SIZE);
268	context_restore(&CPU->saved_context);	275	context_restore(&CPU->saved_context);
269	/* not reached */	276	/* not reached */
270	}	277	}
271		278
272		279
273	/** Scheduler stack switch wrapper	280	/** Scheduler stack switch wrapper
274	*	281	*
275	* Second part of the scheduler() function	282	* Second part of the scheduler() function
276	* using new stack. Handling the actual context	283	* using new stack. Handling the actual context
277	* switch to a new thread.	284	* switch to a new thread.
278	*	285	*
279	*/	286	*/
280	void scheduler_separated_stack(void)	287	void scheduler_separated_stack(void)
281	{	288	{
282	int priority;	289	int priority;
283		290
284	if (THREAD) {	291	if (THREAD) {
285	switch (THREAD->state) {	292	switch (THREAD->state) {
286	case Running:	293	case Running:
287	THREAD->state = Ready;	294	THREAD->state = Ready;
288	spinlock_unlock(&THREAD->lock);	295	spinlock_unlock(&THREAD->lock);
289	thread_ready(THREAD);	296	thread_ready(THREAD);
290	break;	297	break;
291		298
292	case Exiting:	299	case Exiting:
293	frame_free((__address) THREAD->kstack);	300	frame_free((__address) THREAD->kstack);
294	if (THREAD->ustack) {	301	if (THREAD->ustack) {
295	frame_free((__address) THREAD->ustack);	302	frame_free((__address) THREAD->ustack);
296	}	303	}
297		304
298	/*	305	/*
299	* Detach from the containing task.	306	* Detach from the containing task.
300	*/	307	*/
301	spinlock_lock(&TASK->lock);	308	spinlock_lock(&TASK->lock);
302	list_remove(&THREAD->th_link);	309	list_remove(&THREAD->th_link);
303	spinlock_unlock(&TASK->lock);	310	spinlock_unlock(&TASK->lock);
304		311
305	spinlock_unlock(&THREAD->lock);	312	spinlock_unlock(&THREAD->lock);
306		313
307	spinlock_lock(&threads_lock);	314	spinlock_lock(&threads_lock);
308	list_remove(&THREAD->threads_link);	315	list_remove(&THREAD->threads_link);
309	spinlock_unlock(&threads_lock);	316	spinlock_unlock(&threads_lock);
310		317
311	spinlock_lock(&CPU->lock);	318	spinlock_lock(&CPU->lock);
312	if(CPU->fpu_owner==THREAD) CPU->fpu_owner=NULL;	319	if(CPU->fpu_owner==THREAD) CPU->fpu_owner=NULL;
313	spinlock_unlock(&CPU->lock);	320	spinlock_unlock(&CPU->lock);
314		321
315	free(THREAD);	322	free(THREAD);
316		323
317	break;	324	break;
318		325
319	case Sleeping:	326	case Sleeping:
320	/*	327	/*
321	* Prefer the thread after it's woken up.	328	* Prefer the thread after it's woken up.
322	*/	329	*/
323	THREAD->pri = -1;	330	THREAD->pri = -1;
324		331
325	/*	332	/*
326	* We need to release wq->lock which we locked in waitq_sleep().	333	* We need to release wq->lock which we locked in waitq_sleep().
327	* Address of wq->lock is kept in THREAD->sleep_queue.	334	* Address of wq->lock is kept in THREAD->sleep_queue.
328	*/	335	*/
329	spinlock_unlock(&THREAD->sleep_queue->lock);	336	spinlock_unlock(&THREAD->sleep_queue->lock);
330		337
331	/*	338	/*
332	* Check for possible requests for out-of-context invocation.	339	* Check for possible requests for out-of-context invocation.
333	*/	340	*/
334	if (THREAD->call_me) {	341	if (THREAD->call_me) {
335	THREAD->call_me(THREAD->call_me_with);	342	THREAD->call_me(THREAD->call_me_with);
336	THREAD->call_me = NULL;	343	THREAD->call_me = NULL;
337	THREAD->call_me_with = NULL;	344	THREAD->call_me_with = NULL;
338	}	345	}
339		346
340	spinlock_unlock(&THREAD->lock);	347	spinlock_unlock(&THREAD->lock);
341		348
342	break;	349	break;
343		350
344	default:	351	default:
345	/*	352	/*
346	* Entering state is unexpected.	353	* Entering state is unexpected.
347	*/	354	*/
348	panic("tid%d: unexpected state %s\n", THREAD->tid, thread_states[THREAD->state]);	355	panic("tid%d: unexpected state %s\n", THREAD->tid, thread_states[THREAD->state]);
349	break;	356	break;
350	}	357	}
351	THREAD = NULL;	358	THREAD = NULL;
352	}	359	}
353		360
354	THREAD = find_best_thread();	361	THREAD = find_best_thread();
355		362
356	spinlock_lock(&THREAD->lock);	363	spinlock_lock(&THREAD->lock);
357	priority = THREAD->pri;	364	priority = THREAD->pri;
358	spinlock_unlock(&THREAD->lock);	365	spinlock_unlock(&THREAD->lock);
359		366
360	relink_rq(priority);	367	relink_rq(priority);
361		368
362	spinlock_lock(&THREAD->lock);	369	spinlock_lock(&THREAD->lock);
363		370
364	/*	371	/*
365	* If both the old and the new task are the same, lots of work is avoided.	372	* If both the old and the new task are the same, lots of work is avoided.
366	*/	373	*/
367	if (TASK != THREAD->task) {	374	if (TASK != THREAD->task) {
368	vm_t *m1 = NULL;	375	vm_t *m1 = NULL;
369	vm_t *m2;	376	vm_t *m2;
370		377
371	if (TASK) {	378	if (TASK) {
372	spinlock_lock(&TASK->lock);	379	spinlock_lock(&TASK->lock);
373	m1 = TASK->vm;	380	m1 = TASK->vm;
374	spinlock_unlock(&TASK->lock);	381	spinlock_unlock(&TASK->lock);
375	}	382	}
376		383
377	spinlock_lock(&THREAD->task->lock);	384	spinlock_lock(&THREAD->task->lock);
378	m2 = THREAD->task->vm;	385	m2 = THREAD->task->vm;
379	spinlock_unlock(&THREAD->task->lock);	386	spinlock_unlock(&THREAD->task->lock);
380		387
381	/*	388	/*
382	* Note that it is possible for two tasks to share one vm mapping.	389	* Note that it is possible for two tasks to share one vm mapping.
383	*/	390	*/
384	if (m1 != m2) {	391	if (m1 != m2) {
385	/*	392	/*
386	* Both tasks and vm mappings are different.	393	* Both tasks and vm mappings are different.
387	* Replace the old one with the new one.	394	* Replace the old one with the new one.
388	*/	395	*/
389	vm_install(m2);	396	vm_install(m2);
390	}	397	}
391	TASK = THREAD->task;	398	TASK = THREAD->task;
392	}	399	}
393		400
394	THREAD->state = Running;	401	THREAD->state = Running;
395		402
396	#ifdef SCHEDULER_VERBOSE	403	#ifdef SCHEDULER_VERBOSE
397	printf("cpu%d: tid %d (pri=%d,ticks=%d,nrdy=%d)\n", CPU->id, THREAD->tid, THREAD->pri, THREAD->ticks, CPU->nrdy);	404	printf("cpu%d: tid %d (pri=%d,ticks=%d,nrdy=%d)\n", CPU->id, THREAD->tid, THREAD->pri, THREAD->ticks, CPU->nrdy);
398	#endif	405	#endif
399		406
-		407	the_copy(THE, (the_t *) THREAD->kstack);
-		408
400	context_restore(&THREAD->saved_context);	409	context_restore(&THREAD->saved_context);
401	/* not reached */	410	/* not reached */
402	}	411	}
403		412
404		413
405	#ifdef __SMP__	414	#ifdef __SMP__
406	/** Load balancing thread	415	/** Load balancing thread
407	*	416	*
408	* SMP load balancing thread, supervising thread supplies	417	* SMP load balancing thread, supervising thread supplies
409	* for the CPU it's wired to.	418	* for the CPU it's wired to.
410	*	419	*
411	* @param arg Generic thread argument (unused).	420	* @param arg Generic thread argument (unused).
412	*	421	*
413	*/	422	*/
414	void kcpulb(void *arg)	423	void kcpulb(void *arg)
415	{	424	{
416	thread_t *t;	425	thread_t *t;
417	int count, i, j, k = 0;	426	int count, i, j, k = 0;
418	pri_t pri;	427	pri_t pri;
419		428
420	loop:	429	loop:
421	/*	430	/*
422	* Sleep until there's some work to do.	431	* Sleep until there's some work to do.
423	*/	432	*/
424	waitq_sleep(&CPU->kcpulb_wq);	433	waitq_sleep(&CPU->kcpulb_wq);
425		434
426	not_satisfied:	435	not_satisfied:
427	/*	436	/*
428	* Calculate the number of threads that will be migrated/stolen from	437	* Calculate the number of threads that will be migrated/stolen from
429	* other CPU's. Note that situation can have changed between two	438	* other CPU's. Note that situation can have changed between two
430	* passes. Each time get the most up to date counts.	439	* passes. Each time get the most up to date counts.
431	*/	440	*/
432	pri = cpu_priority_high();	441	pri = cpu_priority_high();
433	spinlock_lock(&CPU->lock);	442	spinlock_lock(&CPU->lock);
434	count = nrdy / config.cpu_active;	443	count = nrdy / config.cpu_active;
435	count -= CPU->nrdy;	444	count -= CPU->nrdy;
436	spinlock_unlock(&CPU->lock);	445	spinlock_unlock(&CPU->lock);
437	cpu_priority_restore(pri);	446	cpu_priority_restore(pri);
438		447
439	if (count <= 0)	448	if (count <= 0)
440	goto satisfied;	449	goto satisfied;
441		450
442	/*	451	/*
443	* Searching least priority queues on all CPU's first and most priority queues on all CPU's last.	452	* Searching least priority queues on all CPU's first and most priority queues on all CPU's last.
444	*/	453	*/
445	for (j=RQ_COUNT-1; j >= 0; j--) {	454	for (j=RQ_COUNT-1; j >= 0; j--) {
446	for (i=0; i < config.cpu_active; i++) {	455	for (i=0; i < config.cpu_active; i++) {
447	link_t *l;	456	link_t *l;
448	runq_t *r;	457	runq_t *r;
449	cpu_t *cpu;	458	cpu_t *cpu;
450		459
451	cpu = &cpus[(i + k) % config.cpu_active];	460	cpu = &cpus[(i + k) % config.cpu_active];
452		461
453	/*	462	/*
454	* Not interested in ourselves.	463	* Not interested in ourselves.
455	* Doesn't require interrupt disabling for kcpulb is X_WIRED.	464	* Doesn't require interrupt disabling for kcpulb is X_WIRED.
456	*/	465	*/
457	if (CPU == cpu)	466	if (CPU == cpu)
458	continue;	467	continue;
459		468
460	restart: pri = cpu_priority_high();	469	restart: pri = cpu_priority_high();
461	r = &cpu->rq[j];	470	r = &cpu->rq[j];
462	spinlock_lock(&r->lock);	471	spinlock_lock(&r->lock);
463	if (r->n == 0) {	472	if (r->n == 0) {
464	spinlock_unlock(&r->lock);	473	spinlock_unlock(&r->lock);
465	cpu_priority_restore(pri);	474	cpu_priority_restore(pri);
466	continue;	475	continue;
467	}	476	}
468		477
469	t = NULL;	478	t = NULL;
470	l = r->rq_head.prev; /* search rq from the back */	479	l = r->rq_head.prev; /* search rq from the back */
471	while (l != &r->rq_head) {	480	while (l != &r->rq_head) {
472	t = list_get_instance(l, thread_t, rq_link);	481	t = list_get_instance(l, thread_t, rq_link);
473	/*	482	/*
474	* We don't want to steal CPU-wired threads neither threads already stolen.	483	* We don't want to steal CPU-wired threads neither threads already stolen.
475	* The latter prevents threads from migrating between CPU's without ever being run.	484	* The latter prevents threads from migrating between CPU's without ever being run.
476	* We don't want to steal threads whose FPU context is still in CPU.	485	* We don't want to steal threads whose FPU context is still in CPU.
477	*/	486	*/
478	spinlock_lock(&t->lock);	487	spinlock_lock(&t->lock);
479	if ( (!(t->flags & (X_WIRED \| X_STOLEN))) && (!(t->fpu_context_engaged)) ) {	488	if ( (!(t->flags & (X_WIRED \| X_STOLEN))) && (!(t->fpu_context_engaged)) ) {
480		489
481	/*	490	/*
482	* Remove t from r.	491	* Remove t from r.
483	*/	492	*/
484		493
485	spinlock_unlock(&t->lock);	494	spinlock_unlock(&t->lock);
486		495
487	/*	496	/*
488	* Here we have to avoid deadlock with relink_rq(),	497	* Here we have to avoid deadlock with relink_rq(),
489	* because it locks cpu and r in a different order than we do.	498	* because it locks cpu and r in a different order than we do.
490	*/	499	*/
491	if (!spinlock_trylock(&cpu->lock)) {	500	if (!spinlock_trylock(&cpu->lock)) {

Subversion Repositories HelenOS-historic

(root)/kernel/trunk/generic/src/proc/scheduler.c @ 1757 – Rev 170 → 184