WebSVN – HelenOS – Diff – /branches/sparc/kernel/generic/src/proc/scheduler.c


/*
 * Copyright (C) 2001-2004 Jakub Jermar
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * - Redistributions of source code must retain the above copyright
 *   notice, this list of conditions and the following disclaimer.
 * - Redistributions in binary form must reproduce the above copyright
 *   notice, this list of conditions and the following disclaimer in the
 *   documentation and/or other materials provided with the distribution.
 * - The name of the author may not be used to endorse or promote products
 *   derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
 
#include <proc/scheduler.h>
#include <proc/thread.h>
#include <proc/task.h>
#include <mm/heap.h>
#include <mm/frame.h>
#include <mm/page.h>
#include <mm/vm.h>
#include <arch/asm.h>
#include <arch/faddr.h>
#include <arch/atomic.h>
#include <synch/spinlock.h>
#include <config.h>
#include <context.h>
#include <func.h>
#include <arch.h>
#include <list.h>
#include <panic.h>
#include <typedefs.h>
#include <cpu.h>
#include <print.h>
#include <debug.h>
 
volatile count_t nrdy;
 
 
/** Take actions before new thread runs
 *
 * Perform actions that need to be
 * taken before the newly selected
 * tread is passed control.
 *
 */
void before_thread_runs(void)
{
    before_thread_runs_arch();
#ifdef FPU_LAZY
    if(THREAD==CPU->fpu_owner)
        fpu_enable();
    else
        fpu_disable();
#else
    fpu_enable();
    if (THREAD->fpu_context_exists)
        fpu_context_restore(&(THREAD->saved_fpu_context));
    else {
        fpu_init();
        THREAD->fpu_context_exists=1;
    }
#endif
}
 
#ifdef FPU_LAZY
void scheduler_fpu_lazy_request(void)
{
    fpu_enable();
    if (CPU->fpu_owner != NULL) {  
        fpu_context_save(&CPU->fpu_owner->saved_fpu_context);
        /* don't prevent migration */
        CPU->fpu_owner->fpu_context_engaged=0;
    }
    if (THREAD->fpu_context_exists)
        fpu_context_restore(&THREAD->saved_fpu_context);
    else {
        fpu_init();
        THREAD->fpu_context_exists=1;
    }
    CPU->fpu_owner=THREAD;
    THREAD->fpu_context_engaged = 1;
}
#endif
 
/** Initialize scheduler
 *
 * Initialize kernel scheduler.
 *
 */
void scheduler_init(void)
{
}
 
 
/** Get thread to be scheduled
 *
 * Get the optimal thread to be scheduled
 * according to thread accounting and scheduler
 * policy.
 *
 * @return Thread to be scheduled.
 *
 */
struct thread *find_best_thread(void)
{
    thread_t *t;
    runq_t *r;
    int i, n;
 
    ASSERT(CPU != NULL);
 
loop:
    interrupts_disable();
 
    spinlock_lock(&CPU->lock);
    n = CPU->nrdy;
    spinlock_unlock(&CPU->lock);
 
    interrupts_enable();
   
    if (n == 0) {
        #ifdef __SMP__
        /*
         * If the load balancing thread is not running, wake it up and
         * set CPU-private flag that the kcpulb has been started.
         */
        if (test_and_set(&CPU->kcpulbstarted) == 0) {
            waitq_wakeup(&CPU->kcpulb_wq, 0);
            goto loop;
        }
        #endif /* __SMP__ */
       
        /*
         * For there was nothing to run, the CPU goes to sleep
         * until a hardware interrupt or an IPI comes.
         * This improves energy saving and hyperthreading.
         * On the other hand, several hardware interrupts can be ignored.
         */
         cpu_sleep();
         goto loop;
    }
 
    interrupts_disable();
   
    i = 0;
retry:
    for (; i<RQ_COUNT; i++) {
        r = &CPU->rq[i];
        spinlock_lock(&r->lock);
        if (r->n == 0) {
            /*
             * If this queue is empty, try a lower-priority queue.
             */
            spinlock_unlock(&r->lock);
            continue;
        }
 
        /* avoid deadlock with relink_rq() */
        if (!spinlock_trylock(&CPU->lock)) {
            /*
             * Unlock r and try again.
             */
            spinlock_unlock(&r->lock);
            goto retry;
        }
        CPU->nrdy--;
        spinlock_unlock(&CPU->lock);
 
        atomic_dec((int *) &nrdy);
        r->n--;
 
        /*
         * Take the first thread from the queue.
         */
        t = list_get_instance(r->rq_head.next, thread_t, rq_link);
        list_remove(&t->rq_link);
 
        spinlock_unlock(&r->lock);
 
        spinlock_lock(&t->lock);
        t->cpu = CPU;
 
        t->ticks = us2ticks((i+1)*10000);
        t->priority = i;    /* eventually correct rq index */
 
        /*
         * Clear the X_STOLEN flag so that t can be migrated when load balancing needs emerge.
         */
        t->flags &= ~X_STOLEN;
        spinlock_unlock(&t->lock);
 
        return t;
    }
    goto loop;
 
}
 
 
/** Prevent rq starvation
 *
 * Prevent low priority threads from starving in rq's.
 *
 * When the function decides to relink rq's, it reconnects
 * respective pointers so that in result threads with 'pri'
 * greater or equal 'start' are moved to a higher-priority queue.
 *
 * @param start Threshold priority.
 *
 */
void relink_rq(int start)
{
    link_t head;
    runq_t *r;
    int i, n;
 
    list_initialize(&head);
    spinlock_lock(&CPU->lock);
    if (CPU->needs_relink > NEEDS_RELINK_MAX) {
        for (i = start; i<RQ_COUNT-1; i++) {
            /* remember and empty rq[i + 1] */
            r = &CPU->rq[i + 1];
            spinlock_lock(&r->lock);
            list_concat(&head, &r->rq_head);
            n = r->n;
            r->n = 0;
            spinlock_unlock(&r->lock);
       
            /* append rq[i + 1] to rq[i] */
            r = &CPU->rq[i];
            spinlock_lock(&r->lock);
            list_concat(&r->rq_head, &head);
            r->n += n;
            spinlock_unlock(&r->lock);
        }
        CPU->needs_relink = 0;
    }
    spinlock_unlock(&CPU->lock);               
 
}
 
 
/** The scheduler
 *
 * The thread scheduling procedure.
 *
 */
void scheduler(void)
{
    volatile ipl_t ipl;
 
    ASSERT(CPU != NULL);
 
    ipl = interrupts_disable();
 
    if (haltstate)
        halt();
 
    if (THREAD) {
        spinlock_lock(&THREAD->lock);
#ifndef FPU_LAZY
        fpu_context_save(&(THREAD->saved_fpu_context));
#endif
        if (!context_save(&THREAD->saved_context)) {
            /*
             * This is the place where threads leave scheduler();
             */
            before_thread_runs();
            spinlock_unlock(&THREAD->lock);
            interrupts_restore(THREAD->saved_context.ipl);
            return;
        }
 
        /*
         * Interrupt priority level of preempted thread is recorded here
         * to facilitate scheduler() invocations from interrupts_disable()'d
         * code (e.g. waitq_sleep_timeout()).
         */
        THREAD->saved_context.ipl = ipl;
    }
 
    /*
     * Through the 'THE' structure, we keep track of THREAD, TASK, CPU
     * and preemption counter. At this point THE could be coming either
     * from THREAD's or CPU's stack.
     */
    the_copy(THE, (the_t *) CPU->stack);
 
    /*
     * We may not keep the old stack.
     * Reason: If we kept the old stack and got blocked, for instance, in
     * find_best_thread(), the old thread could get rescheduled by another
     * CPU and overwrite the part of its own stack that was also used by
     * the scheduler on this CPU.
     *
     * Moreover, we have to bypass the compiler-generated POP sequence
     * which is fooled by SP being set to the very top of the stack.
     * Therefore the scheduler() function continues in
     * scheduler_separated_stack().
     */
    context_save(&CPU->saved_context);
    context_set(&CPU->saved_context, FADDR(scheduler_separated_stack), (__address) CPU->stack, CPU_STACK_SIZE);
    context_restore(&CPU->saved_context);
    /* not reached */
}
 
 
/** Scheduler stack switch wrapper
 *
 * Second part of the scheduler() function
 * using new stack. Handling the actual context
 * switch to a new thread.
 *
 */
void scheduler_separated_stack(void)
{
    int priority;
 
    ASSERT(CPU != NULL);
 
    if (THREAD) {
        switch (THREAD->state) {
            case Running:
            THREAD->state = Ready;
            spinlock_unlock(&THREAD->lock);
            thread_ready(THREAD);
            break;
 
            case Exiting:
            frame_free((__address) THREAD->kstack);
            if (THREAD->ustack) {
                frame_free((__address) THREAD->ustack);
            }
 
            /*
             * Detach from the containing task.
             */
            spinlock_lock(&TASK->lock);
            list_remove(&THREAD->th_link);
            spinlock_unlock(&TASK->lock);
 
            spinlock_unlock(&THREAD->lock);
   
            spinlock_lock(&threads_lock);
            list_remove(&THREAD->threads_link);
            spinlock_unlock(&threads_lock);
 
            spinlock_lock(&CPU->lock);
            if(CPU->fpu_owner==THREAD) CPU->fpu_owner=NULL;
            spinlock_unlock(&CPU->lock);
 
            free(THREAD);
 
            break;
   
            case Sleeping:
            /*
             * Prefer the thread after it's woken up.
             */
            THREAD->priority = -1;
 
            /*
             * We need to release wq->lock which we locked in waitq_sleep().
             * Address of wq->lock is kept in THREAD->sleep_queue.
             */
            spinlock_unlock(&THREAD->sleep_queue->lock);
 
            /*
             * Check for possible requests for out-of-context invocation.
             */
            if (THREAD->call_me) {
                THREAD->call_me(THREAD->call_me_with);
                THREAD->call_me = NULL;
                THREAD->call_me_with = NULL;
            }
 
            spinlock_unlock(&THREAD->lock);
 
            break;
 
            default:
            /*
             * Entering state is unexpected.
             */
            panic("tid%d: unexpected state %s\n", THREAD->tid, thread_states[THREAD->state]);
            break;
        }
        THREAD = NULL;
    }
 
 
    THREAD = find_best_thread();
   
    spinlock_lock(&THREAD->lock);
    priority = THREAD->priority;
    spinlock_unlock(&THREAD->lock);
 
    relink_rq(priority);       
 
    spinlock_lock(&THREAD->lock);  
 
    /*
     * If both the old and the new task are the same, lots of work is avoided.
     */
    if (TASK != THREAD->task) {
        vm_t *m1 = NULL;
        vm_t *m2;
 
        if (TASK) {
            spinlock_lock(&TASK->lock);
            m1 = TASK->vm;
            spinlock_unlock(&TASK->lock);
        }
 
        spinlock_lock(&THREAD->task->lock);
        m2 = THREAD->task->vm;
        spinlock_unlock(&THREAD->task->lock);
       
        /*
         * Note that it is possible for two tasks to share one vm mapping.
         */
        if (m1 != m2) {
            /*
             * Both tasks and vm mappings are different.
             * Replace the old one with the new one.
             */
            vm_install(m2);
        }
        TASK = THREAD->task;   
    }
 
    THREAD->state = Running;
 
    #ifdef SCHEDULER_VERBOSE
    printf("cpu%d: tid %d (priority=%d,ticks=%d,nrdy=%d)\n", CPU->id, THREAD->tid, THREAD->priority, THREAD->ticks, CPU->nrdy);
    #endif  
 
    /*
     * Copy the knowledge of CPU, TASK, THREAD and preemption counter to thread's stack.
     */
    the_copy(THE, (the_t *) THREAD->kstack);
   
    context_restore(&THREAD->saved_context);
    /* not reached */
}
 
 
#ifdef __SMP__
/** Load balancing thread
 *
 * SMP load balancing thread, supervising thread supplies
 * for the CPU it's wired to.
 *
 * @param arg Generic thread argument (unused).
 *
 */
void kcpulb(void *arg)
{
    thread_t *t;
    int count, i, j, k = 0;
    ipl_t ipl;
 
loop:
    /*
     * Sleep until there's some work to do.
     */
    waitq_sleep(&CPU->kcpulb_wq);
 
not_satisfied:
    /*
     * Calculate the number of threads that will be migrated/stolen from
     * other CPU's. Note that situation can have changed between two
     * passes. Each time get the most up to date counts.
     */
    ipl = interrupts_disable();
    spinlock_lock(&CPU->lock);
    count = nrdy / config.cpu_active;
    count -= CPU->nrdy;
    spinlock_unlock(&CPU->lock);
    interrupts_restore(ipl);
 
    if (count <= 0)
        goto satisfied;
 
    /*
     * Searching least priority queues on all CPU's first and most priority queues on all CPU's last.
     */
    for (j=RQ_COUNT-1; j >= 0; j--) {
        for (i=0; i < config.cpu_active; i++) {
            link_t *l;
            runq_t *r;
            cpu_t *cpu;
 
            cpu = &cpus[(i + k) % config.cpu_active];
 
            /*
             * Not interested in ourselves.
             * Doesn't require interrupt disabling for kcpulb is X_WIRED.
             */
            if (CPU == cpu)
                continue;              
 
restart:        ipl = interrupts_disable();
            r = &cpu->rq[j];
            spinlock_lock(&r->lock);
            if (r->n == 0) {
                spinlock_unlock(&r->lock);
                interrupts_restore(ipl);
                continue;
            }
       
            t = NULL;
            l = r->rq_head.prev;    /* search rq from the back */
            while (l != &r->rq_head) {
                t = list_get_instance(l, thread_t, rq_link);
                /*
                 * We don't want to steal CPU-wired threads neither threads already stolen.
                 * The latter prevents threads from migrating between CPU's without ever being run.
                 * We don't want to steal threads whose FPU context is still in CPU.
                 */
                spinlock_lock(&t->lock);
                if ( (!(t->flags & (X_WIRED | X_STOLEN))) && (!(t->fpu_context_engaged)) ) {
               
                    /*
                     * Remove t from r.
                     */
 
                    spinlock_unlock(&t->lock);
                   
                    /*
                     * Here we have to avoid deadlock with relink_rq(),
                     * because it locks cpu and r in a different order than we do.
                     */
                    if (!spinlock_trylock(&cpu->lock)) {
                        /* Release all locks and try again. */
                        spinlock_unlock(&r->lock);
                        interrupts_restore(ipl);
                        goto restart;
                    }
                    cpu->nrdy--;
                    spinlock_unlock(&cpu->lock);
 
                    atomic_dec((int *)&nrdy);
 
                    r->n--;
                    list_remove(&t->rq_link);
 
                    break;
                }
                spinlock_unlock(&t->lock);
                l = l->prev;
                t = NULL;
            }
            spinlock_unlock(&r->lock);
 
            if (t) {
                /*
                 * Ready t on local CPU
                 */
                spinlock_lock(&t->lock);
                #ifdef KCPULB_VERBOSE
                printf("kcpulb%d: TID %d -> cpu%d, nrdy=%d, avg=%d\n", CPU->id, t->tid, CPU->id, CPU->nrdy, nrdy / config.cpu_active);
                #endif
                t->flags |= X_STOLEN;
                spinlock_unlock(&t->lock);
   
                thread_ready(t);
 
                interrupts_restore(ipl);
   
                if (--count == 0)
                    goto satisfied;
                   
                /*
                 * We are not satisfied yet, focus on another CPU next time.
                 */
                k++;
               
                continue;
            }
            interrupts_restore(ipl);
        }
    }
 
    if (CPU->nrdy) {
        /*
         * Be a little bit light-weight and let migrated threads run.
         */
        scheduler();
    }
    else {
        /*
         * We failed to migrate a single thread.
         * Something more sophisticated should be done.
         */
        scheduler();
    }
       
    goto not_satisfied;
 
satisfied:
    /*
     * Tell find_best_thread() to wake us up later again.
     */
    CPU->kcpulbstarted = 0;
    goto loop;
}
 
#endif /* __SMP__ */
 

Rev 413	Rev 414
1	/*	1	/*
2	* Copyright (C) 2001-2004 Jakub Jermar	2	* Copyright (C) 2001-2004 Jakub Jermar
3	* All rights reserved.	3	* All rights reserved.
4	*	4	*
5	* Redistribution and use in source and binary forms, with or without	5	* Redistribution and use in source and binary forms, with or without
6	* modification, are permitted provided that the following conditions	6	* modification, are permitted provided that the following conditions
7	* are met:	7	* are met:
8	*	8	*
9	* - Redistributions of source code must retain the above copyright	9	* - Redistributions of source code must retain the above copyright
10	* notice, this list of conditions and the following disclaimer.	10	* notice, this list of conditions and the following disclaimer.
11	* - Redistributions in binary form must reproduce the above copyright	11	* - Redistributions in binary form must reproduce the above copyright
12	* notice, this list of conditions and the following disclaimer in the	12	* notice, this list of conditions and the following disclaimer in the
13	* documentation and/or other materials provided with the distribution.	13	* documentation and/or other materials provided with the distribution.
14	* - The name of the author may not be used to endorse or promote products	14	* - The name of the author may not be used to endorse or promote products
15	* derived from this software without specific prior written permission.	15	* derived from this software without specific prior written permission.
16	*	16	*
17	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR	17	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES	18	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.	19	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,	20	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT	21	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,	22	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY	23	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT	24	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF	25	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.	26	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27	*/	27	*/
28		28
29	#include <proc/scheduler.h>	29	#include <proc/scheduler.h>
30	#include <proc/thread.h>	30	#include <proc/thread.h>
31	#include <proc/task.h>	31	#include <proc/task.h>
32	#include <mm/heap.h>	32	#include <mm/heap.h>
33	#include <mm/frame.h>	33	#include <mm/frame.h>
34	#include <mm/page.h>	34	#include <mm/page.h>
35	#include <mm/vm.h>	35	#include <mm/vm.h>
36	#include <arch/asm.h>	36	#include <arch/asm.h>
37	#include <arch/faddr.h>	37	#include <arch/faddr.h>
38	#include <arch/atomic.h>	38	#include <arch/atomic.h>
39	#include <synch/spinlock.h>	39	#include <synch/spinlock.h>
40	#include <config.h>	40	#include <config.h>
41	#include <context.h>	41	#include <context.h>
42	#include <func.h>	42	#include <func.h>
43	#include <arch.h>	43	#include <arch.h>
44	#include <list.h>	44	#include <list.h>
45	#include <panic.h>	45	#include <panic.h>
46	#include <typedefs.h>	46	#include <typedefs.h>
47	#include <cpu.h>	47	#include <cpu.h>
48	#include <print.h>	48	#include <print.h>
49	#include <debug.h>	49	#include <debug.h>
50		50
51	volatile count_t nrdy;	51	volatile count_t nrdy;
52		52
53		53
54	/** Take actions before new thread runs	54	/** Take actions before new thread runs
55	*	55	*
56	* Perform actions that need to be	56	* Perform actions that need to be
57	* taken before the newly selected	57	* taken before the newly selected
58	* tread is passed control.	58	* tread is passed control.
59	*	59	*
60	*/	60	*/
61	void before_thread_runs(void)	61	void before_thread_runs(void)
62	{	62	{
63	before_thread_runs_arch();	63	before_thread_runs_arch();
64	#ifdef FPU_LAZY	64	#ifdef FPU_LAZY
65	if(THREAD==CPU->fpu_owner)	65	if(THREAD==CPU->fpu_owner)
66	fpu_enable();	66	fpu_enable();
67	else	67	else
68	fpu_disable();	68	fpu_disable();
69	#else	69	#else
70	fpu_enable();	70	fpu_enable();
71	if (THREAD->fpu_context_exists)	71	if (THREAD->fpu_context_exists)
72	fpu_context_restore(&(THREAD->saved_fpu_context));	72	fpu_context_restore(&(THREAD->saved_fpu_context));
73	else {	73	else {
74	fpu_init();	74	fpu_init();
75	THREAD->fpu_context_exists=1;	75	THREAD->fpu_context_exists=1;
76	}	76	}
77	#endif	77	#endif
78	}	78	}
79		79
80	#ifdef FPU_LAZY	80	#ifdef FPU_LAZY
81	void scheduler_fpu_lazy_request(void)	81	void scheduler_fpu_lazy_request(void)
82	{	82	{
83	fpu_enable();	83	fpu_enable();
84	if (CPU->fpu_owner != NULL) {	84	if (CPU->fpu_owner != NULL) {
85	fpu_context_save(&CPU->fpu_owner->saved_fpu_context);	85	fpu_context_save(&CPU->fpu_owner->saved_fpu_context);
86	/* don't prevent migration */	86	/* don't prevent migration */
87	CPU->fpu_owner->fpu_context_engaged=0;	87	CPU->fpu_owner->fpu_context_engaged=0;
88	}	88	}
89	if (THREAD->fpu_context_exists)	89	if (THREAD->fpu_context_exists)
90	fpu_context_restore(&THREAD->saved_fpu_context);	90	fpu_context_restore(&THREAD->saved_fpu_context);
91	else {	91	else {
92	fpu_init();	92	fpu_init();
93	THREAD->fpu_context_exists=1;	93	THREAD->fpu_context_exists=1;
94	}	94	}
95	CPU->fpu_owner=THREAD;	95	CPU->fpu_owner=THREAD;
96	THREAD->fpu_context_engaged = 1;	96	THREAD->fpu_context_engaged = 1;
97	}	97	}
98	#endif	98	#endif
99		99
100	/** Initialize scheduler	100	/** Initialize scheduler
101	*	101	*
102	* Initialize kernel scheduler.	102	* Initialize kernel scheduler.
103	*	103	*
104	*/	104	*/
105	void scheduler_init(void)	105	void scheduler_init(void)
106	{	106	{
107	}	107	}
108		108
109		109
110	/** Get thread to be scheduled	110	/** Get thread to be scheduled
111	*	111	*
112	* Get the optimal thread to be scheduled	112	* Get the optimal thread to be scheduled
113	* according to thread accounting and scheduler	113	* according to thread accounting and scheduler
114	* policy.	114	* policy.
115	*	115	*
116	* @return Thread to be scheduled.	116	* @return Thread to be scheduled.
117	*	117	*
118	*/	118	*/
119	struct thread *find_best_thread(void)	119	struct thread *find_best_thread(void)
120	{	120	{
121	thread_t *t;	121	thread_t *t;
122	runq_t *r;	122	runq_t *r;
123	int i, n;	123	int i, n;
124		124
125	ASSERT(CPU != NULL);	125	ASSERT(CPU != NULL);
126		126
127	loop:	127	loop:
128	interrupts_disable();	128	interrupts_disable();
129		129
130	spinlock_lock(&CPU->lock);	130	spinlock_lock(&CPU->lock);
131	n = CPU->nrdy;	131	n = CPU->nrdy;
132	spinlock_unlock(&CPU->lock);	132	spinlock_unlock(&CPU->lock);
133		133
134	interrupts_enable();	134	interrupts_enable();
135		135
136	if (n == 0) {	136	if (n == 0) {
137	#ifdef __SMP__	137	#ifdef __SMP__
138	/*	138	/*
139	* If the load balancing thread is not running, wake it up and	139	* If the load balancing thread is not running, wake it up and
140	* set CPU-private flag that the kcpulb has been started.	140	* set CPU-private flag that the kcpulb has been started.
141	*/	141	*/
142	if (test_and_set(&CPU->kcpulbstarted) == 0) {	142	if (test_and_set(&CPU->kcpulbstarted) == 0) {
143	waitq_wakeup(&CPU->kcpulb_wq, 0);	143	waitq_wakeup(&CPU->kcpulb_wq, 0);
144	goto loop;	144	goto loop;
145	}	145	}
146	#endif /* __SMP__ */	146	#endif /* __SMP__ */
147		147
148	/*	148	/*
149	* For there was nothing to run, the CPU goes to sleep	149	* For there was nothing to run, the CPU goes to sleep
150	* until a hardware interrupt or an IPI comes.	150	* until a hardware interrupt or an IPI comes.
151	* This improves energy saving and hyperthreading.	151	* This improves energy saving and hyperthreading.
152	* On the other hand, several hardware interrupts can be ignored.	152	* On the other hand, several hardware interrupts can be ignored.
153	*/	153	*/
154	cpu_sleep();	154	cpu_sleep();
155	goto loop;	155	goto loop;
156	}	156	}
157		157
158	interrupts_disable();	158	interrupts_disable();
159		159
160	i = 0;	160	i = 0;
161	retry:	161	retry:
162	for (; i<RQ_COUNT; i++) {	162	for (; i<RQ_COUNT; i++) {
163	r = &CPU->rq[i];	163	r = &CPU->rq[i];
164	spinlock_lock(&r->lock);	164	spinlock_lock(&r->lock);
165	if (r->n == 0) {	165	if (r->n == 0) {
166	/*	166	/*
167	* If this queue is empty, try a lower-priority queue.	167	* If this queue is empty, try a lower-priority queue.
168	*/	168	*/
169	spinlock_unlock(&r->lock);	169	spinlock_unlock(&r->lock);
170	continue;	170	continue;
171	}	171	}
172		172
173	/* avoid deadlock with relink_rq() */	173	/* avoid deadlock with relink_rq() */
174	if (!spinlock_trylock(&CPU->lock)) {	174	if (!spinlock_trylock(&CPU->lock)) {
175	/*	175	/*
176	* Unlock r and try again.	176	* Unlock r and try again.
177	*/	177	*/
178	spinlock_unlock(&r->lock);	178	spinlock_unlock(&r->lock);
179	goto retry;	179	goto retry;
180	}	180	}
181	CPU->nrdy--;	181	CPU->nrdy--;
182	spinlock_unlock(&CPU->lock);	182	spinlock_unlock(&CPU->lock);
183		183
184	atomic_dec((int *) &nrdy);	184	atomic_dec((int *) &nrdy);
185	r->n--;	185	r->n--;
186		186
187	/*	187	/*
188	* Take the first thread from the queue.	188	* Take the first thread from the queue.
189	*/	189	*/
190	t = list_get_instance(r->rq_head.next, thread_t, rq_link);	190	t = list_get_instance(r->rq_head.next, thread_t, rq_link);
191	list_remove(&t->rq_link);	191	list_remove(&t->rq_link);
192		192
193	spinlock_unlock(&r->lock);	193	spinlock_unlock(&r->lock);
194		194
195	spinlock_lock(&t->lock);	195	spinlock_lock(&t->lock);
196	t->cpu = CPU;	196	t->cpu = CPU;
197		197
198	t->ticks = us2ticks((i+1)*10000);	198	t->ticks = us2ticks((i+1)*10000);
199	t->priority = i; /* eventually correct rq index */	199	t->priority = i; /* eventually correct rq index */
200		200
201	/*	201	/*
202	* Clear the X_STOLEN flag so that t can be migrated when load balancing needs emerge.	202	* Clear the X_STOLEN flag so that t can be migrated when load balancing needs emerge.
203	*/	203	*/
204	t->flags &= ~X_STOLEN;	204	t->flags &= ~X_STOLEN;
205	spinlock_unlock(&t->lock);	205	spinlock_unlock(&t->lock);
206		206
207	return t;	207	return t;
208	}	208	}
209	goto loop;	209	goto loop;
210		210
211	}	211	}
212		212
213		213
214	/** Prevent rq starvation	214	/** Prevent rq starvation
215	*	215	*
216	* Prevent low priority threads from starving in rq's.	216	* Prevent low priority threads from starving in rq's.
217	*	217	*
218	* When the function decides to relink rq's, it reconnects	218	* When the function decides to relink rq's, it reconnects
219	* respective pointers so that in result threads with 'pri'	219	* respective pointers so that in result threads with 'pri'
220	* greater or equal 'start' are moved to a higher-priority queue.	220	* greater or equal 'start' are moved to a higher-priority queue.
221	*	221	*
222	* @param start Threshold priority.	222	* @param start Threshold priority.
223	*	223	*
224	*/	224	*/
225	void relink_rq(int start)	225	void relink_rq(int start)
226	{	226	{
227	link_t head;	227	link_t head;
228	runq_t *r;	228	runq_t *r;
229	int i, n;	229	int i, n;
230		230
231	list_initialize(&head);	231	list_initialize(&head);
232	spinlock_lock(&CPU->lock);	232	spinlock_lock(&CPU->lock);
233	if (CPU->needs_relink > NEEDS_RELINK_MAX) {	233	if (CPU->needs_relink > NEEDS_RELINK_MAX) {
234	for (i = start; i<RQ_COUNT-1; i++) {	234	for (i = start; i<RQ_COUNT-1; i++) {
235	/* remember and empty rq[i + 1] */	235	/* remember and empty rq[i + 1] */
236	r = &CPU->rq[i + 1];	236	r = &CPU->rq[i + 1];
237	spinlock_lock(&r->lock);	237	spinlock_lock(&r->lock);
238	list_concat(&head, &r->rq_head);	238	list_concat(&head, &r->rq_head);
239	n = r->n;	239	n = r->n;
240	r->n = 0;	240	r->n = 0;
241	spinlock_unlock(&r->lock);	241	spinlock_unlock(&r->lock);
242		242
243	/* append rq[i + 1] to rq[i] */	243	/* append rq[i + 1] to rq[i] */
244	r = &CPU->rq[i];	244	r = &CPU->rq[i];
245	spinlock_lock(&r->lock);	245	spinlock_lock(&r->lock);
246	list_concat(&r->rq_head, &head);	246	list_concat(&r->rq_head, &head);
247	r->n += n;	247	r->n += n;
248	spinlock_unlock(&r->lock);	248	spinlock_unlock(&r->lock);
249	}	249	}
250	CPU->needs_relink = 0;	250	CPU->needs_relink = 0;
251	}	251	}
252	spinlock_unlock(&CPU->lock);	252	spinlock_unlock(&CPU->lock);
253		253
254	}	254	}
255		255
256		256
257	/** The scheduler	257	/** The scheduler
258	*	258	*
259	* The thread scheduling procedure.	259	* The thread scheduling procedure.
260	*	260	*
261	*/	261	*/
262	void scheduler(void)	262	void scheduler(void)
263	{	263	{
264	volatile ipl_t ipl;	264	volatile ipl_t ipl;
265		265
266	ASSERT(CPU != NULL);	266	ASSERT(CPU != NULL);
267		267
268	ipl = interrupts_disable();	268	ipl = interrupts_disable();
269		269
270	if (haltstate)	270	if (haltstate)
271	halt();	271	halt();
272		272
273	if (THREAD) {	273	if (THREAD) {
274	spinlock_lock(&THREAD->lock);	274	spinlock_lock(&THREAD->lock);
275	#ifndef FPU_LAZY	275	#ifndef FPU_LAZY
276	fpu_context_save(&(THREAD->saved_fpu_context));	276	fpu_context_save(&(THREAD->saved_fpu_context));
277	#endif	277	#endif
278	if (!context_save(&THREAD->saved_context)) {	278	if (!context_save(&THREAD->saved_context)) {
279	/*	279	/*
280	* This is the place where threads leave scheduler();	280	* This is the place where threads leave scheduler();
281	*/	281	*/
282	before_thread_runs();	282	before_thread_runs();
283	spinlock_unlock(&THREAD->lock);	283	spinlock_unlock(&THREAD->lock);
284	interrupts_restore(THREAD->saved_context.ipl);	284	interrupts_restore(THREAD->saved_context.ipl);
285	return;	285	return;
286	}	286	}
287		287
288	/*	288	/*
289	* Interrupt priority level of preempted thread is recorded here	289	* Interrupt priority level of preempted thread is recorded here
290	* to facilitate scheduler() invocations from interrupts_disable()'d	290	* to facilitate scheduler() invocations from interrupts_disable()'d
291	* code (e.g. waitq_sleep_timeout()).	291	* code (e.g. waitq_sleep_timeout()).
292	*/	292	*/
293	THREAD->saved_context.ipl = ipl;	293	THREAD->saved_context.ipl = ipl;
294	}	294	}
295		295
296	/*	296	/*
297	* Through the 'THE' structure, we keep track of THREAD, TASK, CPU	297	* Through the 'THE' structure, we keep track of THREAD, TASK, CPU
298	* and preemption counter. At this point THE could be coming either	298	* and preemption counter. At this point THE could be coming either
299	* from THREAD's or CPU's stack.	299	* from THREAD's or CPU's stack.
300	*/	300	*/
301	the_copy(THE, (the_t *) CPU->stack);	301	the_copy(THE, (the_t *) CPU->stack);
302		302
303	/*	303	/*
304	* We may not keep the old stack.	304	* We may not keep the old stack.
305	* Reason: If we kept the old stack and got blocked, for instance, in	305	* Reason: If we kept the old stack and got blocked, for instance, in
306	* find_best_thread(), the old thread could get rescheduled by another	306	* find_best_thread(), the old thread could get rescheduled by another
307	* CPU and overwrite the part of its own stack that was also used by	307	* CPU and overwrite the part of its own stack that was also used by
308	* the scheduler on this CPU.	308	* the scheduler on this CPU.
309	*	309	*
310	* Moreover, we have to bypass the compiler-generated POP sequence	310	* Moreover, we have to bypass the compiler-generated POP sequence
311	* which is fooled by SP being set to the very top of the stack.	311	* which is fooled by SP being set to the very top of the stack.
312	* Therefore the scheduler() function continues in	312	* Therefore the scheduler() function continues in
313	* scheduler_separated_stack().	313	* scheduler_separated_stack().
314	*/	314	*/
315	context_save(&CPU->saved_context);	315	context_save(&CPU->saved_context);
316	context_set(&CPU->saved_context, FADDR(scheduler_separated_stack), CPU->stack, CPU_STACK_SIZE);	316	context_set(&CPU->saved_context, FADDR(scheduler_separated_stack), (__address) CPU->stack, CPU_STACK_SIZE);
317	context_restore(&CPU->saved_context);	317	context_restore(&CPU->saved_context);
318	/* not reached */	318	/* not reached */
319	}	319	}
320		320
321		321
322	/** Scheduler stack switch wrapper	322	/** Scheduler stack switch wrapper
323	*	323	*
324	* Second part of the scheduler() function	324	* Second part of the scheduler() function
325	* using new stack. Handling the actual context	325	* using new stack. Handling the actual context
326	* switch to a new thread.	326	* switch to a new thread.
327	*	327	*
328	*/	328	*/
329	void scheduler_separated_stack(void)	329	void scheduler_separated_stack(void)
330	{	330	{
331	int priority;	331	int priority;
332		332
333	ASSERT(CPU != NULL);	333	ASSERT(CPU != NULL);
334		334
335	if (THREAD) {	335	if (THREAD) {
336	switch (THREAD->state) {	336	switch (THREAD->state) {
337	case Running:	337	case Running:
338	THREAD->state = Ready;	338	THREAD->state = Ready;
339	spinlock_unlock(&THREAD->lock);	339	spinlock_unlock(&THREAD->lock);
340	thread_ready(THREAD);	340	thread_ready(THREAD);
341	break;	341	break;
342		342
343	case Exiting:	343	case Exiting:
344	frame_free((__address) THREAD->kstack);	344	frame_free((__address) THREAD->kstack);
345	if (THREAD->ustack) {	345	if (THREAD->ustack) {
346	frame_free((__address) THREAD->ustack);	346	frame_free((__address) THREAD->ustack);
347	}	347	}
348		348
349	/*	349	/*
350	* Detach from the containing task.	350	* Detach from the containing task.
351	*/	351	*/
352	spinlock_lock(&TASK->lock);	352	spinlock_lock(&TASK->lock);
353	list_remove(&THREAD->th_link);	353	list_remove(&THREAD->th_link);
354	spinlock_unlock(&TASK->lock);	354	spinlock_unlock(&TASK->lock);
355		355
356	spinlock_unlock(&THREAD->lock);	356	spinlock_unlock(&THREAD->lock);
357		357
358	spinlock_lock(&threads_lock);	358	spinlock_lock(&threads_lock);
359	list_remove(&THREAD->threads_link);	359	list_remove(&THREAD->threads_link);
360	spinlock_unlock(&threads_lock);	360	spinlock_unlock(&threads_lock);
361		361
362	spinlock_lock(&CPU->lock);	362	spinlock_lock(&CPU->lock);
363	if(CPU->fpu_owner==THREAD) CPU->fpu_owner=NULL;	363	if(CPU->fpu_owner==THREAD) CPU->fpu_owner=NULL;
364	spinlock_unlock(&CPU->lock);	364	spinlock_unlock(&CPU->lock);
365		365
366	free(THREAD);	366	free(THREAD);
367		367
368	break;	368	break;
369		369
370	case Sleeping:	370	case Sleeping:
371	/*	371	/*
372	* Prefer the thread after it's woken up.	372	* Prefer the thread after it's woken up.
373	*/	373	*/
374	THREAD->priority = -1;	374	THREAD->priority = -1;
375		375
376	/*	376	/*
377	* We need to release wq->lock which we locked in waitq_sleep().	377	* We need to release wq->lock which we locked in waitq_sleep().
378	* Address of wq->lock is kept in THREAD->sleep_queue.	378	* Address of wq->lock is kept in THREAD->sleep_queue.
379	*/	379	*/
380	spinlock_unlock(&THREAD->sleep_queue->lock);	380	spinlock_unlock(&THREAD->sleep_queue->lock);
381		381
382	/*	382	/*
383	* Check for possible requests for out-of-context invocation.	383	* Check for possible requests for out-of-context invocation.
384	*/	384	*/
385	if (THREAD->call_me) {	385	if (THREAD->call_me) {
386	THREAD->call_me(THREAD->call_me_with);	386	THREAD->call_me(THREAD->call_me_with);
387	THREAD->call_me = NULL;	387	THREAD->call_me = NULL;
388	THREAD->call_me_with = NULL;	388	THREAD->call_me_with = NULL;
389	}	389	}
390		390
391	spinlock_unlock(&THREAD->lock);	391	spinlock_unlock(&THREAD->lock);
392		392
393	break;	393	break;
394		394
395	default:	395	default:
396	/*	396	/*
397	* Entering state is unexpected.	397	* Entering state is unexpected.
398	*/	398	*/
399	panic("tid%d: unexpected state %s\n", THREAD->tid, thread_states[THREAD->state]);	399	panic("tid%d: unexpected state %s\n", THREAD->tid, thread_states[THREAD->state]);
400	break;	400	break;
401	}	401	}
402	THREAD = NULL;	402	THREAD = NULL;
403	}	403	}
404		404
405		405
406	THREAD = find_best_thread();	406	THREAD = find_best_thread();
407		407
408	spinlock_lock(&THREAD->lock);	408	spinlock_lock(&THREAD->lock);
409	priority = THREAD->priority;	409	priority = THREAD->priority;
410	spinlock_unlock(&THREAD->lock);	410	spinlock_unlock(&THREAD->lock);
411		411
412	relink_rq(priority);	412	relink_rq(priority);
413		413
414	spinlock_lock(&THREAD->lock);	414	spinlock_lock(&THREAD->lock);
415		415
416	/*	416	/*
417	* If both the old and the new task are the same, lots of work is avoided.	417	* If both the old and the new task are the same, lots of work is avoided.
418	*/	418	*/
419	if (TASK != THREAD->task) {	419	if (TASK != THREAD->task) {
420	vm_t *m1 = NULL;	420	vm_t *m1 = NULL;
421	vm_t *m2;	421	vm_t *m2;
422		422
423	if (TASK) {	423	if (TASK) {
424	spinlock_lock(&TASK->lock);	424	spinlock_lock(&TASK->lock);
425	m1 = TASK->vm;	425	m1 = TASK->vm;
426	spinlock_unlock(&TASK->lock);	426	spinlock_unlock(&TASK->lock);
427	}	427	}
428		428
429	spinlock_lock(&THREAD->task->lock);	429	spinlock_lock(&THREAD->task->lock);
430	m2 = THREAD->task->vm;	430	m2 = THREAD->task->vm;
431	spinlock_unlock(&THREAD->task->lock);	431	spinlock_unlock(&THREAD->task->lock);
432		432
433	/*	433	/*
434	* Note that it is possible for two tasks to share one vm mapping.	434	* Note that it is possible for two tasks to share one vm mapping.
435	*/	435	*/
436	if (m1 != m2) {	436	if (m1 != m2) {
437	/*	437	/*
438	* Both tasks and vm mappings are different.	438	* Both tasks and vm mappings are different.
439	* Replace the old one with the new one.	439	* Replace the old one with the new one.
440	*/	440	*/
441	vm_install(m2);	441	vm_install(m2);
442	}	442	}
443	TASK = THREAD->task;	443	TASK = THREAD->task;
444	}	444	}
445		445
446	THREAD->state = Running;	446	THREAD->state = Running;
447		447
448	#ifdef SCHEDULER_VERBOSE	448	#ifdef SCHEDULER_VERBOSE
449	printf("cpu%d: tid %d (priority=%d,ticks=%d,nrdy=%d)\n", CPU->id, THREAD->tid, THREAD->priority, THREAD->ticks, CPU->nrdy);	449	printf("cpu%d: tid %d (priority=%d,ticks=%d,nrdy=%d)\n", CPU->id, THREAD->tid, THREAD->priority, THREAD->ticks, CPU->nrdy);
450	#endif	450	#endif
451		451
452	/*	452	/*
453	* Copy the knowledge of CPU, TASK, THREAD and preemption counter to thread's stack.	453	* Copy the knowledge of CPU, TASK, THREAD and preemption counter to thread's stack.
454	*/	454	*/
455	the_copy(THE, (the_t *) THREAD->kstack);	455	the_copy(THE, (the_t *) THREAD->kstack);
456		456
457	context_restore(&THREAD->saved_context);	457	context_restore(&THREAD->saved_context);
458	/* not reached */	458	/* not reached */
459	}	459	}
460		460
461		461
462	#ifdef __SMP__	462	#ifdef __SMP__
463	/** Load balancing thread	463	/** Load balancing thread
464	*	464	*
465	* SMP load balancing thread, supervising thread supplies	465	* SMP load balancing thread, supervising thread supplies
466	* for the CPU it's wired to.	466	* for the CPU it's wired to.
467	*	467	*
468	* @param arg Generic thread argument (unused).	468	* @param arg Generic thread argument (unused).
469	*	469	*
470	*/	470	*/
471	void kcpulb(void *arg)	471	void kcpulb(void *arg)
472	{	472	{
473	thread_t *t;	473	thread_t *t;
474	int count, i, j, k = 0;	474	int count, i, j, k = 0;
475	ipl_t ipl;	475	ipl_t ipl;
476		476
477	loop:	477	loop:
478	/*	478	/*
479	* Sleep until there's some work to do.	479	* Sleep until there's some work to do.
480	*/	480	*/
481	waitq_sleep(&CPU->kcpulb_wq);	481	waitq_sleep(&CPU->kcpulb_wq);
482		482
483	not_satisfied:	483	not_satisfied:
484	/*	484	/*
485	* Calculate the number of threads that will be migrated/stolen from	485	* Calculate the number of threads that will be migrated/stolen from
486	* other CPU's. Note that situation can have changed between two	486	* other CPU's. Note that situation can have changed between two
487	* passes. Each time get the most up to date counts.	487	* passes. Each time get the most up to date counts.
488	*/	488	*/
489	ipl = interrupts_disable();	489	ipl = interrupts_disable();
490	spinlock_lock(&CPU->lock);	490	spinlock_lock(&CPU->lock);
491	count = nrdy / config.cpu_active;	491	count = nrdy / config.cpu_active;
492	count -= CPU->nrdy;	492	count -= CPU->nrdy;
493	spinlock_unlock(&CPU->lock);	493	spinlock_unlock(&CPU->lock);
494	interrupts_restore(ipl);	494	interrupts_restore(ipl);
495		495
496	if (count <= 0)	496	if (count <= 0)
497	goto satisfied;	497	goto satisfied;
498		498
499	/*	499	/*
500	* Searching least priority queues on all CPU's first and most priority queues on all CPU's last.	500	* Searching least priority queues on all CPU's first and most priority queues on all CPU's last.

Subversion Repositories HelenOS

(root)/branches/sparc/kernel/generic/src/proc/scheduler.c @ 4687 – Rev 413 → 414