WebSVN – HelenOS-historic – Diff – /kernel/trunk/generic/src/proc/scheduler.c


/*
 * Copyright (C) 2001-2004 Jakub Jermar
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * - Redistributions of source code must retain the above copyright
 *   notice, this list of conditions and the following disclaimer.
 * - Redistributions in binary form must reproduce the above copyright
 *   notice, this list of conditions and the following disclaimer in the
 *   documentation and/or other materials provided with the distribution.
 * - The name of the author may not be used to endorse or promote products
 *   derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
 
#include <proc/scheduler.h>
#include <proc/thread.h>
#include <proc/task.h>
#include <cpu.h>
#include <mm/vm.h>
#include <config.h>
#include <context.h>
#include <func.h>
#include <arch.h>
#include <arch/asm.h>
#include <list.h>
#include <panic.h>
#include <typedefs.h>
#include <mm/page.h>
#include <synch/spinlock.h>
#include <arch/faddr.h>
#include <arch/atomic.h>
#include <print.h>
#include <mm/frame.h>
#include <mm/heap.h>
#include <debug.h>
 
volatile count_t nrdy;
 
 
/** Take actions before new thread runs
 *
 * Perform actions that need to be
 * taken before the newly selected
 * tread is passed control.
 *
 */
void before_thread_runs(void)
{
    before_thread_runs_arch();
    fpu_context_restore(&(THREAD->saved_fpu_context));
}
 
 
/** Initialize scheduler
 *
 * Initialize kernel scheduler.
 *
 */
void scheduler_init(void)
{
}
 
 
/** Get thread to be scheduled
 *
 * Get the optimal thread to be scheduled
 * according to thread accounting and scheduler
 * policy.
 *
 * @return Thread to be scheduled.
 *
 */
struct thread *find_best_thread(void)
{
    thread_t *t;
    runq_t *r;
    int i, n;
 
    ASSERT(CPU != NULL);
 
loop:
    cpu_priority_high();
 
    spinlock_lock(&CPU->lock);
    n = CPU->nrdy;
    spinlock_unlock(&CPU->lock);
 
    cpu_priority_low();
   
    if (n == 0) {
        #ifdef __SMP__
        /*
         * If the load balancing thread is not running, wake it up and
         * set CPU-private flag that the kcpulb has been started.
         */
        if (test_and_set(&CPU->kcpulbstarted) == 0) {
            waitq_wakeup(&CPU->kcpulb_wq, 0);
            goto loop;
        }
        #endif /* __SMP__ */
       
        /*
         * For there was nothing to run, the CPU goes to sleep
         * until a hardware interrupt or an IPI comes.
         * This improves energy saving and hyperthreading.
         * On the other hand, several hardware interrupts can be ignored.
         */
         cpu_sleep();
         goto loop;
    }
 
    cpu_priority_high();
   
    i = 0;
retry:
    for (; i<RQ_COUNT; i++) {
        r = &CPU->rq[i];
        spinlock_lock(&r->lock);
        if (r->n == 0) {
            /*
             * If this queue is empty, try a lower-priority queue.
             */
            spinlock_unlock(&r->lock);
            continue;
        }
 
        /* avoid deadlock with relink_rq() */
        if (!spinlock_trylock(&CPU->lock)) {
            /*
             * Unlock r and try again.
             */
            spinlock_unlock(&r->lock);
            goto retry;
        }
        CPU->nrdy--;
        spinlock_unlock(&CPU->lock);
 
        atomic_dec((int *) &nrdy);
        r->n--;
 
        /*
         * Take the first thread from the queue.
         */
        t = list_get_instance(r->rq_head.next, thread_t, rq_link);
        list_remove(&t->rq_link);
 
        spinlock_unlock(&r->lock);
 
        spinlock_lock(&t->lock);
        t->cpu = CPU;
 
        t->ticks = us2ticks((i+1)*10000);
        t->pri = i; /* eventually correct rq index */
 
        /*
         * Clear the X_STOLEN flag so that t can be migrated when load balancing needs emerge.
         */
        t->flags &= ~X_STOLEN;
        spinlock_unlock(&t->lock);
 
        return t;
    }
    goto loop;
 
}
 
 
/** Prevent rq starvation
 *
 * Prevent low priority threads from starving in rq's.
 *
 * When the function decides to relink rq's, it reconnects
 * respective pointers so that in result threads with 'pri'
 * greater or equal 'start' are moved to a higher-priority queue.
 *
 * @param start Threshold priority.
 *
 */
void relink_rq(int start)
{
    link_t head;
    runq_t *r;
    int i, n;
 
    list_initialize(&head);
    spinlock_lock(&CPU->lock);
    if (CPU->needs_relink > NEEDS_RELINK_MAX) {
        for (i = start; i<RQ_COUNT-1; i++) {
            /* remember and empty rq[i + 1] */
            r = &CPU->rq[i + 1];
            spinlock_lock(&r->lock);
            list_concat(&head, &r->rq_head);
            n = r->n;
            r->n = 0;
            spinlock_unlock(&r->lock);
       
            /* append rq[i + 1] to rq[i] */
            r = &CPU->rq[i];
            spinlock_lock(&r->lock);
            list_concat(&r->rq_head, &head);
            r->n += n;
            spinlock_unlock(&r->lock);
        }
        CPU->needs_relink = 0;
    }
    spinlock_unlock(&CPU->lock);               
 
}
 
 
/** The scheduler
 *
 * The thread scheduling procedure.
 *
 */
void scheduler(void)
{
    volatile pri_t pri;
 
    ASSERT(CPU != NULL);
 
    pri = cpu_priority_high();
 
    if (haltstate)
        halt();
 
    if (THREAD) {
        spinlock_lock(&THREAD->lock);
        fpu_context_save(&(THREAD->saved_fpu_context));
        if (!context_save(&THREAD->saved_context)) {
            /*
             * This is the place where threads leave scheduler();
             */
            before_thread_runs();
            spinlock_unlock(&THREAD->lock);
            cpu_priority_restore(THREAD->saved_context.pri);
            return;
        }
 
        /*
         * CPU priority of preempted thread is recorded here
         * to facilitate scheduler() invocations from
         * cpu_priority_high()'ed code (e.g. waitq_sleep_timeout()).
         */
        THREAD->saved_context.pri = pri;
    }
 
    /*
     * Through the 'THE' structure, we keep track of THREAD, TASK, CPU
     * and preemption counter. At this point THE could be coming either
     * from THREAD's or CPU's stack.
     */
    the_copy(THE, (the_t *) CPU->stack);
 
    /*
     * We may not keep the old stack.
     * Reason: If we kept the old stack and got blocked, for instance, in
     * find_best_thread(), the old thread could get rescheduled by another
     * CPU and overwrite the part of its own stack that was also used by
     * the scheduler on this CPU.
     *
     * Moreover, we have to bypass the compiler-generated POP sequence
     * which is fooled by SP being set to the very top of the stack.
     * Therefore the scheduler() function continues in
     * scheduler_separated_stack().
     */
    context_save(&CPU->saved_context);
    context_set(&CPU->saved_context, FADDR(scheduler_separated_stack), CPU->stack, CPU_STACK_SIZE);
    context_restore(&CPU->saved_context);
    /* not reached */
}
 
 
/** Scheduler stack switch wrapper
 *
 * Second part of the scheduler() function
 * using new stack. Handling the actual context
 * switch to a new thread.
 *
 */
void scheduler_separated_stack(void)
{
    int priority;
 
    ASSERT(CPU != NULL);
 
    if (THREAD) {
        switch (THREAD->state) {
            case Running:
            THREAD->state = Ready;
            spinlock_unlock(&THREAD->lock);
            thread_ready(THREAD);
            break;
 
            case Exiting:
            frame_free((__address) THREAD->kstack);
            if (THREAD->ustack) {
                frame_free((__address) THREAD->ustack);
            }
 
            /*
             * Detach from the containing task.
             */
            spinlock_lock(&TASK->lock);
            list_remove(&THREAD->th_link);
            spinlock_unlock(&TASK->lock);
 
            spinlock_unlock(&THREAD->lock);
   
            spinlock_lock(&threads_lock);
            list_remove(&THREAD->threads_link);
            spinlock_unlock(&threads_lock);
 
            spinlock_lock(&CPU->lock);
            if(CPU->fpu_owner==THREAD) CPU->fpu_owner=NULL;
            spinlock_unlock(&CPU->lock);
 
            free(THREAD);
 
            break;
   
            case Sleeping:
            /*
             * Prefer the thread after it's woken up.
             */
            THREAD->pri = -1;
 
            /*
             * We need to release wq->lock which we locked in waitq_sleep().
             * Address of wq->lock is kept in THREAD->sleep_queue.
             */
            spinlock_unlock(&THREAD->sleep_queue->lock);
 
            /*
             * Check for possible requests for out-of-context invocation.
             */
            if (THREAD->call_me) {
                THREAD->call_me(THREAD->call_me_with);
                THREAD->call_me = NULL;
                THREAD->call_me_with = NULL;
            }
 
            spinlock_unlock(&THREAD->lock);
 
            break;
 
            default:
            /*
             * Entering state is unexpected.
             */
            panic("tid%d: unexpected state %s\n", THREAD->tid, thread_states[THREAD->state]);
            break;
        }
        THREAD = NULL;
    }
 
 
    THREAD = find_best_thread();
   
    spinlock_lock(&THREAD->lock);
    priority = THREAD->pri;
    spinlock_unlock(&THREAD->lock);
 
    relink_rq(priority);       
 
    spinlock_lock(&THREAD->lock);  
 
    /*
     * If both the old and the new task are the same, lots of work is avoided.
     */
    if (TASK != THREAD->task) {
        vm_t *m1 = NULL;
        vm_t *m2;
 
        if (TASK) {
            spinlock_lock(&TASK->lock);
            m1 = TASK->vm;
            spinlock_unlock(&TASK->lock);
        }
 
        spinlock_lock(&THREAD->task->lock);
        m2 = THREAD->task->vm;
        spinlock_unlock(&THREAD->task->lock);
       
        /*
         * Note that it is possible for two tasks to share one vm mapping.
         */
        if (m1 != m2) {
            /*
             * Both tasks and vm mappings are different.
             * Replace the old one with the new one.
             */
            vm_install(m2);
        }
        TASK = THREAD->task;   
    }
 
    THREAD->state = Running;
 
    #ifdef SCHEDULER_VERBOSE
    printf("cpu%d: tid %d (pri=%d,ticks=%d,nrdy=%d)\n", CPU->id, THREAD->tid, THREAD->pri, THREAD->ticks, CPU->nrdy);
    #endif  
 
    /*
     * Copy the knowledge of CPU, TASK, THREAD and preemption counter to thread's stack.
     */
    the_copy(THE, (the_t *) THREAD->kstack);
   
    context_restore(&THREAD->saved_context);
    /* not reached */
}
 
 
#ifdef __SMP__
/** Load balancing thread
 *
 * SMP load balancing thread, supervising thread supplies
 * for the CPU it's wired to.
 *
 * @param arg Generic thread argument (unused).
 *
 */
void kcpulb(void *arg)
{
    thread_t *t;
    int count, i, j, k = 0;
    pri_t pri;
 
loop:
    /*
     * Sleep until there's some work to do.
     */
    waitq_sleep(&CPU->kcpulb_wq);
 
not_satisfied:
    /*
     * Calculate the number of threads that will be migrated/stolen from
     * other CPU's. Note that situation can have changed between two
     * passes. Each time get the most up to date counts.
     */
    pri = cpu_priority_high();
    spinlock_lock(&CPU->lock);
    count = nrdy / config.cpu_active;
    count -= CPU->nrdy;
    spinlock_unlock(&CPU->lock);
    cpu_priority_restore(pri);
 
    if (count <= 0)
        goto satisfied;
 
    /*
     * Searching least priority queues on all CPU's first and most priority queues on all CPU's last.
     */
    for (j=RQ_COUNT-1; j >= 0; j--) {
        for (i=0; i < config.cpu_active; i++) {
            link_t *l;
            runq_t *r;
            cpu_t *cpu;
 
            cpu = &cpus[(i + k) % config.cpu_active];
 
            /*
             * Not interested in ourselves.
             * Doesn't require interrupt disabling for kcpulb is X_WIRED.
             */
            if (CPU == cpu)
                continue;              
 
restart:        pri = cpu_priority_high();
            r = &cpu->rq[j];
            spinlock_lock(&r->lock);
            if (r->n == 0) {
                spinlock_unlock(&r->lock);
                cpu_priority_restore(pri);
                continue;
            }
       
            t = NULL;
            l = r->rq_head.prev;    /* search rq from the back */
            while (l != &r->rq_head) {
                t = list_get_instance(l, thread_t, rq_link);
                /*
                 * We don't want to steal CPU-wired threads neither threads already stolen.
                 * The latter prevents threads from migrating between CPU's without ever being run.
                 * We don't want to steal threads whose FPU context is still in CPU.
                 */
                spinlock_lock(&t->lock);
                if ( (!(t->flags & (X_WIRED | X_STOLEN))) && (!(t->fpu_context_engaged)) ) {
               
                    /*
                     * Remove t from r.
                     */
 
                    spinlock_unlock(&t->lock);
                   
                    /*
                     * Here we have to avoid deadlock with relink_rq(),
                     * because it locks cpu and r in a different order than we do.
                     */
                    if (!spinlock_trylock(&cpu->lock)) {
                        /* Release all locks and try again. */
                        spinlock_unlock(&r->lock);
                        cpu_priority_restore(pri);
                        goto restart;
                    }
                    cpu->nrdy--;
                    spinlock_unlock(&cpu->lock);
 
                    atomic_dec(&nrdy);
 
                    r->n--;
                    list_remove(&t->rq_link);
 
                    break;
                }
                spinlock_unlock(&t->lock);
                l = l->prev;
                t = NULL;
            }
            spinlock_unlock(&r->lock);
 
            if (t) {
                /*
                 * Ready t on local CPU
                 */
                spinlock_lock(&t->lock);
                #ifdef KCPULB_VERBOSE
                printf("kcpulb%d: TID %d -> cpu%d, nrdy=%d, avg=%d\n", CPU->id, t->tid, CPU->id, CPU->nrdy, nrdy / config.cpu_active);
                #endif
                t->flags |= X_STOLEN;
                spinlock_unlock(&t->lock);
   
                thread_ready(t);
 
                cpu_priority_restore(pri);
   
                if (--count == 0)
                    goto satisfied;
                   
                /*
                 * We are not satisfied yet, focus on another CPU next time.
                 */
                k++;
               
                continue;
            }
            cpu_priority_restore(pri);
        }
    }
 
    if (CPU->nrdy) {
        /*
         * Be a little bit light-weight and let migrated threads run.
         */
        scheduler();
    }
    else {
        /*
         * We failed to migrate a single thread.
         * Something more sophisticated should be done.
         */
        scheduler();
    }
       
    goto not_satisfied;
 
satisfied:
    /*
     * Tell find_best_thread() to wake us up later again.
     */
    CPU->kcpulbstarted = 0;
    goto loop;
}
 
#endif /* __SMP__ */
 

Rev 241	Rev 248
1	/*	1	/*
2	* Copyright (C) 2001-2004 Jakub Jermar	2	* Copyright (C) 2001-2004 Jakub Jermar
3	* All rights reserved.	3	* All rights reserved.
4	*	4	*
5	* Redistribution and use in source and binary forms, with or without	5	* Redistribution and use in source and binary forms, with or without
6	* modification, are permitted provided that the following conditions	6	* modification, are permitted provided that the following conditions
7	* are met:	7	* are met:
8	*	8	*
9	* - Redistributions of source code must retain the above copyright	9	* - Redistributions of source code must retain the above copyright
10	* notice, this list of conditions and the following disclaimer.	10	* notice, this list of conditions and the following disclaimer.
11	* - Redistributions in binary form must reproduce the above copyright	11	* - Redistributions in binary form must reproduce the above copyright
12	* notice, this list of conditions and the following disclaimer in the	12	* notice, this list of conditions and the following disclaimer in the
13	* documentation and/or other materials provided with the distribution.	13	* documentation and/or other materials provided with the distribution.
14	* - The name of the author may not be used to endorse or promote products	14	* - The name of the author may not be used to endorse or promote products
15	* derived from this software without specific prior written permission.	15	* derived from this software without specific prior written permission.
16	*	16	*
17	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR	17	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES	18	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.	19	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,	20	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT	21	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,	22	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY	23	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT	24	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF	25	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.	26	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27	*/	27	*/
28		28
29	#include <proc/scheduler.h>	29	#include <proc/scheduler.h>
30	#include <proc/thread.h>	30	#include <proc/thread.h>
31	#include <proc/task.h>	31	#include <proc/task.h>
32	#include <cpu.h>	32	#include <cpu.h>
33	#include <mm/vm.h>	33	#include <mm/vm.h>
34	#include <config.h>	34	#include <config.h>
35	#include <context.h>	35	#include <context.h>
36	#include <func.h>	36	#include <func.h>
37	#include <arch.h>	37	#include <arch.h>
38	#include <arch/asm.h>	38	#include <arch/asm.h>
39	#include <list.h>	39	#include <list.h>
40	#include <panic.h>	40	#include <panic.h>
41	#include <typedefs.h>	41	#include <typedefs.h>
42	#include <mm/page.h>	42	#include <mm/page.h>
43	#include <synch/spinlock.h>	43	#include <synch/spinlock.h>
44	#include <arch/faddr.h>	44	#include <arch/faddr.h>
45	#include <arch/atomic.h>	45	#include <arch/atomic.h>
46	#include <print.h>	46	#include <print.h>
47	#include <mm/frame.h>	47	#include <mm/frame.h>
48	#include <mm/heap.h>	48	#include <mm/heap.h>
49	#include <debug.h>	49	#include <debug.h>
50		50
51	volatile count_t nrdy;	51	volatile count_t nrdy;
52		52
53		53
54	/** Take actions before new thread runs	54	/** Take actions before new thread runs
55	*	55	*
56	* Perform actions that need to be	56	* Perform actions that need to be
57	* taken before the newly selected	57	* taken before the newly selected
58	* tread is passed control.	58	* tread is passed control.
59	*	59	*
60	*/	60	*/
61	void before_thread_runs(void)	61	void before_thread_runs(void)
62	{	62	{
63	before_thread_runs_arch();	63	before_thread_runs_arch();
64	fpu_context_restore(&(THREAD->saved_fpu_context));	64	fpu_context_restore(&(THREAD->saved_fpu_context));
65	}	65	}
66		66
67		67
68	/** Initialize scheduler	68	/** Initialize scheduler
69	*	69	*
70	* Initialize kernel scheduler.	70	* Initialize kernel scheduler.
71	*	71	*
72	*/	72	*/
73	void scheduler_init(void)	73	void scheduler_init(void)
74	{	74	{
75	}	75	}
76		76
77		77
78	/** Get thread to be scheduled	78	/** Get thread to be scheduled
79	*	79	*
80	* Get the optimal thread to be scheduled	80	* Get the optimal thread to be scheduled
81	* according to thread accounting and scheduler	81	* according to thread accounting and scheduler
82	* policy.	82	* policy.
83	*	83	*
84	* @return Thread to be scheduled.	84	* @return Thread to be scheduled.
85	*	85	*
86	*/	86	*/
87	struct thread *find_best_thread(void)	87	struct thread *find_best_thread(void)
88	{	88	{
89	thread_t *t;	89	thread_t *t;
90	runq_t *r;	90	runq_t *r;
91	int i, n;	91	int i, n;
92		92
93	ASSERT(CPU != NULL);	93	ASSERT(CPU != NULL);
94		94
95	loop:	95	loop:
96	cpu_priority_high();	96	cpu_priority_high();
97		97
98	spinlock_lock(&CPU->lock);	98	spinlock_lock(&CPU->lock);
99	n = CPU->nrdy;	99	n = CPU->nrdy;
100	spinlock_unlock(&CPU->lock);	100	spinlock_unlock(&CPU->lock);
101		101
102	cpu_priority_low();	102	cpu_priority_low();
103		103
104	if (n == 0) {	104	if (n == 0) {
105	#ifdef __SMP__	105	#ifdef __SMP__
106	/*	106	/*
107	* If the load balancing thread is not running, wake it up and	107	* If the load balancing thread is not running, wake it up and
108	* set CPU-private flag that the kcpulb has been started.	108	* set CPU-private flag that the kcpulb has been started.
109	*/	109	*/
110	if (test_and_set(&CPU->kcpulbstarted) == 0) {	110	if (test_and_set(&CPU->kcpulbstarted) == 0) {
111	waitq_wakeup(&CPU->kcpulb_wq, 0);	111	waitq_wakeup(&CPU->kcpulb_wq, 0);
112	goto loop;	112	goto loop;
113	}	113	}
114	#endif /* __SMP__ */	114	#endif /* __SMP__ */
115		115
116	/*	116	/*
117	* For there was nothing to run, the CPU goes to sleep	117	* For there was nothing to run, the CPU goes to sleep
118	* until a hardware interrupt or an IPI comes.	118	* until a hardware interrupt or an IPI comes.
119	* This improves energy saving and hyperthreading.	119	* This improves energy saving and hyperthreading.
120	* On the other hand, several hardware interrupts can be ignored.	120	* On the other hand, several hardware interrupts can be ignored.
121	*/	121	*/
122	cpu_sleep();	122	cpu_sleep();
123	goto loop;	123	goto loop;
124	}	124	}
125		125
126	cpu_priority_high();	126	cpu_priority_high();
127		127
128	i = 0;	128	i = 0;
129	retry:	129	retry:
130	for (; i<RQ_COUNT; i++) {	130	for (; i<RQ_COUNT; i++) {
131	r = &CPU->rq[i];	131	r = &CPU->rq[i];
132	spinlock_lock(&r->lock);	132	spinlock_lock(&r->lock);
133	if (r->n == 0) {	133	if (r->n == 0) {
134	/*	134	/*
135	* If this queue is empty, try a lower-priority queue.	135	* If this queue is empty, try a lower-priority queue.
136	*/	136	*/
137	spinlock_unlock(&r->lock);	137	spinlock_unlock(&r->lock);
138	continue;	138	continue;
139	}	139	}
140		140
141	/* avoid deadlock with relink_rq() */	141	/* avoid deadlock with relink_rq() */
142	if (!spinlock_trylock(&CPU->lock)) {	142	if (!spinlock_trylock(&CPU->lock)) {
143	/*	143	/*
144	* Unlock r and try again.	144	* Unlock r and try again.
145	*/	145	*/
146	spinlock_unlock(&r->lock);	146	spinlock_unlock(&r->lock);
147	goto retry;	147	goto retry;
148	}	148	}
149	CPU->nrdy--;	149	CPU->nrdy--;
150	spinlock_unlock(&CPU->lock);	150	spinlock_unlock(&CPU->lock);
151		151
152	atomic_dec(&nrdy);	152	atomic_dec((int *) &nrdy);
153	r->n--;	153	r->n--;
154		154
155	/*	155	/*
156	* Take the first thread from the queue.	156	* Take the first thread from the queue.
157	*/	157	*/
158	t = list_get_instance(r->rq_head.next, thread_t, rq_link);	158	t = list_get_instance(r->rq_head.next, thread_t, rq_link);
159	list_remove(&t->rq_link);	159	list_remove(&t->rq_link);
160		160
161	spinlock_unlock(&r->lock);	161	spinlock_unlock(&r->lock);
162		162
163	spinlock_lock(&t->lock);	163	spinlock_lock(&t->lock);
164	t->cpu = CPU;	164	t->cpu = CPU;
165		165
166	t->ticks = us2ticks((i+1)*10000);	166	t->ticks = us2ticks((i+1)*10000);
167	t->pri = i; /* eventually correct rq index */	167	t->pri = i; /* eventually correct rq index */
168		168
169	/*	169	/*
170	* Clear the X_STOLEN flag so that t can be migrated when load balancing needs emerge.	170	* Clear the X_STOLEN flag so that t can be migrated when load balancing needs emerge.
171	*/	171	*/
172	t->flags &= ~X_STOLEN;	172	t->flags &= ~X_STOLEN;
173	spinlock_unlock(&t->lock);	173	spinlock_unlock(&t->lock);
174		174
175	return t;	175	return t;
176	}	176	}
177	goto loop;	177	goto loop;
178		178
179	}	179	}
180		180
181		181
182	/** Prevent rq starvation	182	/** Prevent rq starvation
183	*	183	*
184	* Prevent low priority threads from starving in rq's.	184	* Prevent low priority threads from starving in rq's.
185	*	185	*
186	* When the function decides to relink rq's, it reconnects	186	* When the function decides to relink rq's, it reconnects
187	* respective pointers so that in result threads with 'pri'	187	* respective pointers so that in result threads with 'pri'
188	* greater or equal 'start' are moved to a higher-priority queue.	188	* greater or equal 'start' are moved to a higher-priority queue.
189	*	189	*
190	* @param start Threshold priority.	190	* @param start Threshold priority.
191	*	191	*
192	*/	192	*/
193	void relink_rq(int start)	193	void relink_rq(int start)
194	{	194	{
195	link_t head;	195	link_t head;
196	runq_t *r;	196	runq_t *r;
197	int i, n;	197	int i, n;
198		198
199	list_initialize(&head);	199	list_initialize(&head);
200	spinlock_lock(&CPU->lock);	200	spinlock_lock(&CPU->lock);
201	if (CPU->needs_relink > NEEDS_RELINK_MAX) {	201	if (CPU->needs_relink > NEEDS_RELINK_MAX) {
202	for (i = start; i<RQ_COUNT-1; i++) {	202	for (i = start; i<RQ_COUNT-1; i++) {
203	/* remember and empty rq[i + 1] */	203	/* remember and empty rq[i + 1] */
204	r = &CPU->rq[i + 1];	204	r = &CPU->rq[i + 1];
205	spinlock_lock(&r->lock);	205	spinlock_lock(&r->lock);
206	list_concat(&head, &r->rq_head);	206	list_concat(&head, &r->rq_head);
207	n = r->n;	207	n = r->n;
208	r->n = 0;	208	r->n = 0;
209	spinlock_unlock(&r->lock);	209	spinlock_unlock(&r->lock);
210		210
211	/* append rq[i + 1] to rq[i] */	211	/* append rq[i + 1] to rq[i] */
212	r = &CPU->rq[i];	212	r = &CPU->rq[i];
213	spinlock_lock(&r->lock);	213	spinlock_lock(&r->lock);
214	list_concat(&r->rq_head, &head);	214	list_concat(&r->rq_head, &head);
215	r->n += n;	215	r->n += n;
216	spinlock_unlock(&r->lock);	216	spinlock_unlock(&r->lock);
217	}	217	}
218	CPU->needs_relink = 0;	218	CPU->needs_relink = 0;
219	}	219	}
220	spinlock_unlock(&CPU->lock);	220	spinlock_unlock(&CPU->lock);
221		221
222	}	222	}
223		223
224		224
225	/** The scheduler	225	/** The scheduler
226	*	226	*
227	* The thread scheduling procedure.	227	* The thread scheduling procedure.
228	*	228	*
229	*/	229	*/
230	void scheduler(void)	230	void scheduler(void)
231	{	231	{
232	volatile pri_t pri;	232	volatile pri_t pri;
233		233
234	ASSERT(CPU != NULL);	234	ASSERT(CPU != NULL);
235		235
236	pri = cpu_priority_high();	236	pri = cpu_priority_high();
237		237
238	if (haltstate)	238	if (haltstate)
239	halt();	239	halt();
240		240
241	if (THREAD) {	241	if (THREAD) {
242	spinlock_lock(&THREAD->lock);	242	spinlock_lock(&THREAD->lock);
243	fpu_context_save(&(THREAD->saved_fpu_context));	243	fpu_context_save(&(THREAD->saved_fpu_context));
244	if (!context_save(&THREAD->saved_context)) {	244	if (!context_save(&THREAD->saved_context)) {
245	/*	245	/*
246	* This is the place where threads leave scheduler();	246	* This is the place where threads leave scheduler();
247	*/	247	*/
248	before_thread_runs();	248	before_thread_runs();
249	spinlock_unlock(&THREAD->lock);	249	spinlock_unlock(&THREAD->lock);
250	cpu_priority_restore(THREAD->saved_context.pri);	250	cpu_priority_restore(THREAD->saved_context.pri);
251	return;	251	return;
252	}	252	}
253		253
254	/*	254	/*
255	* CPU priority of preempted thread is recorded here	255	* CPU priority of preempted thread is recorded here
256	* to facilitate scheduler() invocations from	256	* to facilitate scheduler() invocations from
257	* cpu_priority_high()'ed code (e.g. waitq_sleep_timeout()).	257	* cpu_priority_high()'ed code (e.g. waitq_sleep_timeout()).
258	*/	258	*/
259	THREAD->saved_context.pri = pri;	259	THREAD->saved_context.pri = pri;
260	}	260	}
261		261
262	/*	262	/*
263	* Through the 'THE' structure, we keep track of THREAD, TASK, CPU	263	* Through the 'THE' structure, we keep track of THREAD, TASK, CPU
264	* and preemption counter. At this point THE could be coming either	264	* and preemption counter. At this point THE could be coming either
265	* from THREAD's or CPU's stack.	265	* from THREAD's or CPU's stack.
266	*/	266	*/
267	the_copy(THE, (the_t *) CPU->stack);	267	the_copy(THE, (the_t *) CPU->stack);
268		268
269	/*	269	/*
270	* We may not keep the old stack.	270	* We may not keep the old stack.
271	* Reason: If we kept the old stack and got blocked, for instance, in	271	* Reason: If we kept the old stack and got blocked, for instance, in
272	* find_best_thread(), the old thread could get rescheduled by another	272	* find_best_thread(), the old thread could get rescheduled by another
273	* CPU and overwrite the part of its own stack that was also used by	273	* CPU and overwrite the part of its own stack that was also used by
274	* the scheduler on this CPU.	274	* the scheduler on this CPU.
275	*	275	*
276	* Moreover, we have to bypass the compiler-generated POP sequence	276	* Moreover, we have to bypass the compiler-generated POP sequence
277	* which is fooled by SP being set to the very top of the stack.	277	* which is fooled by SP being set to the very top of the stack.
278	* Therefore the scheduler() function continues in	278	* Therefore the scheduler() function continues in
279	* scheduler_separated_stack().	279	* scheduler_separated_stack().
280	*/	280	*/
281	context_save(&CPU->saved_context);	281	context_save(&CPU->saved_context);
282	context_set(&CPU->saved_context, FADDR(scheduler_separated_stack), CPU->stack, CPU_STACK_SIZE);	282	context_set(&CPU->saved_context, FADDR(scheduler_separated_stack), CPU->stack, CPU_STACK_SIZE);
283	context_restore(&CPU->saved_context);	283	context_restore(&CPU->saved_context);
284	/* not reached */	284	/* not reached */
285	}	285	}
286		286
287		287
288	/** Scheduler stack switch wrapper	288	/** Scheduler stack switch wrapper
289	*	289	*
290	* Second part of the scheduler() function	290	* Second part of the scheduler() function
291	* using new stack. Handling the actual context	291	* using new stack. Handling the actual context
292	* switch to a new thread.	292	* switch to a new thread.
293	*	293	*
294	*/	294	*/
295	void scheduler_separated_stack(void)	295	void scheduler_separated_stack(void)
296	{	296	{
297	int priority;	297	int priority;
298		298
299	ASSERT(CPU != NULL);	299	ASSERT(CPU != NULL);
300		300
301	if (THREAD) {	301	if (THREAD) {
302	switch (THREAD->state) {	302	switch (THREAD->state) {
303	case Running:	303	case Running:
304	THREAD->state = Ready;	304	THREAD->state = Ready;
305	spinlock_unlock(&THREAD->lock);	305	spinlock_unlock(&THREAD->lock);
306	thread_ready(THREAD);	306	thread_ready(THREAD);
307	break;	307	break;
308		308
309	case Exiting:	309	case Exiting:
310	frame_free((__address) THREAD->kstack);	310	frame_free((__address) THREAD->kstack);
311	if (THREAD->ustack) {	311	if (THREAD->ustack) {
312	frame_free((__address) THREAD->ustack);	312	frame_free((__address) THREAD->ustack);
313	}	313	}
314		314
315	/*	315	/*
316	* Detach from the containing task.	316	* Detach from the containing task.
317	*/	317	*/
318	spinlock_lock(&TASK->lock);	318	spinlock_lock(&TASK->lock);
319	list_remove(&THREAD->th_link);	319	list_remove(&THREAD->th_link);
320	spinlock_unlock(&TASK->lock);	320	spinlock_unlock(&TASK->lock);
321		321
322	spinlock_unlock(&THREAD->lock);	322	spinlock_unlock(&THREAD->lock);
323		323
324	spinlock_lock(&threads_lock);	324	spinlock_lock(&threads_lock);
325	list_remove(&THREAD->threads_link);	325	list_remove(&THREAD->threads_link);
326	spinlock_unlock(&threads_lock);	326	spinlock_unlock(&threads_lock);
327		327
328	spinlock_lock(&CPU->lock);	328	spinlock_lock(&CPU->lock);
329	if(CPU->fpu_owner==THREAD) CPU->fpu_owner=NULL;	329	if(CPU->fpu_owner==THREAD) CPU->fpu_owner=NULL;
330	spinlock_unlock(&CPU->lock);	330	spinlock_unlock(&CPU->lock);
331		331
332	free(THREAD);	332	free(THREAD);
333		333
334	break;	334	break;
335		335
336	case Sleeping:	336	case Sleeping:
337	/*	337	/*
338	* Prefer the thread after it's woken up.	338	* Prefer the thread after it's woken up.
339	*/	339	*/
340	THREAD->pri = -1;	340	THREAD->pri = -1;
341		341
342	/*	342	/*
343	* We need to release wq->lock which we locked in waitq_sleep().	343	* We need to release wq->lock which we locked in waitq_sleep().
344	* Address of wq->lock is kept in THREAD->sleep_queue.	344	* Address of wq->lock is kept in THREAD->sleep_queue.
345	*/	345	*/
346	spinlock_unlock(&THREAD->sleep_queue->lock);	346	spinlock_unlock(&THREAD->sleep_queue->lock);
347		347
348	/*	348	/*
349	* Check for possible requests for out-of-context invocation.	349	* Check for possible requests for out-of-context invocation.
350	*/	350	*/
351	if (THREAD->call_me) {	351	if (THREAD->call_me) {
352	THREAD->call_me(THREAD->call_me_with);	352	THREAD->call_me(THREAD->call_me_with);
353	THREAD->call_me = NULL;	353	THREAD->call_me = NULL;
354	THREAD->call_me_with = NULL;	354	THREAD->call_me_with = NULL;
355	}	355	}
356		356
357	spinlock_unlock(&THREAD->lock);	357	spinlock_unlock(&THREAD->lock);
358		358
359	break;	359	break;
360		360
361	default:	361	default:
362	/*	362	/*
363	* Entering state is unexpected.	363	* Entering state is unexpected.
364	*/	364	*/
365	panic("tid%d: unexpected state %s\n", THREAD->tid, thread_states[THREAD->state]);	365	panic("tid%d: unexpected state %s\n", THREAD->tid, thread_states[THREAD->state]);
366	break;	366	break;
367	}	367	}
368	THREAD = NULL;	368	THREAD = NULL;
369	}	369	}
370		370
371		371
372	THREAD = find_best_thread();	372	THREAD = find_best_thread();
373		373
374	spinlock_lock(&THREAD->lock);	374	spinlock_lock(&THREAD->lock);
375	priority = THREAD->pri;	375	priority = THREAD->pri;
376	spinlock_unlock(&THREAD->lock);	376	spinlock_unlock(&THREAD->lock);
377		377
378	relink_rq(priority);	378	relink_rq(priority);
379		379
380	spinlock_lock(&THREAD->lock);	380	spinlock_lock(&THREAD->lock);
381		381
382	/*	382	/*
383	* If both the old and the new task are the same, lots of work is avoided.	383	* If both the old and the new task are the same, lots of work is avoided.
384	*/	384	*/
385	if (TASK != THREAD->task) {	385	if (TASK != THREAD->task) {
386	vm_t *m1 = NULL;	386	vm_t *m1 = NULL;
387	vm_t *m2;	387	vm_t *m2;
388		388
389	if (TASK) {	389	if (TASK) {
390	spinlock_lock(&TASK->lock);	390	spinlock_lock(&TASK->lock);
391	m1 = TASK->vm;	391	m1 = TASK->vm;
392	spinlock_unlock(&TASK->lock);	392	spinlock_unlock(&TASK->lock);
393	}	393	}
394		394
395	spinlock_lock(&THREAD->task->lock);	395	spinlock_lock(&THREAD->task->lock);
396	m2 = THREAD->task->vm;	396	m2 = THREAD->task->vm;
397	spinlock_unlock(&THREAD->task->lock);	397	spinlock_unlock(&THREAD->task->lock);
398		398
399	/*	399	/*
400	* Note that it is possible for two tasks to share one vm mapping.	400	* Note that it is possible for two tasks to share one vm mapping.
401	*/	401	*/
402	if (m1 != m2) {	402	if (m1 != m2) {
403	/*	403	/*
404	* Both tasks and vm mappings are different.	404	* Both tasks and vm mappings are different.
405	* Replace the old one with the new one.	405	* Replace the old one with the new one.
406	*/	406	*/
407	vm_install(m2);	407	vm_install(m2);
408	}	408	}
409	TASK = THREAD->task;	409	TASK = THREAD->task;
410	}	410	}
411		411
412	THREAD->state = Running;	412	THREAD->state = Running;
413		413
414	#ifdef SCHEDULER_VERBOSE	414	#ifdef SCHEDULER_VERBOSE
415	printf("cpu%d: tid %d (pri=%d,ticks=%d,nrdy=%d)\n", CPU->id, THREAD->tid, THREAD->pri, THREAD->ticks, CPU->nrdy);	415	printf("cpu%d: tid %d (pri=%d,ticks=%d,nrdy=%d)\n", CPU->id, THREAD->tid, THREAD->pri, THREAD->ticks, CPU->nrdy);
416	#endif	416	#endif
417		417
418	/*	418	/*
419	* Copy the knowledge of CPU, TASK, THREAD and preemption counter to thread's stack.	419	* Copy the knowledge of CPU, TASK, THREAD and preemption counter to thread's stack.
420	*/	420	*/
421	the_copy(THE, (the_t *) THREAD->kstack);	421	the_copy(THE, (the_t *) THREAD->kstack);
422		422
423	context_restore(&THREAD->saved_context);	423	context_restore(&THREAD->saved_context);
424	/* not reached */	424	/* not reached */
425	}	425	}
426		426
427		427
428	#ifdef __SMP__	428	#ifdef __SMP__
429	/** Load balancing thread	429	/** Load balancing thread
430	*	430	*
431	* SMP load balancing thread, supervising thread supplies	431	* SMP load balancing thread, supervising thread supplies
432	* for the CPU it's wired to.	432	* for the CPU it's wired to.
433	*	433	*
434	* @param arg Generic thread argument (unused).	434	* @param arg Generic thread argument (unused).
435	*	435	*
436	*/	436	*/
437	void kcpulb(void *arg)	437	void kcpulb(void *arg)
438	{	438	{
439	thread_t *t;	439	thread_t *t;
440	int count, i, j, k = 0;	440	int count, i, j, k = 0;
441	pri_t pri;	441	pri_t pri;
442		442
443	loop:	443	loop:
444	/*	444	/*
445	* Sleep until there's some work to do.	445	* Sleep until there's some work to do.
446	*/	446	*/
447	waitq_sleep(&CPU->kcpulb_wq);	447	waitq_sleep(&CPU->kcpulb_wq);
448		448
449	not_satisfied:	449	not_satisfied:
450	/*	450	/*
451	* Calculate the number of threads that will be migrated/stolen from	451	* Calculate the number of threads that will be migrated/stolen from
452	* other CPU's. Note that situation can have changed between two	452	* other CPU's. Note that situation can have changed between two
453	* passes. Each time get the most up to date counts.	453	* passes. Each time get the most up to date counts.
454	*/	454	*/
455	pri = cpu_priority_high();	455	pri = cpu_priority_high();
456	spinlock_lock(&CPU->lock);	456	spinlock_lock(&CPU->lock);
457	count = nrdy / config.cpu_active;	457	count = nrdy / config.cpu_active;
458	count -= CPU->nrdy;	458	count -= CPU->nrdy;
459	spinlock_unlock(&CPU->lock);	459	spinlock_unlock(&CPU->lock);
460	cpu_priority_restore(pri);	460	cpu_priority_restore(pri);
461		461
462	if (count <= 0)	462	if (count <= 0)
463	goto satisfied;	463	goto satisfied;
464		464
465	/*	465	/*
466	* Searching least priority queues on all CPU's first and most priority queues on all CPU's last.	466	* Searching least priority queues on all CPU's first and most priority queues on all CPU's last.
467	*/	467	*/
468	for (j=RQ_COUNT-1; j >= 0; j--) {	468	for (j=RQ_COUNT-1; j >= 0; j--) {
469	for (i=0; i < config.cpu_active; i++) {	469	for (i=0; i < config.cpu_active; i++) {
470	link_t *l;	470	link_t *l;
471	runq_t *r;	471	runq_t *r;
472	cpu_t *cpu;	472	cpu_t *cpu;
473		473
474	cpu = &cpus[(i + k) % config.cpu_active];	474	cpu = &cpus[(i + k) % config.cpu_active];
475		475
476	/*	476	/*
477	* Not interested in ourselves.	477	* Not interested in ourselves.
478	* Doesn't require interrupt disabling for kcpulb is X_WIRED.	478	* Doesn't require interrupt disabling for kcpulb is X_WIRED.
479	*/	479	*/
480	if (CPU == cpu)	480	if (CPU == cpu)
481	continue;	481	continue;
482		482
483	restart: pri = cpu_priority_high();	483	restart: pri = cpu_priority_high();
484	r = &cpu->rq[j];	484	r = &cpu->rq[j];
485	spinlock_lock(&r->lock);	485	spinlock_lock(&r->lock);
486	if (r->n == 0) {	486	if (r->n == 0) {
487	spinlock_unlock(&r->lock);	487	spinlock_unlock(&r->lock);
488	cpu_priority_restore(pri);	488	cpu_priority_restore(pri);
489	continue;	489	continue;
490	}	490	}
491		491
492	t = NULL;	492	t = NULL;
493	l = r->rq_head.prev; /* search rq from the back */	493	l = r->rq_head.prev; /* search rq from the back */
494	while (l != &r->rq_head) {	494	while (l != &r->rq_head) {
495	t = list_get_instance(l, thread_t, rq_link);	495	t = list_get_instance(l, thread_t, rq_link);
496	/*	496	/*
497	* We don't want to steal CPU-wired threads neither threads already stolen.	497	* We don't want to steal CPU-wired threads neither threads already stolen.
498	* The latter prevents threads from migrating between CPU's without ever being run.	498	* The latter prevents threads from migrating between CPU's without ever being run.
499	* We don't want to steal threads whose FPU context is still in CPU.	499	* We don't want to steal threads whose FPU context is still in CPU.
500	*/	500	*/

Subversion Repositories HelenOS-historic

(root)/kernel/trunk/generic/src/proc/scheduler.c @ 1705 – Rev 241 → 248