Rev 2040 | Rev 2068 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed
Rev 2040 | Rev 2067 | ||
---|---|---|---|
Line 1... | Line 1... | ||
1 | /* |
1 | /* |
2 | * Copyright (C) 2001-2004 Jakub Jermar |
2 | * Copyright (C) 2001-2007 Jakub Jermar |
3 | * All rights reserved. |
3 | * All rights reserved. |
4 | * |
4 | * |
5 | * Redistribution and use in source and binary forms, with or without |
5 | * Redistribution and use in source and binary forms, with or without |
6 | * modification, are permitted provided that the following conditions |
6 | * modification, are permitted provided that the following conditions |
7 | * are met: |
7 | * are met: |
Line 140... | Line 140... | ||
140 | /* Allocate FPU context */ |
140 | /* Allocate FPU context */ |
141 | if (!THREAD->saved_fpu_context) { |
141 | if (!THREAD->saved_fpu_context) { |
142 | /* Might sleep */ |
142 | /* Might sleep */ |
143 | spinlock_unlock(&THREAD->lock); |
143 | spinlock_unlock(&THREAD->lock); |
144 | spinlock_unlock(&CPU->lock); |
144 | spinlock_unlock(&CPU->lock); |
- | 145 | THREAD->saved_fpu_context = |
|
145 | THREAD->saved_fpu_context = slab_alloc(fpu_context_slab, 0); |
146 | slab_alloc(fpu_context_slab, 0); |
146 | /* We may have switched CPUs during slab_alloc */ |
147 | /* We may have switched CPUs during slab_alloc */ |
147 | goto restart; |
148 | goto restart; |
148 | } |
149 | } |
149 | fpu_init(); |
150 | fpu_init(); |
150 | THREAD->fpu_context_exists = 1; |
151 | THREAD->fpu_context_exists = 1; |
Line 229... | Line 230... | ||
229 | spinlock_unlock(&r->lock); |
230 | spinlock_unlock(&r->lock); |
230 | 231 | ||
231 | spinlock_lock(&t->lock); |
232 | spinlock_lock(&t->lock); |
232 | t->cpu = CPU; |
233 | t->cpu = CPU; |
233 | 234 | ||
234 | t->ticks = us2ticks((i+1)*10000); |
235 | t->ticks = us2ticks((i + 1) * 10000); |
235 | t->priority = i; /* correct rq index */ |
236 | t->priority = i; /* correct rq index */ |
236 | 237 | ||
237 | /* |
238 | /* |
238 | * Clear the THREAD_FLAG_STOLEN flag so that t can be migrated |
239 | * Clear the THREAD_FLAG_STOLEN flag so that t can be migrated |
239 | * when load balancing needs emerge. |
240 | * when load balancing needs emerge. |
Line 265... | Line 266... | ||
265 | int i, n; |
266 | int i, n; |
266 | 267 | ||
267 | list_initialize(&head); |
268 | list_initialize(&head); |
268 | spinlock_lock(&CPU->lock); |
269 | spinlock_lock(&CPU->lock); |
269 | if (CPU->needs_relink > NEEDS_RELINK_MAX) { |
270 | if (CPU->needs_relink > NEEDS_RELINK_MAX) { |
270 | for (i = start; i<RQ_COUNT-1; i++) { |
271 | for (i = start; i < RQ_COUNT - 1; i++) { |
271 | /* remember and empty rq[i + 1] */ |
272 | /* remember and empty rq[i + 1] */ |
272 | r = &CPU->rq[i + 1]; |
273 | r = &CPU->rq[i + 1]; |
273 | spinlock_lock(&r->lock); |
274 | spinlock_lock(&r->lock); |
274 | list_concat(&head, &r->rq_head); |
275 | list_concat(&head, &r->rq_head); |
275 | n = r->n; |
276 | n = r->n; |
Line 329... | Line 330... | ||
329 | 330 | ||
330 | return; |
331 | return; |
331 | } |
332 | } |
332 | 333 | ||
333 | /* |
334 | /* |
334 | * Interrupt priority level of preempted thread is recorded here |
335 | * Interrupt priority level of preempted thread is recorded |
335 | * to facilitate scheduler() invocations from interrupts_disable()'d |
336 | * here to facilitate scheduler() invocations from |
336 | * code (e.g. waitq_sleep_timeout()). |
337 | * interrupts_disable()'d code (e.g. waitq_sleep_timeout()). |
337 | */ |
338 | */ |
338 | THREAD->saved_context.ipl = ipl; |
339 | THREAD->saved_context.ipl = ipl; |
339 | } |
340 | } |
340 | 341 | ||
341 | /* |
342 | /* |
Line 392... | Line 393... | ||
392 | repeat: |
393 | repeat: |
393 | if (THREAD->detached) { |
394 | if (THREAD->detached) { |
394 | thread_destroy(THREAD); |
395 | thread_destroy(THREAD); |
395 | } else { |
396 | } else { |
396 | /* |
397 | /* |
397 | * The thread structure is kept allocated until somebody |
398 | * The thread structure is kept allocated until |
398 | * calls thread_detach() on it. |
399 | * somebody calls thread_detach() on it. |
399 | */ |
400 | */ |
400 | if (!spinlock_trylock(&THREAD->join_wq.lock)) { |
401 | if (!spinlock_trylock(&THREAD->join_wq.lock)) { |
401 | /* |
402 | /* |
402 | * Avoid deadlock. |
403 | * Avoid deadlock. |
403 | */ |
404 | */ |
Line 419... | Line 420... | ||
419 | * Prefer the thread after it's woken up. |
420 | * Prefer the thread after it's woken up. |
420 | */ |
421 | */ |
421 | THREAD->priority = -1; |
422 | THREAD->priority = -1; |
422 | 423 | ||
423 | /* |
424 | /* |
424 | * We need to release wq->lock which we locked in waitq_sleep(). |
425 | * We need to release wq->lock which we locked in |
425 | * Address of wq->lock is kept in THREAD->sleep_queue. |
426 | * waitq_sleep(). Address of wq->lock is kept in |
- | 427 | * THREAD->sleep_queue. |
|
426 | */ |
428 | */ |
427 | spinlock_unlock(&THREAD->sleep_queue->lock); |
429 | spinlock_unlock(&THREAD->sleep_queue->lock); |
428 | 430 | ||
429 | /* |
431 | /* |
430 | * Check for possible requests for out-of-context invocation. |
432 | * Check for possible requests for out-of-context |
- | 433 | * invocation. |
|
431 | */ |
434 | */ |
432 | if (THREAD->call_me) { |
435 | if (THREAD->call_me) { |
433 | THREAD->call_me(THREAD->call_me_with); |
436 | THREAD->call_me(THREAD->call_me_with); |
434 | THREAD->call_me = NULL; |
437 | THREAD->call_me = NULL; |
435 | THREAD->call_me_with = NULL; |
438 | THREAD->call_me_with = NULL; |
Line 441... | Line 444... | ||
441 | 444 | ||
442 | default: |
445 | default: |
443 | /* |
446 | /* |
444 | * Entering state is unexpected. |
447 | * Entering state is unexpected. |
445 | */ |
448 | */ |
446 | panic("tid%d: unexpected state %s\n", THREAD->tid, thread_states[THREAD->state]); |
449 | panic("tid%d: unexpected state %s\n", THREAD->tid, |
- | 450 | thread_states[THREAD->state]); |
|
447 | break; |
451 | break; |
448 | } |
452 | } |
449 | 453 | ||
450 | THREAD = NULL; |
454 | THREAD = NULL; |
451 | } |
455 | } |
Line 457... | Line 461... | ||
457 | spinlock_unlock(&THREAD->lock); |
461 | spinlock_unlock(&THREAD->lock); |
458 | 462 | ||
459 | relink_rq(priority); |
463 | relink_rq(priority); |
460 | 464 | ||
461 | /* |
465 | /* |
462 | * If both the old and the new task are the same, lots of work is avoided. |
466 | * If both the old and the new task are the same, lots of work is |
- | 467 | * avoided. |
|
463 | */ |
468 | */ |
464 | if (TASK != THREAD->task) { |
469 | if (TASK != THREAD->task) { |
465 | as_t *as1 = NULL; |
470 | as_t *as1 = NULL; |
466 | as_t *as2; |
471 | as_t *as2; |
467 | 472 | ||
Line 474... | Line 479... | ||
474 | spinlock_lock(&THREAD->task->lock); |
479 | spinlock_lock(&THREAD->task->lock); |
475 | as2 = THREAD->task->as; |
480 | as2 = THREAD->task->as; |
476 | spinlock_unlock(&THREAD->task->lock); |
481 | spinlock_unlock(&THREAD->task->lock); |
477 | 482 | ||
478 | /* |
483 | /* |
479 | * Note that it is possible for two tasks to share one address space. |
484 | * Note that it is possible for two tasks to share one address |
- | 485 | * space. |
|
480 | */ |
486 | */ |
481 | if (as1 != as2) { |
487 | if (as1 != as2) { |
482 | /* |
488 | /* |
483 | * Both tasks and address spaces are different. |
489 | * Both tasks and address spaces are different. |
484 | * Replace the old one with the new one. |
490 | * Replace the old one with the new one. |
Line 491... | Line 497... | ||
491 | 497 | ||
492 | spinlock_lock(&THREAD->lock); |
498 | spinlock_lock(&THREAD->lock); |
493 | THREAD->state = Running; |
499 | THREAD->state = Running; |
494 | 500 | ||
495 | #ifdef SCHEDULER_VERBOSE |
501 | #ifdef SCHEDULER_VERBOSE |
496 | printf("cpu%d: tid %d (priority=%d,ticks=%lld,nrdy=%ld)\n", |
502 | printf("cpu%d: tid %d (priority=%d, ticks=%lld, nrdy=%ld)\n", |
497 | CPU->id, THREAD->tid, THREAD->priority, THREAD->ticks, atomic_get(&CPU->nrdy)); |
503 | CPU->id, THREAD->tid, THREAD->priority, THREAD->ticks, |
- | 504 | atomic_get(&CPU->nrdy)); |
|
498 | #endif |
505 | #endif |
499 | 506 | ||
500 | /* |
507 | /* |
501 | * Some architectures provide late kernel PA2KA(identity) |
508 | * Some architectures provide late kernel PA2KA(identity) |
502 | * mapping in a page fault handler. However, the page fault |
509 | * mapping in a page fault handler. However, the page fault |
Line 506... | Line 513... | ||
506 | * function must be executed before the switch to the new stack. |
513 | * function must be executed before the switch to the new stack. |
507 | */ |
514 | */ |
508 | before_thread_runs(); |
515 | before_thread_runs(); |
509 | 516 | ||
510 | /* |
517 | /* |
511 | * Copy the knowledge of CPU, TASK, THREAD and preemption counter to thread's stack. |
518 | * Copy the knowledge of CPU, TASK, THREAD and preemption counter to |
- | 519 | * thread's stack. |
|
512 | */ |
520 | */ |
513 | the_copy(THE, (the_t *) THREAD->kstack); |
521 | the_copy(THE, (the_t *) THREAD->kstack); |
514 | 522 | ||
515 | context_restore(&THREAD->saved_context); |
523 | context_restore(&THREAD->saved_context); |
516 | /* not reached */ |
524 | /* not reached */ |
Line 553... | Line 561... | ||
553 | 561 | ||
554 | if (count <= 0) |
562 | if (count <= 0) |
555 | goto satisfied; |
563 | goto satisfied; |
556 | 564 | ||
557 | /* |
565 | /* |
558 | * Searching least priority queues on all CPU's first and most priority queues on all CPU's last. |
566 | * Searching least priority queues on all CPU's first and most priority |
- | 567 | * queues on all CPU's last. |
|
559 | */ |
568 | */ |
560 | for (j=RQ_COUNT-1; j >= 0; j--) { |
569 | for (j= RQ_COUNT - 1; j >= 0; j--) { |
561 | for (i=0; i < config.cpu_active; i++) { |
570 | for (i = 0; i < config.cpu_active; i++) { |
562 | link_t *l; |
571 | link_t *l; |
563 | runq_t *r; |
572 | runq_t *r; |
564 | cpu_t *cpu; |
573 | cpu_t *cpu; |
565 | 574 | ||
566 | cpu = &cpus[(i + k) % config.cpu_active]; |
575 | cpu = &cpus[(i + k) % config.cpu_active]; |
567 | 576 | ||
568 | /* |
577 | /* |
569 | * Not interested in ourselves. |
578 | * Not interested in ourselves. |
570 | * Doesn't require interrupt disabling for kcpulb has THREAD_FLAG_WIRED. |
579 | * Doesn't require interrupt disabling for kcpulb has |
- | 580 | * THREAD_FLAG_WIRED. |
|
571 | */ |
581 | */ |
572 | if (CPU == cpu) |
582 | if (CPU == cpu) |
573 | continue; |
583 | continue; |
574 | if (atomic_get(&cpu->nrdy) <= average) |
584 | if (atomic_get(&cpu->nrdy) <= average) |
575 | continue; |
585 | continue; |
Line 586... | Line 596... | ||
586 | t = NULL; |
596 | t = NULL; |
587 | l = r->rq_head.prev; /* search rq from the back */ |
597 | l = r->rq_head.prev; /* search rq from the back */ |
588 | while (l != &r->rq_head) { |
598 | while (l != &r->rq_head) { |
589 | t = list_get_instance(l, thread_t, rq_link); |
599 | t = list_get_instance(l, thread_t, rq_link); |
590 | /* |
600 | /* |
591 | * We don't want to steal CPU-wired threads neither threads already |
601 | * We don't want to steal CPU-wired threads |
- | 602 | * neither threads already stolen. The latter |
|
592 | * stolen. The latter prevents threads from migrating between CPU's |
603 | * prevents threads from migrating between CPU's |
593 | * without ever being run. We don't want to steal threads whose FPU |
604 | * without ever being run. We don't want to |
594 | * context is still in CPU. |
605 | * steal threads whose FPU context is still in |
- | 606 | * CPU. |
|
595 | */ |
607 | */ |
596 | spinlock_lock(&t->lock); |
608 | spinlock_lock(&t->lock); |
597 | if ((!(t->flags & (THREAD_FLAG_WIRED | THREAD_FLAG_STOLEN))) && |
609 | if ((!(t->flags & (THREAD_FLAG_WIRED | |
- | 610 | THREAD_FLAG_STOLEN))) && |
|
598 | (!(t->fpu_context_engaged)) ) { |
611 | (!(t->fpu_context_engaged)) ) { |
599 | /* |
612 | /* |
600 | * Remove t from r. |
613 | * Remove t from r. |
601 | */ |
614 | */ |
602 | spinlock_unlock(&t->lock); |
615 | spinlock_unlock(&t->lock); |
Line 619... | Line 632... | ||
619 | /* |
632 | /* |
620 | * Ready t on local CPU |
633 | * Ready t on local CPU |
621 | */ |
634 | */ |
622 | spinlock_lock(&t->lock); |
635 | spinlock_lock(&t->lock); |
623 | #ifdef KCPULB_VERBOSE |
636 | #ifdef KCPULB_VERBOSE |
624 | printf("kcpulb%d: TID %d -> cpu%d, nrdy=%ld, avg=%nd\n", |
637 | printf("kcpulb%d: TID %d -> cpu%d, nrdy=%ld, " |
625 | CPU->id, t->tid, CPU->id, atomic_get(&CPU->nrdy), |
638 | "avg=%nd\n", CPU->id, t->tid, CPU->id, |
- | 639 | atomic_get(&CPU->nrdy), |
|
626 | atomic_get(&nrdy) / config.cpu_active); |
640 | atomic_get(&nrdy) / config.cpu_active); |
627 | #endif |
641 | #endif |
628 | t->flags |= THREAD_FLAG_STOLEN; |
642 | t->flags |= THREAD_FLAG_STOLEN; |
629 | t->state = Entering; |
643 | t->state = Entering; |
630 | spinlock_unlock(&t->lock); |
644 | spinlock_unlock(&t->lock); |
Line 635... | Line 649... | ||
635 | 649 | ||
636 | if (--count == 0) |
650 | if (--count == 0) |
637 | goto satisfied; |
651 | goto satisfied; |
638 | 652 | ||
639 | /* |
653 | /* |
640 | * We are not satisfied yet, focus on another CPU next time. |
654 | * We are not satisfied yet, focus on another |
- | 655 | * CPU next time. |
|
641 | */ |
656 | */ |
642 | k++; |
657 | k++; |
643 | 658 | ||
644 | continue; |
659 | continue; |
645 | } |
660 | } |
Line 686... | Line 701... | ||
686 | if (!cpus[cpu].active) |
701 | if (!cpus[cpu].active) |
687 | continue; |
702 | continue; |
688 | 703 | ||
689 | spinlock_lock(&cpus[cpu].lock); |
704 | spinlock_lock(&cpus[cpu].lock); |
690 | printf("cpu%d: address=%p, nrdy=%ld, needs_relink=%ld\n", |
705 | printf("cpu%d: address=%p, nrdy=%ld, needs_relink=%ld\n", |
691 | cpus[cpu].id, &cpus[cpu], atomic_get(&cpus[cpu].nrdy), cpus[cpu].needs_relink); |
706 | cpus[cpu].id, &cpus[cpu], atomic_get(&cpus[cpu].nrdy), |
- | 707 | cpus[cpu].needs_relink); |
|
692 | 708 | ||
693 | for (i=0; i<RQ_COUNT; i++) { |
709 | for (i = 0; i < RQ_COUNT; i++) { |
694 | r = &cpus[cpu].rq[i]; |
710 | r = &cpus[cpu].rq[i]; |
695 | spinlock_lock(&r->lock); |
711 | spinlock_lock(&r->lock); |
696 | if (!r->n) { |
712 | if (!r->n) { |
697 | spinlock_unlock(&r->lock); |
713 | spinlock_unlock(&r->lock); |
698 | continue; |
714 | continue; |
699 | } |
715 | } |
700 | printf("\trq[%d]: ", i); |
716 | printf("\trq[%d]: ", i); |
701 | for (cur=r->rq_head.next; cur!=&r->rq_head; cur=cur->next) { |
717 | for (cur = r->rq_head.next; cur != &r->rq_head; |
- | 718 | cur = cur->next) { |
|
702 | t = list_get_instance(cur, thread_t, rq_link); |
719 | t = list_get_instance(cur, thread_t, rq_link); |
703 | printf("%d(%s) ", t->tid, |
720 | printf("%d(%s) ", t->tid, |
704 | thread_states[t->state]); |
721 | thread_states[t->state]); |
705 | } |
722 | } |
706 | printf("\n"); |
723 | printf("\n"); |
707 | spinlock_unlock(&r->lock); |
724 | spinlock_unlock(&r->lock); |
708 | } |
725 | } |
709 | spinlock_unlock(&cpus[cpu].lock); |
726 | spinlock_unlock(&cpus[cpu].lock); |