Rev 2040 | Rev 2068 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed
| Rev 2040 | Rev 2067 | ||
|---|---|---|---|
| Line 1... | Line 1... | ||
| 1 | /* |
1 | /* |
| 2 | * Copyright (C) 2001-2004 Jakub Jermar |
2 | * Copyright (C) 2001-2007 Jakub Jermar |
| 3 | * All rights reserved. |
3 | * All rights reserved. |
| 4 | * |
4 | * |
| 5 | * Redistribution and use in source and binary forms, with or without |
5 | * Redistribution and use in source and binary forms, with or without |
| 6 | * modification, are permitted provided that the following conditions |
6 | * modification, are permitted provided that the following conditions |
| 7 | * are met: |
7 | * are met: |
| Line 140... | Line 140... | ||
| 140 | /* Allocate FPU context */ |
140 | /* Allocate FPU context */ |
| 141 | if (!THREAD->saved_fpu_context) { |
141 | if (!THREAD->saved_fpu_context) { |
| 142 | /* Might sleep */ |
142 | /* Might sleep */ |
| 143 | spinlock_unlock(&THREAD->lock); |
143 | spinlock_unlock(&THREAD->lock); |
| 144 | spinlock_unlock(&CPU->lock); |
144 | spinlock_unlock(&CPU->lock); |
| - | 145 | THREAD->saved_fpu_context = |
|
| 145 | THREAD->saved_fpu_context = slab_alloc(fpu_context_slab, 0); |
146 | slab_alloc(fpu_context_slab, 0); |
| 146 | /* We may have switched CPUs during slab_alloc */ |
147 | /* We may have switched CPUs during slab_alloc */ |
| 147 | goto restart; |
148 | goto restart; |
| 148 | } |
149 | } |
| 149 | fpu_init(); |
150 | fpu_init(); |
| 150 | THREAD->fpu_context_exists = 1; |
151 | THREAD->fpu_context_exists = 1; |
| Line 229... | Line 230... | ||
| 229 | spinlock_unlock(&r->lock); |
230 | spinlock_unlock(&r->lock); |
| 230 | 231 | ||
| 231 | spinlock_lock(&t->lock); |
232 | spinlock_lock(&t->lock); |
| 232 | t->cpu = CPU; |
233 | t->cpu = CPU; |
| 233 | 234 | ||
| 234 | t->ticks = us2ticks((i+1)*10000); |
235 | t->ticks = us2ticks((i + 1) * 10000); |
| 235 | t->priority = i; /* correct rq index */ |
236 | t->priority = i; /* correct rq index */ |
| 236 | 237 | ||
| 237 | /* |
238 | /* |
| 238 | * Clear the THREAD_FLAG_STOLEN flag so that t can be migrated |
239 | * Clear the THREAD_FLAG_STOLEN flag so that t can be migrated |
| 239 | * when load balancing needs emerge. |
240 | * when load balancing needs emerge. |
| Line 265... | Line 266... | ||
| 265 | int i, n; |
266 | int i, n; |
| 266 | 267 | ||
| 267 | list_initialize(&head); |
268 | list_initialize(&head); |
| 268 | spinlock_lock(&CPU->lock); |
269 | spinlock_lock(&CPU->lock); |
| 269 | if (CPU->needs_relink > NEEDS_RELINK_MAX) { |
270 | if (CPU->needs_relink > NEEDS_RELINK_MAX) { |
| 270 | for (i = start; i<RQ_COUNT-1; i++) { |
271 | for (i = start; i < RQ_COUNT - 1; i++) { |
| 271 | /* remember and empty rq[i + 1] */ |
272 | /* remember and empty rq[i + 1] */ |
| 272 | r = &CPU->rq[i + 1]; |
273 | r = &CPU->rq[i + 1]; |
| 273 | spinlock_lock(&r->lock); |
274 | spinlock_lock(&r->lock); |
| 274 | list_concat(&head, &r->rq_head); |
275 | list_concat(&head, &r->rq_head); |
| 275 | n = r->n; |
276 | n = r->n; |
| Line 329... | Line 330... | ||
| 329 | 330 | ||
| 330 | return; |
331 | return; |
| 331 | } |
332 | } |
| 332 | 333 | ||
| 333 | /* |
334 | /* |
| 334 | * Interrupt priority level of preempted thread is recorded here |
335 | * Interrupt priority level of preempted thread is recorded |
| 335 | * to facilitate scheduler() invocations from interrupts_disable()'d |
336 | * here to facilitate scheduler() invocations from |
| 336 | * code (e.g. waitq_sleep_timeout()). |
337 | * interrupts_disable()'d code (e.g. waitq_sleep_timeout()). |
| 337 | */ |
338 | */ |
| 338 | THREAD->saved_context.ipl = ipl; |
339 | THREAD->saved_context.ipl = ipl; |
| 339 | } |
340 | } |
| 340 | 341 | ||
| 341 | /* |
342 | /* |
| Line 392... | Line 393... | ||
| 392 | repeat: |
393 | repeat: |
| 393 | if (THREAD->detached) { |
394 | if (THREAD->detached) { |
| 394 | thread_destroy(THREAD); |
395 | thread_destroy(THREAD); |
| 395 | } else { |
396 | } else { |
| 396 | /* |
397 | /* |
| 397 | * The thread structure is kept allocated until somebody |
398 | * The thread structure is kept allocated until |
| 398 | * calls thread_detach() on it. |
399 | * somebody calls thread_detach() on it. |
| 399 | */ |
400 | */ |
| 400 | if (!spinlock_trylock(&THREAD->join_wq.lock)) { |
401 | if (!spinlock_trylock(&THREAD->join_wq.lock)) { |
| 401 | /* |
402 | /* |
| 402 | * Avoid deadlock. |
403 | * Avoid deadlock. |
| 403 | */ |
404 | */ |
| Line 419... | Line 420... | ||
| 419 | * Prefer the thread after it's woken up. |
420 | * Prefer the thread after it's woken up. |
| 420 | */ |
421 | */ |
| 421 | THREAD->priority = -1; |
422 | THREAD->priority = -1; |
| 422 | 423 | ||
| 423 | /* |
424 | /* |
| 424 | * We need to release wq->lock which we locked in waitq_sleep(). |
425 | * We need to release wq->lock which we locked in |
| 425 | * Address of wq->lock is kept in THREAD->sleep_queue. |
426 | * waitq_sleep(). Address of wq->lock is kept in |
| - | 427 | * THREAD->sleep_queue. |
|
| 426 | */ |
428 | */ |
| 427 | spinlock_unlock(&THREAD->sleep_queue->lock); |
429 | spinlock_unlock(&THREAD->sleep_queue->lock); |
| 428 | 430 | ||
| 429 | /* |
431 | /* |
| 430 | * Check for possible requests for out-of-context invocation. |
432 | * Check for possible requests for out-of-context |
| - | 433 | * invocation. |
|
| 431 | */ |
434 | */ |
| 432 | if (THREAD->call_me) { |
435 | if (THREAD->call_me) { |
| 433 | THREAD->call_me(THREAD->call_me_with); |
436 | THREAD->call_me(THREAD->call_me_with); |
| 434 | THREAD->call_me = NULL; |
437 | THREAD->call_me = NULL; |
| 435 | THREAD->call_me_with = NULL; |
438 | THREAD->call_me_with = NULL; |
| Line 441... | Line 444... | ||
| 441 | 444 | ||
| 442 | default: |
445 | default: |
| 443 | /* |
446 | /* |
| 444 | * Entering state is unexpected. |
447 | * Entering state is unexpected. |
| 445 | */ |
448 | */ |
| 446 | panic("tid%d: unexpected state %s\n", THREAD->tid, thread_states[THREAD->state]); |
449 | panic("tid%d: unexpected state %s\n", THREAD->tid, |
| - | 450 | thread_states[THREAD->state]); |
|
| 447 | break; |
451 | break; |
| 448 | } |
452 | } |
| 449 | 453 | ||
| 450 | THREAD = NULL; |
454 | THREAD = NULL; |
| 451 | } |
455 | } |
| Line 457... | Line 461... | ||
| 457 | spinlock_unlock(&THREAD->lock); |
461 | spinlock_unlock(&THREAD->lock); |
| 458 | 462 | ||
| 459 | relink_rq(priority); |
463 | relink_rq(priority); |
| 460 | 464 | ||
| 461 | /* |
465 | /* |
| 462 | * If both the old and the new task are the same, lots of work is avoided. |
466 | * If both the old and the new task are the same, lots of work is |
| - | 467 | * avoided. |
|
| 463 | */ |
468 | */ |
| 464 | if (TASK != THREAD->task) { |
469 | if (TASK != THREAD->task) { |
| 465 | as_t *as1 = NULL; |
470 | as_t *as1 = NULL; |
| 466 | as_t *as2; |
471 | as_t *as2; |
| 467 | 472 | ||
| Line 474... | Line 479... | ||
| 474 | spinlock_lock(&THREAD->task->lock); |
479 | spinlock_lock(&THREAD->task->lock); |
| 475 | as2 = THREAD->task->as; |
480 | as2 = THREAD->task->as; |
| 476 | spinlock_unlock(&THREAD->task->lock); |
481 | spinlock_unlock(&THREAD->task->lock); |
| 477 | 482 | ||
| 478 | /* |
483 | /* |
| 479 | * Note that it is possible for two tasks to share one address space. |
484 | * Note that it is possible for two tasks to share one address |
| - | 485 | * space. |
|
| 480 | */ |
486 | */ |
| 481 | if (as1 != as2) { |
487 | if (as1 != as2) { |
| 482 | /* |
488 | /* |
| 483 | * Both tasks and address spaces are different. |
489 | * Both tasks and address spaces are different. |
| 484 | * Replace the old one with the new one. |
490 | * Replace the old one with the new one. |
| Line 491... | Line 497... | ||
| 491 | 497 | ||
| 492 | spinlock_lock(&THREAD->lock); |
498 | spinlock_lock(&THREAD->lock); |
| 493 | THREAD->state = Running; |
499 | THREAD->state = Running; |
| 494 | 500 | ||
| 495 | #ifdef SCHEDULER_VERBOSE |
501 | #ifdef SCHEDULER_VERBOSE |
| 496 | printf("cpu%d: tid %d (priority=%d,ticks=%lld,nrdy=%ld)\n", |
502 | printf("cpu%d: tid %d (priority=%d, ticks=%lld, nrdy=%ld)\n", |
| 497 | CPU->id, THREAD->tid, THREAD->priority, THREAD->ticks, atomic_get(&CPU->nrdy)); |
503 | CPU->id, THREAD->tid, THREAD->priority, THREAD->ticks, |
| - | 504 | atomic_get(&CPU->nrdy)); |
|
| 498 | #endif |
505 | #endif |
| 499 | 506 | ||
| 500 | /* |
507 | /* |
| 501 | * Some architectures provide late kernel PA2KA(identity) |
508 | * Some architectures provide late kernel PA2KA(identity) |
| 502 | * mapping in a page fault handler. However, the page fault |
509 | * mapping in a page fault handler. However, the page fault |
| Line 506... | Line 513... | ||
| 506 | * function must be executed before the switch to the new stack. |
513 | * function must be executed before the switch to the new stack. |
| 507 | */ |
514 | */ |
| 508 | before_thread_runs(); |
515 | before_thread_runs(); |
| 509 | 516 | ||
| 510 | /* |
517 | /* |
| 511 | * Copy the knowledge of CPU, TASK, THREAD and preemption counter to thread's stack. |
518 | * Copy the knowledge of CPU, TASK, THREAD and preemption counter to |
| - | 519 | * thread's stack. |
|
| 512 | */ |
520 | */ |
| 513 | the_copy(THE, (the_t *) THREAD->kstack); |
521 | the_copy(THE, (the_t *) THREAD->kstack); |
| 514 | 522 | ||
| 515 | context_restore(&THREAD->saved_context); |
523 | context_restore(&THREAD->saved_context); |
| 516 | /* not reached */ |
524 | /* not reached */ |
| Line 553... | Line 561... | ||
| 553 | 561 | ||
| 554 | if (count <= 0) |
562 | if (count <= 0) |
| 555 | goto satisfied; |
563 | goto satisfied; |
| 556 | 564 | ||
| 557 | /* |
565 | /* |
| 558 | * Searching least priority queues on all CPU's first and most priority queues on all CPU's last. |
566 | * Searching least priority queues on all CPU's first and most priority |
| - | 567 | * queues on all CPU's last. |
|
| 559 | */ |
568 | */ |
| 560 | for (j=RQ_COUNT-1; j >= 0; j--) { |
569 | for (j= RQ_COUNT - 1; j >= 0; j--) { |
| 561 | for (i=0; i < config.cpu_active; i++) { |
570 | for (i = 0; i < config.cpu_active; i++) { |
| 562 | link_t *l; |
571 | link_t *l; |
| 563 | runq_t *r; |
572 | runq_t *r; |
| 564 | cpu_t *cpu; |
573 | cpu_t *cpu; |
| 565 | 574 | ||
| 566 | cpu = &cpus[(i + k) % config.cpu_active]; |
575 | cpu = &cpus[(i + k) % config.cpu_active]; |
| 567 | 576 | ||
| 568 | /* |
577 | /* |
| 569 | * Not interested in ourselves. |
578 | * Not interested in ourselves. |
| 570 | * Doesn't require interrupt disabling for kcpulb has THREAD_FLAG_WIRED. |
579 | * Doesn't require interrupt disabling for kcpulb has |
| - | 580 | * THREAD_FLAG_WIRED. |
|
| 571 | */ |
581 | */ |
| 572 | if (CPU == cpu) |
582 | if (CPU == cpu) |
| 573 | continue; |
583 | continue; |
| 574 | if (atomic_get(&cpu->nrdy) <= average) |
584 | if (atomic_get(&cpu->nrdy) <= average) |
| 575 | continue; |
585 | continue; |
| Line 586... | Line 596... | ||
| 586 | t = NULL; |
596 | t = NULL; |
| 587 | l = r->rq_head.prev; /* search rq from the back */ |
597 | l = r->rq_head.prev; /* search rq from the back */ |
| 588 | while (l != &r->rq_head) { |
598 | while (l != &r->rq_head) { |
| 589 | t = list_get_instance(l, thread_t, rq_link); |
599 | t = list_get_instance(l, thread_t, rq_link); |
| 590 | /* |
600 | /* |
| 591 | * We don't want to steal CPU-wired threads neither threads already |
601 | * We don't want to steal CPU-wired threads |
| - | 602 | * neither threads already stolen. The latter |
|
| 592 | * stolen. The latter prevents threads from migrating between CPU's |
603 | * prevents threads from migrating between CPU's |
| 593 | * without ever being run. We don't want to steal threads whose FPU |
604 | * without ever being run. We don't want to |
| 594 | * context is still in CPU. |
605 | * steal threads whose FPU context is still in |
| - | 606 | * CPU. |
|
| 595 | */ |
607 | */ |
| 596 | spinlock_lock(&t->lock); |
608 | spinlock_lock(&t->lock); |
| 597 | if ((!(t->flags & (THREAD_FLAG_WIRED | THREAD_FLAG_STOLEN))) && |
609 | if ((!(t->flags & (THREAD_FLAG_WIRED | |
| - | 610 | THREAD_FLAG_STOLEN))) && |
|
| 598 | (!(t->fpu_context_engaged)) ) { |
611 | (!(t->fpu_context_engaged)) ) { |
| 599 | /* |
612 | /* |
| 600 | * Remove t from r. |
613 | * Remove t from r. |
| 601 | */ |
614 | */ |
| 602 | spinlock_unlock(&t->lock); |
615 | spinlock_unlock(&t->lock); |
| Line 619... | Line 632... | ||
| 619 | /* |
632 | /* |
| 620 | * Ready t on local CPU |
633 | * Ready t on local CPU |
| 621 | */ |
634 | */ |
| 622 | spinlock_lock(&t->lock); |
635 | spinlock_lock(&t->lock); |
| 623 | #ifdef KCPULB_VERBOSE |
636 | #ifdef KCPULB_VERBOSE |
| 624 | printf("kcpulb%d: TID %d -> cpu%d, nrdy=%ld, avg=%nd\n", |
637 | printf("kcpulb%d: TID %d -> cpu%d, nrdy=%ld, " |
| 625 | CPU->id, t->tid, CPU->id, atomic_get(&CPU->nrdy), |
638 | "avg=%nd\n", CPU->id, t->tid, CPU->id, |
| - | 639 | atomic_get(&CPU->nrdy), |
|
| 626 | atomic_get(&nrdy) / config.cpu_active); |
640 | atomic_get(&nrdy) / config.cpu_active); |
| 627 | #endif |
641 | #endif |
| 628 | t->flags |= THREAD_FLAG_STOLEN; |
642 | t->flags |= THREAD_FLAG_STOLEN; |
| 629 | t->state = Entering; |
643 | t->state = Entering; |
| 630 | spinlock_unlock(&t->lock); |
644 | spinlock_unlock(&t->lock); |
| Line 635... | Line 649... | ||
| 635 | 649 | ||
| 636 | if (--count == 0) |
650 | if (--count == 0) |
| 637 | goto satisfied; |
651 | goto satisfied; |
| 638 | 652 | ||
| 639 | /* |
653 | /* |
| 640 | * We are not satisfied yet, focus on another CPU next time. |
654 | * We are not satisfied yet, focus on another |
| - | 655 | * CPU next time. |
|
| 641 | */ |
656 | */ |
| 642 | k++; |
657 | k++; |
| 643 | 658 | ||
| 644 | continue; |
659 | continue; |
| 645 | } |
660 | } |
| Line 686... | Line 701... | ||
| 686 | if (!cpus[cpu].active) |
701 | if (!cpus[cpu].active) |
| 687 | continue; |
702 | continue; |
| 688 | 703 | ||
| 689 | spinlock_lock(&cpus[cpu].lock); |
704 | spinlock_lock(&cpus[cpu].lock); |
| 690 | printf("cpu%d: address=%p, nrdy=%ld, needs_relink=%ld\n", |
705 | printf("cpu%d: address=%p, nrdy=%ld, needs_relink=%ld\n", |
| 691 | cpus[cpu].id, &cpus[cpu], atomic_get(&cpus[cpu].nrdy), cpus[cpu].needs_relink); |
706 | cpus[cpu].id, &cpus[cpu], atomic_get(&cpus[cpu].nrdy), |
| - | 707 | cpus[cpu].needs_relink); |
|
| 692 | 708 | ||
| 693 | for (i=0; i<RQ_COUNT; i++) { |
709 | for (i = 0; i < RQ_COUNT; i++) { |
| 694 | r = &cpus[cpu].rq[i]; |
710 | r = &cpus[cpu].rq[i]; |
| 695 | spinlock_lock(&r->lock); |
711 | spinlock_lock(&r->lock); |
| 696 | if (!r->n) { |
712 | if (!r->n) { |
| 697 | spinlock_unlock(&r->lock); |
713 | spinlock_unlock(&r->lock); |
| 698 | continue; |
714 | continue; |
| 699 | } |
715 | } |
| 700 | printf("\trq[%d]: ", i); |
716 | printf("\trq[%d]: ", i); |
| 701 | for (cur=r->rq_head.next; cur!=&r->rq_head; cur=cur->next) { |
717 | for (cur = r->rq_head.next; cur != &r->rq_head; |
| - | 718 | cur = cur->next) { |
|
| 702 | t = list_get_instance(cur, thread_t, rq_link); |
719 | t = list_get_instance(cur, thread_t, rq_link); |
| 703 | printf("%d(%s) ", t->tid, |
720 | printf("%d(%s) ", t->tid, |
| 704 | thread_states[t->state]); |
721 | thread_states[t->state]); |
| 705 | } |
722 | } |
| 706 | printf("\n"); |
723 | printf("\n"); |
| 707 | spinlock_unlock(&r->lock); |
724 | spinlock_unlock(&r->lock); |
| 708 | } |
725 | } |
| 709 | spinlock_unlock(&cpus[cpu].lock); |
726 | spinlock_unlock(&cpus[cpu].lock); |