Subversion Repositories HelenOS

/*

 * Copyright (c) 2006 Jakub Jermar

 * Copyright (c) 2009 Pavel Rimsky

 * All rights reserved.

 *

 * Redistribution and use in source and binary forms, with or without

 * modification, are permitted provided that the following conditions

 * are met:

 *

 * - Redistributions of source code must retain the above copyright

 *   notice, this list of conditions and the following disclaimer.

 * - Redistributions in binary form must reproduce the above copyright

 *   notice, this list of conditions and the following disclaimer in the

 *   documentation and/or other materials provided with the distribution.

 * - The name of the author may not be used to endorse or promote products

 *   derived from this software without specific prior written permission.

 *

 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR

 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES

 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.

 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,

 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT

 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,

 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY

 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF

 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 */

/** @addtogroup sparc64

 * @{

 */

/** @file

 */

#include <smp/smp.h>

#include <smp/ipi.h>

#include <genarch/ofw/ofw_tree.h>

#include <cpu.h>

#include <arch/cpu.h>

#include <arch/boot/boot.h>

#include <arch.h>

#include <config.h>

#include <macros.h>

#include <func.h>

#include <arch/types.h>

#include <synch/synch.h>

#include <synch/waitq.h>

#include <print.h>

#include <arch/sun4v/hypercall.h>

#include <arch/sun4v/md.h>

#include <arch/sun4v/ipi.h>

#include <time/delay.h>

#include <arch/smp/sun4v/smp.h>

/** hypervisor code of the "running" state of the CPU */

#define CPU_STATE_RUNNING   2

/** maximum possible number of processor cores */

#define MAX_NUM_CORES       8

/** needed in the CPU_START hypercall */

extern void kernel_image_start(void);

/** needed in the CPU_START hypercall */

extern void *trap_table;

/** number of execution units detected */

uint8_t exec_unit_count = 0;

/** execution units (processor cores) */

exec_unit_t exec_units[MAX_NUM_CORES];

/** CPU structures */

extern cpu_t *cpus;

/** maximum number of strands per a physical core detected */

unsigned int max_core_strands = 0;

#ifdef CONFIG_SIMICS_SMP_HACK

/**

 * Copies a piece of HelenOS code to the place where OBP had its IPI handler.

 * By sending an IPI by the BSP to the AP the code will be executed.

 * The code will jump to the first instruction of the kernel. This is

 * a workaround how to make APs execute HelenOS code on Simics.

 */

static void simics_smp_hack_init(void) {

    asm volatile (

        "setx temp_cpu_mondo_handler, %g4, %g6 \n"

        "setx 0x80200f80, %g4, %g7 \n"

        "ldx [%g6], %g4 \n"

        "stxa %g4, [%g7] 0x14 \n"

        "membar #Sync \n"

        "add %g7, 0x8, %g7 \n"

        "ldx [%g6 + 0x8], %g4 \n"

        "stxa %g4, [%g7] 0x14 \n"

        "membar #Sync \n"

        "add %g7, 0x8, %g7 \n"

        "ldx [%g6 + 0x10], %g4 \n"

        "stxa %g4, [%g7] 0x14 \n"

        "membar #Sync \n"

        "add %g7, 0x8, %g7 \n"

        "ldx [%g6 + 0x18], %g4 \n"

        "stxa %g4, [%g7] 0x14 \n"

        "membar #Sync \n"

        "add %g7, 0x8, %g7 \n"

        "ldx [%g6 + 0x20], %g4 \n"

        "stxa %g4, [%g7] 0x14 \n"

        "membar #Sync \n"

        "add %g7, 0x8, %g7 \n"

        "ldx [%g6 + 0x28], %g4 \n"

        "stxa %g4, [%g7] 0x14 \n"

        "membar #Sync \n"

        "add %g7, 0x8, %g7 \n"

        "ldx [%g6 + 0x30], %g4 \n"

        "stxa %g4, [%g7] 0x14 \n"

        "membar #Sync \n"

        "add %g7, 0x8, %g7 \n"

        "ldx [%g6 + 0x38], %g4 \n"

        "stxa %g4, [%g7] 0x14 \n"

        "membar #Sync \n"

        "add %g7, 0x8, %g7 \n"

        "ldx [%g6 + 0x40], %g4 \n"

        "stxa %g4, [%g7] 0x14 \n"

        "membar #Sync \n"

        "flush %i7"

        );

}

#endif

/**

 * Proposes the optimal number of ready threads for each virtual processor

 * in the given processor core so that the processor core is as busy as the

 * average processor core. The proposed number of ready threads will be

 * stored to the proposed_nrdy variable of the cpu_arch_t struture.

 */

bool calculate_optimal_nrdy(exec_unit_t *exec_unit) {

    /* calculate the number of threads the core will steal */

    int avg = atomic_get(&nrdy) / exec_unit_count;

    int to_steal = avg - atomic_get(&(exec_units->nrdy));

    if (to_steal < 0) {

        return true;

    } else if (to_steal == 0) {

        return false;

    }

    /* initialize the proposals with the real numbers of ready threads */

    unsigned int k;

    for (k = 0; k < exec_unit->strand_count; k++) {

        exec_units->cpus[k]->arch.proposed_nrdy =

            atomic_get(&(exec_unit->cpus[k]->nrdy));

    }

    /* distribute the threads to be stolen to the core's CPUs */

    int j;

    for (j = to_steal; j > 0; j--) {

        unsigned int k;

        unsigned int least_busy = 0;

        unsigned int least_busy_nrdy =

            exec_unit->cpus[0]->arch.proposed_nrdy;

        /* for each stolen thread, give it to the least busy CPU */

        for (k = 0; k < exec_unit->strand_count; k++) {

            if (exec_unit->cpus[k]->arch.proposed_nrdy

                    < least_busy_nrdy) {

                least_busy = k;

                least_busy_nrdy =

                    exec_unit->cpus[k]->arch.proposed_nrdy;

            }

        }

        exec_unit->cpus[least_busy]->arch.proposed_nrdy++;

    }

    return false;

}

/**

 * Finds out which execution units belong to particular CPUs. By execution unit

 * we mean the physical core the logical processor is backed by. Since each

 * Niagara physical core has just one integer execution unit and we will

 * ignore other execution units than the integer ones, we will use the terms

 * "integer execution unit", "execution unit" and "physical core"

 * interchangeably.

 *

 * The physical cores are detected by browsing the children of the CPU node

 * in the machine description and looking for a node representing an integer

 * execution unit. Once the integer execution unit of a particular CPU is

 * known, the ID of the CPU is added to the list of cpuids of the corresponding

 * execution unit structure (exec_unit_t). If an execution unit is encountered

 * for the first time, a new execution unit structure (exec_unit_t) must be

 * created first and added to the execution units array (exec_units).

 *

 * If the function fails to find an execution unit for a CPU (this may happen

 * on machines with older firmware or on Simics), it performs a fallback code

 * which pretends there exists just one execution unit and all CPUs belong to

 * it.

 *

 * Finally, the array of all execution units is reordered such that its element

 * which represents the physical core of the the bootstrap CPU is at index 0.

 * Moreover, the array of CPU IDs within the BSP's physical core structure is

 * reordered such that the element which represents the ID of the BSP is at

 * index 0. This is done because we would like the CPUs to be woken up

 * such that the 0-index CPU of the 0-index execution unit is

 * woken up first. And since the BSP is already woken up, we would like it to be

 * at 0-th position of the 0-th execution unit structure.

 *

 * Apart from that, the code also counts the total number of CPUs and stores

 * it to the global config.cpu_count variable.

 */

static void detect_execution_units(void)

{

    /* ID of the bootstrap processor */

    uint64_t myid;

    /* total number of CPUs detected */

    count_t cpu_count = 0;

    /* will be set to 1 if detecting the physical cores fails */

    bool exec_unit_assign_error = 0;

    /* index of the bootstrap physical core in the array of cores */

    unsigned int bsp_exec_unit_index = 0;

    /* index of the BSP ID inside the array of bootstrap core's cpuids */

    unsigned int bsp_core_strand_index = 0;

    __hypercall_fast_ret1(0, 0, 0, 0, 0, CPU_MYID, &myid);

    md_node_t node = md_get_root();

    /* walk through all the CPU nodes in the MD*/

    while (md_next_node(&node, "cpu")) {

        uint64_t cpuid;

        md_get_integer_property(node, "id", &cpuid);

        cpu_count++;

        /*

         * if failed in previous CPUs, don't try

         * to detect physical cores any more

         */

        if (exec_unit_assign_error)

            continue;

        /* detect exec. unit for the CPU represented by current node */

        uint64_t exec_unit_id = 0;

        md_child_iter_t it = md_get_child_iterator(node);

        while (md_next_child(&it)) {

            md_node_t child = md_get_child_node(it);

            const char *exec_unit_type;

            md_get_string_property(child, "type", &exec_unit_type);

            /* each physical core has just 1 integer exec. unit */

            if (strcmp(exec_unit_type, "integer") == 0) {

                exec_unit_id = child;

                break;

            }

        }

        /* execution unit detected successfully */

        if (exec_unit_id != 0) {

            /* find the exec. unit in array of existing units */

            unsigned int i = 0;

            for (i = 0; i < exec_unit_count; i++) {

                if (exec_units[i].exec_unit_id == exec_unit_id)

                    break;

            }

            /*

             * execution unit just met has not been met before, so

             * create a new entry in array of all execution units

             */

            if (i == exec_unit_count) {

                exec_units[i].exec_unit_id = exec_unit_id;

                exec_units[i].strand_count = 0;

                atomic_set(&(exec_units[i].nrdy), 0);

                spinlock_initialize(&(exec_units[i].proposed_nrdy_lock), "proposed nrdy lock");

                exec_unit_count++;

            }

            /*

             * remember the exec. unit and strand of the BSP

             */

            if (cpuid == myid) {

                bsp_exec_unit_index = i;

                bsp_core_strand_index = exec_units[i].strand_count;

            }

            /* add the CPU just met to the exec. unit's list */

            exec_units[i].cpuids[exec_units[i].strand_count] = cpuid;

            exec_units[i].strand_count++;

            max_core_strands =

                exec_units[i].strand_count > max_core_strands ?

                exec_units[i].strand_count : max_core_strands;

        /* detecting execution unit failed */

        } else {

            exec_unit_assign_error = 1;

        }

    }      

    /* save the number of CPUs to a globally accessible variable */

    config.cpu_count = cpu_count;

    /*

     * A fallback code which will be executed if finding out which

     * execution units belong to particular CPUs fails. Pretend there

     * exists just one execution unit and all CPUs belong to it.

     */

    if (exec_unit_assign_error) {

        bsp_exec_unit_index = 0;

        exec_unit_count = 1;

        exec_units[0].strand_count = cpu_count;

        exec_units[0].exec_unit_id = 1;

        spinlock_initialize(&(exec_units[0].proposed_nrdy_lock), "proposed nrdy lock");

        atomic_set(&(exec_units[0].nrdy), 0);

        max_core_strands = cpu_count;

        /* browse CPUs again, assign them the fictional exec. unit */

        node = md_get_root();

        unsigned int i = 0;

        while (md_next_node(&node, "cpu")) {

            uint64_t cpuid;

            md_get_integer_property(node, "id", &cpuid);

            if (cpuid == myid) {

                bsp_core_strand_index = i;

            }

            exec_units[0].cpuids[i++] = cpuid;

        }

    }

    /*

     * Reorder the execution units array elements and the cpuid array

     * elements so that the BSP will always be the very first CPU of

     * the very first execution unit.

     */

    exec_unit_t temp_exec_unit = exec_units[0];

    exec_units[0] = exec_units[bsp_exec_unit_index];

    exec_units[bsp_exec_unit_index] = temp_exec_unit;

    uint64_t temp_cpuid = exec_units[0].cpuids[0];

    exec_units[0].cpuids[0] = exec_units[0].cpuids[bsp_exec_unit_index];

    exec_units[0].cpuids[bsp_core_strand_index] = temp_cpuid;

}

/**

 * Determine number of processors and detect physical cores. On Simics

 * copy the code which will be executed by the AP when the BSP sends an

 * IPI to it in order to make it execute HelenOS code.

 */

void smp_init(void)

{

    detect_execution_units();

#ifdef CONFIG_SIMICS_SMP_HACK

    simics_smp_hack_init();

#endif

}

/**

 * For each CPU sets the value of cpus[i].arch.id, where i is the

 * index of the CPU in the cpus variable, to the cpuid of the i-th processor

 * to be run. The CPUs are run such that the CPU represented by cpus[0]

 * is run first, cpus[1] is run after it, and cpus[cpu_count - 1] is run as the

 * last one.

 *

 * The CPU IDs are set such that during waking the CPUs up the

 * processor cores will be alternated, i.e. first one CPU from the first core

 * will be run, after that one CPU from the second CPU core will be run,...

 * then one CPU from the last core will be run, after that another CPU

 * from the first core will be run, then another CPU from the second core

 * will be run,... then another CPU from the last core will be run, and so on.

 */

static void init_cpuids(void)

{

    unsigned int cur_core_strand;

    unsigned int cur_core;

    unsigned int cur_cpu = 0;

    for (cur_core_strand = 0; cur_core_strand < max_core_strands; cur_core_strand++) {

        for (cur_core = 0; cur_core < exec_unit_count; cur_core++) {

            if (cur_core_strand > exec_units[cur_core].strand_count)

                continue;

            cpus[cur_cpu].arch.exec_unit = &(exec_units[cur_core]);

            atomic_add(&(exec_units[cur_core].nrdy), atomic_get(&(cpus[cur_cpu].nrdy)));

            cpus[cur_cpu].arch.id = exec_units[cur_core].cpuids[cur_core_strand];

            exec_units[cur_core].cpus[cur_core_strand] = &(cpus[cur_cpu]);

            cur_cpu++;

        }

    }

}

/**

 * Wakes up a single CPU.

 *

 * @param cpuid ID of the CPU to be woken up

 */

static bool wake_cpu(uint64_t cpuid)

{

#ifdef CONFIG_SIMICS_SMP_HACK

    ipi_unicast_to((void (*)(void)) 1234, cpuid);

#else

    /* stop the CPU before making it execute our code */

    if (__hypercall_fast1(CPU_STOP, cpuid) != EOK)

        return false;

    /* wait for the CPU to stop */

    uint64_t state;

    __hypercall_fast_ret1(cpuid, 0, 0, 0, 0,

        CPU_STATE, &state);

    while (state == CPU_STATE_RUNNING) {

        __hypercall_fast_ret1(cpuid, 0, 0, 0, 0,

            CPU_STATE, &state);

    }

    /* make the CPU run again and execute HelenOS code */

    if (__hypercall_fast4(

        CPU_START, cpuid,

        (uint64_t) KA2PA(kernel_image_start),

        KA2PA(trap_table), bootinfo.physmem_start          

        ) != EOK)

            return false;

#endif

    if (waitq_sleep_timeout(&ap_completion_wq, 10000000, SYNCH_FLAGS_NONE) ==

            ESYNCH_TIMEOUT)

        printf("%s: waiting for processor (cpuid = %" PRIu32

        ") timed out\n", __func__, cpuid);

    return true;

}

/** Wake application processors up. */

void kmp(void *arg)

{

    init_cpuids();

    unsigned int i;

    for (i = 1; i < config.cpu_count; i++) {

        wake_cpu(cpus[i].arch.id);

    }

}

/** @}

 */