Subversion Repositories HelenOS

Rev

Rev 3057 | Rev 3187 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. /*
  2.  * Copyright (c) 2001-2005 Jakub Jermar
  3.  * Copyright (c) 2005 Sergey Bondari
  4.  * All rights reserved.
  5.  *
  6.  * Redistribution and use in source and binary forms, with or without
  7.  * modification, are permitted provided that the following conditions
  8.  * are met:
  9.  *
  10.  * - Redistributions of source code must retain the above copyright
  11.  *   notice, this list of conditions and the following disclaimer.
  12.  * - Redistributions in binary form must reproduce the above copyright
  13.  *   notice, this list of conditions and the following disclaimer in the
  14.  *   documentation and/or other materials provided with the distribution.
  15.  * - The name of the author may not be used to endorse or promote products
  16.  *   derived from this software without specific prior written permission.
  17.  *
  18.  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  19.  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  20.  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  21.  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  22.  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  23.  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24.  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25.  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26.  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  27.  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28.  */
  29.  
  30. /** @addtogroup genericmm
  31.  * @{
  32.  */
  33.  
  34. /**
  35.  * @file
  36.  * @brief   Physical frame allocator.
  37.  *
  38.  * This file contains the physical frame allocator and memory zone management.
  39.  * The frame allocator is built on top of the buddy allocator.
  40.  *
  41.  * @see buddy.c
  42.  */
  43.  
  44. /*
  45.  * Locking order
  46.  *
  47.  * In order to access particular zone, the process must first lock
  48.  * the zones.lock, then lock the zone and then unlock the zones.lock.
  49.  * This insures, that we can fiddle with the zones in runtime without
  50.  * affecting the processes.
  51.  *
  52.  */
  53.  
  54. #include <arch/types.h>
  55. #include <mm/frame.h>
  56. #include <mm/as.h>
  57. #include <panic.h>
  58. #include <debug.h>
  59. #include <adt/list.h>
  60. #include <synch/spinlock.h>
  61. #include <arch/asm.h>
  62. #include <arch.h>
  63. #include <print.h>
  64. #include <align.h>
  65. #include <mm/slab.h>
  66. #include <bitops.h>
  67. #include <macros.h>
  68. #include <config.h>
  69.  
  70. typedef struct {
  71.     count_t refcount;   /**< tracking of shared frames  */
  72.     uint8_t buddy_order;    /**< buddy system block order */
  73.     link_t buddy_link;  /**< link to the next free block inside one
  74.                      order */
  75.     void *parent;           /**< If allocated by slab, this points there */
  76. } frame_t;
  77.  
  78. typedef struct {
  79.     SPINLOCK_DECLARE(lock); /**< this lock protects everything below */
  80.     pfn_t base;     /**< frame_no of the first frame in the frames
  81.                      array */
  82.     count_t count;          /**< Size of zone */
  83.  
  84.     frame_t *frames;    /**< array of frame_t structures in this
  85.                      zone */
  86.     count_t free_count; /**< number of free frame_t structures */
  87.     count_t busy_count; /**< number of busy frame_t structures */
  88.    
  89.     buddy_system_t *buddy_system; /**< buddy system for the zone */
  90.     int flags;
  91. } zone_t;
  92.  
  93. /*
  94.  * The zoneinfo.lock must be locked when accessing zoneinfo structure.
  95.  * Some of the attributes in zone_t structures are 'read-only'
  96.  */
  97.  
  98. typedef struct {
  99.     SPINLOCK_DECLARE(lock);
  100.     unsigned int count;
  101.     zone_t *info[ZONES_MAX];
  102. } zones_t;
  103.  
  104. static zones_t zones;
  105.  
  106.  
  107. /********************/
  108. /* Helper functions */
  109. /********************/
  110.  
  111. static inline index_t frame_index(zone_t *zone, frame_t *frame)
  112. {
  113.     return (index_t) (frame - zone->frames);
  114. }
  115.  
  116. static inline index_t frame_index_abs(zone_t *zone, frame_t *frame)
  117. {
  118.     return (index_t) (frame - zone->frames) + zone->base;
  119. }
  120.  
  121. static inline int frame_index_valid(zone_t *zone, index_t index)
  122. {
  123.     return (index < zone->count);
  124. }
  125.  
  126. /** Compute pfn_t from frame_t pointer & zone pointer */
  127. static index_t make_frame_index(zone_t *zone, frame_t *frame)
  128. {
  129.     return (frame - zone->frames);
  130. }
  131.  
  132. /** Initialize frame structure.
  133.  *
  134.  * @param framei    Frame structure to be initialized.
  135.  */
  136. static void frame_initialize(frame_t *frame)
  137. {
  138.     frame->refcount = 1;
  139.     frame->buddy_order = 0;
  140. }
  141.  
  142. /**********************/
  143. /* Zoneinfo functions */
  144. /**********************/
  145.  
  146. /** Insert-sort zone into zones list.
  147.  *
  148.  * @param newzone   New zone to be inserted into zone list.
  149.  * @return      Zone number on success, -1 on error.
  150.  */
  151. static int zones_add_zone(zone_t *newzone)
  152. {
  153.     unsigned int i, j;
  154.     ipl_t ipl;
  155.     zone_t *z;
  156.  
  157.     ipl = interrupts_disable();
  158.     spinlock_lock(&zones.lock);
  159.    
  160.     /* Try to merge */
  161.     if (zones.count + 1 == ZONES_MAX) {
  162.         printf("Maximum zone count %u exceeded!\n", ZONES_MAX);
  163.         spinlock_unlock(&zones.lock);
  164.         interrupts_restore(ipl);
  165.         return -1;
  166.     }
  167.    
  168.     for (i = 0; i < zones.count; i++) {
  169.         /* Check for overflow */
  170.         z = zones.info[i];
  171.         if (overlaps(newzone->base, newzone->count, z->base,
  172.             z->count)) {
  173.             printf("Zones overlap!\n");
  174.             return -1;
  175.         }
  176.         if (newzone->base < z->base)
  177.             break;
  178.     }
  179.    
  180.     /* Move other zones up */
  181.     for (j = i; j < zones.count; j++)
  182.         zones.info[j + 1] = zones.info[j];
  183.    
  184.     zones.info[i] = newzone;
  185.     zones.count++;
  186.    
  187.     spinlock_unlock(&zones.lock);
  188.     interrupts_restore(ipl);
  189.  
  190.     return i;
  191. }
  192.  
  193. /**
  194.  * Try to find a zone where can we find the frame.
  195.  *
  196.  * Assume interrupts are disabled.
  197.  *
  198.  * @param frame     Frame number contained in zone.
  199.  * @param pzone     If not null, it is used as zone hint. Zone index is
  200.  *          filled into the variable on success.
  201.  * @return      Pointer to locked zone containing frame.
  202.  */
  203. static zone_t *find_zone_and_lock(pfn_t frame, unsigned int *pzone)
  204. {
  205.     unsigned int i;
  206.     unsigned int hint = pzone ? *pzone : 0;
  207.     zone_t *z;
  208.    
  209.     spinlock_lock(&zones.lock);
  210.  
  211.     if (hint >= zones.count)
  212.         hint = 0;
  213.    
  214.     i = hint;
  215.     do {
  216.         z = zones.info[i];
  217.         spinlock_lock(&z->lock);
  218.         if (z->base <= frame && z->base + z->count > frame) {
  219.             /* Unlock the global lock */
  220.             spinlock_unlock(&zones.lock);
  221.             if (pzone)
  222.                 *pzone = i;
  223.             return z;
  224.         }
  225.         spinlock_unlock(&z->lock);
  226.  
  227.         i++;
  228.         if (i >= zones.count)
  229.             i = 0;
  230.     } while(i != hint);
  231.  
  232.     spinlock_unlock(&zones.lock);
  233.     return NULL;
  234. }
  235.  
  236. /** @return True if zone can allocate specified order */
  237. static int zone_can_alloc(zone_t *z, uint8_t order)
  238. {
  239.     return buddy_system_can_alloc(z->buddy_system, order);
  240. }
  241.  
  242. /** Find and lock zone that can allocate order frames.
  243.  *
  244.  * Assume interrupts are disabled.
  245.  *
  246.  * @param order     Size (2^order) of free space we are trying to find.
  247.  * @param pzone     Pointer to preferred zone or NULL, on return contains
  248.  *          zone number.
  249.  */
  250. static zone_t *find_free_zone_and_lock(uint8_t order, unsigned int *pzone)
  251. {
  252.     unsigned int i;
  253.     zone_t *z;
  254.     unsigned int hint = pzone ? *pzone : 0;
  255.    
  256.     spinlock_lock(&zones.lock);
  257.     if (hint >= zones.count)
  258.         hint = 0;
  259.     i = hint;
  260.     do {
  261.         z = zones.info[i];
  262.        
  263.         spinlock_lock(&z->lock);
  264.  
  265.         /* Check if the zone has 2^order frames area available  */
  266.         if (zone_can_alloc(z, order)) {
  267.             spinlock_unlock(&zones.lock);
  268.             if (pzone)
  269.                 *pzone = i;
  270.             return z;
  271.         }
  272.         spinlock_unlock(&z->lock);
  273.         if (++i >= zones.count)
  274.             i = 0;
  275.     } while(i != hint);
  276.     spinlock_unlock(&zones.lock);
  277.     return NULL;
  278. }
  279.  
  280. /**************************/
  281. /* Buddy system functions */
  282. /**************************/
  283.  
  284. /** Buddy system find_block implementation.
  285.  *
  286.  * Find block that is parent of current list.
  287.  * That means go to lower addresses, until such block is found
  288.  *
  289.  * @param order     Order of parent must be different then this
  290.  *          parameter!!
  291.  */
  292. static link_t *zone_buddy_find_block(buddy_system_t *b, link_t *child,
  293.     uint8_t order)
  294. {
  295.     frame_t *frame;
  296.     zone_t *zone;
  297.     index_t index;
  298.    
  299.     frame = list_get_instance(child, frame_t, buddy_link);
  300.     zone = (zone_t *) b->data;
  301.  
  302.     index = frame_index(zone, frame);
  303.     do {
  304.         if (zone->frames[index].buddy_order != order) {
  305.             return &zone->frames[index].buddy_link;
  306.         }
  307.     } while(index-- > 0);
  308.     return NULL;
  309. }
  310.  
  311. static void zone_buddy_print_id(buddy_system_t *b, link_t *block)
  312. {
  313.     frame_t *frame;
  314.     zone_t *zone;
  315.     index_t index;
  316.  
  317.     frame = list_get_instance(block, frame_t, buddy_link);
  318.     zone = (zone_t *) b->data;
  319.     index = frame_index(zone, frame);
  320.     printf("%" PRIi, index);
  321. }                    
  322.  
  323. /** Buddy system find_buddy implementation.
  324.  *
  325.  * @param b     Buddy system.
  326.  * @param block     Block for which buddy should be found.
  327.  *
  328.  * @return      Buddy for given block if found.
  329.  */
  330. static link_t *zone_buddy_find_buddy(buddy_system_t *b, link_t *block)
  331. {
  332.     frame_t *frame;
  333.     zone_t *zone;
  334.     index_t index;
  335.     bool is_left, is_right;
  336.  
  337.     frame = list_get_instance(block, frame_t, buddy_link);
  338.     zone = (zone_t *) b->data;
  339.     ASSERT(IS_BUDDY_ORDER_OK(frame_index_abs(zone, frame),
  340.         frame->buddy_order));
  341.    
  342.     is_left = IS_BUDDY_LEFT_BLOCK_ABS(zone, frame);
  343.     is_right = IS_BUDDY_RIGHT_BLOCK_ABS(zone, frame);
  344.  
  345.     ASSERT(is_left ^ is_right);
  346.     if (is_left) {
  347.         index = (frame_index(zone, frame)) +
  348.             (1 << frame->buddy_order);
  349.     } else {    /* if (is_right) */
  350.         index = (frame_index(zone, frame)) -
  351.             (1 << frame->buddy_order);
  352.     }
  353.    
  354.     if (frame_index_valid(zone, index)) {
  355.         if (zone->frames[index].buddy_order == frame->buddy_order &&
  356.             zone->frames[index].refcount == 0) {
  357.             return &zone->frames[index].buddy_link;
  358.         }
  359.     }
  360.  
  361.     return NULL;   
  362. }
  363.  
  364. /** Buddy system bisect implementation.
  365.  *
  366.  * @param b     Buddy system.
  367.  * @param block     Block to bisect.
  368.  *
  369.  * @return      Right block.
  370.  */
  371. static link_t *zone_buddy_bisect(buddy_system_t *b, link_t *block)
  372. {
  373.     frame_t *frame_l, *frame_r;
  374.  
  375.     frame_l = list_get_instance(block, frame_t, buddy_link);
  376.     frame_r = (frame_l + (1 << (frame_l->buddy_order - 1)));
  377.    
  378.     return &frame_r->buddy_link;
  379. }
  380.  
  381. /** Buddy system coalesce implementation.
  382.  *
  383.  * @param b     Buddy system.
  384.  * @param block_1   First block.
  385.  * @param block_2   First block's buddy.
  386.  *
  387.  * @return      Coalesced block (actually block that represents lower
  388.  *          address).
  389.  */
  390. static link_t *zone_buddy_coalesce(buddy_system_t *b, link_t *block_1,
  391.     link_t *block_2)
  392. {
  393.     frame_t *frame1, *frame2;
  394.    
  395.     frame1 = list_get_instance(block_1, frame_t, buddy_link);
  396.     frame2 = list_get_instance(block_2, frame_t, buddy_link);
  397.    
  398.     return frame1 < frame2 ? block_1 : block_2;
  399. }
  400.  
  401. /** Buddy system set_order implementation.
  402.  *
  403.  * @param b     Buddy system.
  404.  * @param block     Buddy system block.
  405.  * @param order     Order to set.
  406.  */
  407. static void zone_buddy_set_order(buddy_system_t *b, link_t *block,
  408.     uint8_t order)
  409. {
  410.     frame_t *frame;
  411.     frame = list_get_instance(block, frame_t, buddy_link);
  412.     frame->buddy_order = order;
  413. }
  414.  
  415. /** Buddy system get_order implementation.
  416.  *
  417.  * @param b     Buddy system.
  418.  * @param block     Buddy system block.
  419.  *
  420.  * @return      Order of block.
  421.  */
  422. static uint8_t zone_buddy_get_order(buddy_system_t *b, link_t *block)
  423. {
  424.     frame_t *frame;
  425.     frame = list_get_instance(block, frame_t, buddy_link);
  426.     return frame->buddy_order;
  427. }
  428.  
  429. /** Buddy system mark_busy implementation.
  430.  *
  431.  * @param b     Buddy system.
  432.  * @param block     Buddy system block.
  433.  */
  434. static void zone_buddy_mark_busy(buddy_system_t *b, link_t * block)
  435. {
  436.     frame_t * frame;
  437.  
  438.     frame = list_get_instance(block, frame_t, buddy_link);
  439.     frame->refcount = 1;
  440. }
  441.  
  442. /** Buddy system mark_available implementation.
  443.  *
  444.  * @param b     Buddy system.
  445.  * @param block     Buddy system block.
  446.  */
  447. static void zone_buddy_mark_available(buddy_system_t *b, link_t *block)
  448. {
  449.     frame_t *frame;
  450.     frame = list_get_instance(block, frame_t, buddy_link);
  451.     frame->refcount = 0;
  452. }
  453.  
  454. static buddy_system_operations_t zone_buddy_system_operations = {
  455.     .find_buddy = zone_buddy_find_buddy,
  456.     .bisect = zone_buddy_bisect,
  457.     .coalesce = zone_buddy_coalesce,
  458.     .set_order = zone_buddy_set_order,
  459.     .get_order = zone_buddy_get_order,
  460.     .mark_busy = zone_buddy_mark_busy,
  461.     .mark_available = zone_buddy_mark_available,
  462.     .find_block = zone_buddy_find_block,
  463.     .print_id = zone_buddy_print_id
  464. };
  465.  
  466. /******************/
  467. /* Zone functions */
  468. /******************/
  469.  
  470. /** Allocate frame in particular zone.
  471.  *
  472.  * Assume zone is locked.
  473.  * Panics if allocation is impossible.
  474.  *
  475.  * @param zone      Zone to allocate from.
  476.  * @param order     Allocate exactly 2^order frames.
  477.  *
  478.  * @return      Frame index in zone.
  479.  *
  480.  */
  481. static pfn_t zone_frame_alloc(zone_t *zone, uint8_t order)
  482. {
  483.     pfn_t v;
  484.     link_t *tmp;
  485.     frame_t *frame;
  486.  
  487.     /* Allocate frames from zone buddy system */
  488.     tmp = buddy_system_alloc(zone->buddy_system, order);
  489.    
  490.     ASSERT(tmp);
  491.    
  492.     /* Update zone information. */
  493.     zone->free_count -= (1 << order);
  494.     zone->busy_count += (1 << order);
  495.  
  496.     /* Frame will be actually a first frame of the block. */
  497.     frame = list_get_instance(tmp, frame_t, buddy_link);
  498.    
  499.     /* get frame address */
  500.     v = make_frame_index(zone, frame);
  501.     return v;
  502. }
  503.  
  504. /** Free frame from zone.
  505.  *
  506.  * Assume zone is locked.
  507.  *
  508.  * @param zone      Pointer to zone from which the frame is to be freed.
  509.  * @param frame_idx Frame index relative to zone.
  510.  */
  511. static void zone_frame_free(zone_t *zone, index_t frame_idx)
  512. {
  513.     frame_t *frame;
  514.     uint8_t order;
  515.  
  516.     frame = &zone->frames[frame_idx];
  517.    
  518.     /* remember frame order */
  519.     order = frame->buddy_order;
  520.  
  521.     ASSERT(frame->refcount);
  522.  
  523.     if (!--frame->refcount) {
  524.         buddy_system_free(zone->buddy_system, &frame->buddy_link);
  525.    
  526.         /* Update zone information. */
  527.         zone->free_count += (1 << order);
  528.         zone->busy_count -= (1 << order);
  529.     }
  530. }
  531.  
  532. /** Return frame from zone. */
  533. static frame_t * zone_get_frame(zone_t *zone, index_t frame_idx)
  534. {
  535.     ASSERT(frame_idx < zone->count);
  536.     return &zone->frames[frame_idx];
  537. }
  538.  
  539. /** Mark frame in zone unavailable to allocation. */
  540. static void zone_mark_unavailable(zone_t *zone, index_t frame_idx)
  541. {
  542.     frame_t *frame;
  543.     link_t *link;
  544.  
  545.     frame = zone_get_frame(zone, frame_idx);
  546.     if (frame->refcount)
  547.         return;
  548.     link = buddy_system_alloc_block(zone->buddy_system,
  549.         &frame->buddy_link);
  550.     ASSERT(link);
  551.     zone->free_count--;
  552. }
  553.  
  554. /** Join two zones.
  555.  *
  556.  * Expect zone_t *z to point to space at least zone_conf_size large.
  557.  *
  558.  * Assume z1 & z2 are locked.
  559.  *
  560.  * @param z     Target zone structure pointer.
  561.  * @param z1        Zone to merge.
  562.  * @param z2        Zone to merge.
  563.  */
  564. static void _zone_merge(zone_t *z, zone_t *z1, zone_t *z2)
  565. {
  566.     uint8_t max_order;
  567.     unsigned int i;
  568.     int z2idx;
  569.     pfn_t frame_idx;
  570.     frame_t *frame;
  571.  
  572.     ASSERT(!overlaps(z1->base, z1->count, z2->base, z2->count));
  573.     ASSERT(z1->base < z2->base);
  574.  
  575.     spinlock_initialize(&z->lock, "zone_lock");
  576.     z->base = z1->base;
  577.     z->count = z2->base + z2->count - z1->base;
  578.     z->flags = z1->flags & z2->flags;
  579.  
  580.     z->free_count = z1->free_count + z2->free_count;
  581.     z->busy_count = z1->busy_count + z2->busy_count;
  582.    
  583.     max_order = fnzb(z->count);
  584.  
  585.     z->buddy_system = (buddy_system_t *) &z[1];
  586.     buddy_system_create(z->buddy_system, max_order,
  587.         &zone_buddy_system_operations, (void *) z);
  588.  
  589.     z->frames = (frame_t *)((uint8_t *) z->buddy_system +
  590.         buddy_conf_size(max_order));
  591.     for (i = 0; i < z->count; i++) {
  592.         /* This marks all frames busy */
  593.         frame_initialize(&z->frames[i]);
  594.     }
  595.     /* Copy frames from both zones to preserve full frame orders,
  596.      * parents etc. Set all free frames with refcount=0 to 1, because
  597.      * we add all free frames to buddy allocator later again, clear
  598.      * order to 0. Don't set busy frames with refcount=0, as they
  599.      * will not be reallocated during merge and it would make later
  600.      * problems with allocation/free.
  601.      */
  602.     for (i = 0; i < z1->count; i++)
  603.         z->frames[i] = z1->frames[i];
  604.     for (i = 0; i < z2->count; i++) {
  605.         z2idx = i + (z2->base - z1->base);
  606.         z->frames[z2idx] = z2->frames[i];
  607.     }
  608.     i = 0;
  609.     while (i < z->count) {
  610.         if (z->frames[i].refcount) {
  611.             /* skip busy frames */
  612.             i += 1 << z->frames[i].buddy_order;
  613.         } else { /* Free frames, set refcount=1 */
  614.             /* All free frames have refcount=0, we need not
  615.              * to check the order */
  616.             z->frames[i].refcount = 1;
  617.             z->frames[i].buddy_order = 0;
  618.             i++;
  619.         }
  620.     }
  621.     /* Add free blocks from the 2 original zones */
  622.     while (zone_can_alloc(z1, 0)) {
  623.         frame_idx = zone_frame_alloc(z1, 0);
  624.         frame = &z->frames[frame_idx];
  625.         frame->refcount = 0;
  626.         buddy_system_free(z->buddy_system, &frame->buddy_link);
  627.     }
  628.     while (zone_can_alloc(z2, 0)) {
  629.         frame_idx = zone_frame_alloc(z2, 0);
  630.         frame = &z->frames[frame_idx + (z2->base - z1->base)];
  631.         frame->refcount = 0;
  632.         buddy_system_free(z->buddy_system, &frame->buddy_link);
  633.     }
  634. }
  635.  
  636. /** Return old configuration frames into the zone.
  637.  *
  638.  * We have several cases
  639.  * - the conf. data is outside of zone -> exit, shall we call frame_free??
  640.  * - the conf. data was created by zone_create or
  641.  *   updated with reduce_region -> free every frame
  642.  *
  643.  * @param newzone   The actual zone where freeing should occur.
  644.  * @param oldzone   Pointer to old zone configuration data that should
  645.  *          be freed from new zone.
  646.  */
  647. static void return_config_frames(zone_t *newzone, zone_t *oldzone)
  648. {
  649.     pfn_t pfn;
  650.     frame_t *frame;
  651.     count_t cframes;
  652.     unsigned int i;
  653.  
  654.     pfn = ADDR2PFN((uintptr_t)KA2PA(oldzone));
  655.     cframes = SIZE2FRAMES(zone_conf_size(oldzone->count));
  656.    
  657.     if (pfn < newzone->base || pfn >= newzone->base + newzone->count)
  658.         return;
  659.  
  660.     frame = &newzone->frames[pfn - newzone->base];
  661.     ASSERT(!frame->buddy_order);
  662.  
  663.     for (i = 0; i < cframes; i++) {
  664.         newzone->busy_count++;
  665.         zone_frame_free(newzone, pfn+i-newzone->base);
  666.     }
  667. }
  668.  
  669. /** Reduce allocated block to count of order 0 frames.
  670.  *
  671.  * The allocated block need 2^order frames of space. Reduce all frames
  672.  * in block to order 0 and free the unneeded frames. This means, that
  673.  * when freeing the previously allocated block starting with frame_idx,
  674.  * you have to free every frame.
  675.  *
  676.  * @param zone
  677.  * @param frame_idx     Index to block.
  678.  * @param count         Allocated space in block.
  679.  */
  680. static void zone_reduce_region(zone_t *zone, pfn_t frame_idx, count_t count)
  681. {
  682.     count_t i;
  683.     uint8_t order;
  684.     frame_t *frame;
  685.    
  686.     ASSERT(frame_idx + count < zone->count);
  687.  
  688.     order = zone->frames[frame_idx].buddy_order;
  689.     ASSERT((count_t) (1 << order) >= count);
  690.  
  691.     /* Reduce all blocks to order 0 */
  692.     for (i = 0; i < (count_t) (1 << order); i++) {
  693.         frame = &zone->frames[i + frame_idx];
  694.         frame->buddy_order = 0;
  695.         if (!frame->refcount)
  696.             frame->refcount = 1;
  697.         ASSERT(frame->refcount == 1);
  698.     }
  699.     /* Free unneeded frames */
  700.     for (i = count; i < (count_t) (1 << order); i++) {
  701.         zone_frame_free(zone, i + frame_idx);
  702.     }
  703. }
  704.  
  705. /** Merge zones z1 and z2.
  706.  *
  707.  * - the zones must be 2 zones with no zone existing in between,
  708.  *   which means that z2 = z1+1
  709.  *
  710.  * - When you create a new zone, the frame allocator configuration does
  711.  *   not to be 2^order size. Once the allocator is running it is no longer
  712.  *   possible, merged configuration data occupies more space :-/
  713.  */
  714. void zone_merge(unsigned int z1, unsigned int z2)
  715. {
  716.     ipl_t ipl;
  717.     zone_t *zone1, *zone2, *newzone;
  718.     unsigned int cframes;
  719.     uint8_t order;
  720.     unsigned int i;
  721.     pfn_t pfn;
  722.  
  723.     ipl = interrupts_disable();
  724.     spinlock_lock(&zones.lock);
  725.  
  726.     if ((z1 >= zones.count) || (z2 >= zones.count))
  727.         goto errout;
  728.     /* We can join only 2 zones with none existing inbetween */
  729.     if (z2-z1 != 1)
  730.         goto errout;
  731.  
  732.     zone1 = zones.info[z1];
  733.     zone2 = zones.info[z2];
  734.     spinlock_lock(&zone1->lock);
  735.     spinlock_lock(&zone2->lock);
  736.  
  737.     cframes = SIZE2FRAMES(zone_conf_size(zone2->base + zone2->count -
  738.         zone1->base));
  739.     if (cframes == 1)
  740.         order = 0;
  741.     else
  742.         order = fnzb(cframes - 1) + 1;
  743.  
  744.     /* Allocate zonedata inside one of the zones */
  745.     if (zone_can_alloc(zone1, order))
  746.         pfn = zone1->base + zone_frame_alloc(zone1, order);
  747.     else if (zone_can_alloc(zone2, order))
  748.         pfn = zone2->base + zone_frame_alloc(zone2, order);
  749.     else
  750.         goto errout2;
  751.  
  752.     newzone = (zone_t *) PA2KA(PFN2ADDR(pfn));
  753.  
  754.     _zone_merge(newzone, zone1, zone2);
  755.  
  756.     /* Free unneeded config frames */
  757.     zone_reduce_region(newzone, pfn - newzone->base,  cframes);
  758.     /* Subtract zone information from busy frames */
  759.     newzone->busy_count -= cframes;
  760.  
  761.     /* Replace existing zones in zoneinfo list */
  762.     zones.info[z1] = newzone;
  763.     for (i = z2 + 1; i < zones.count; i++)
  764.         zones.info[i - 1] = zones.info[i];
  765.     zones.count--;
  766.  
  767.     /* Free old zone information */
  768.     return_config_frames(newzone, zone1);
  769.     return_config_frames(newzone, zone2);
  770. errout2:
  771.     /* Nobody is allowed to enter to zone, so we are safe
  772.      * to touch the spinlocks last time */
  773.     spinlock_unlock(&zone1->lock);
  774.     spinlock_unlock(&zone2->lock);
  775. errout:
  776.     spinlock_unlock(&zones.lock);
  777.     interrupts_restore(ipl);
  778. }
  779.  
  780. /** Merge all zones into one big zone.
  781.  *
  782.  * It is reasonable to do this on systems whose bios reports parts in chunks,
  783.  * so that we could have 1 zone (it's faster).
  784.  */
  785. void zone_merge_all(void)
  786. {
  787.     int count = zones.count;
  788.  
  789.     while (zones.count > 1 && --count) {
  790.         zone_merge(0,1);
  791.         break;
  792.     }
  793. }
  794.  
  795. /** Create new frame zone.
  796.  *
  797.  * @param start     Physical address of the first frame within the zone.
  798.  * @param count     Count of frames in zone.
  799.  * @param z     Address of configuration information of zone.
  800.  * @param flags     Zone flags.
  801.  *
  802.  * @return      Initialized zone.
  803.  */
  804. static void zone_construct(pfn_t start, count_t count, zone_t *z, int flags)
  805. {
  806.     unsigned int i;
  807.     uint8_t max_order;
  808.  
  809.     spinlock_initialize(&z->lock, "zone_lock");
  810.     z->base = start;
  811.     z->count = count;
  812.     z->flags = flags;
  813.     z->free_count = count;
  814.     z->busy_count = 0;
  815.  
  816.     /*
  817.      * Compute order for buddy system, initialize
  818.      */
  819.     max_order = fnzb(count);
  820.     z->buddy_system = (buddy_system_t *)&z[1];
  821.    
  822.     buddy_system_create(z->buddy_system, max_order,
  823.         &zone_buddy_system_operations, (void *) z);
  824.    
  825.     /* Allocate frames _after_ the conframe */
  826.     /* Check sizes */
  827.     z->frames = (frame_t *)((uint8_t *) z->buddy_system +
  828.         buddy_conf_size(max_order));
  829.     for (i = 0; i < count; i++) {
  830.         frame_initialize(&z->frames[i]);
  831.     }
  832.    
  833.     /* Stuffing frames */
  834.     for (i = 0; i < count; i++) {
  835.         z->frames[i].refcount = 0;
  836.         buddy_system_free(z->buddy_system, &z->frames[i].buddy_link);
  837.     }
  838. }
  839.  
  840. /** Compute configuration data size for zone.
  841.  *
  842.  * @param count     Size of zone in frames.
  843.  * @return      Size of zone configuration info (in bytes).
  844.  */
  845. uintptr_t zone_conf_size(count_t count)
  846. {
  847.     int size = sizeof(zone_t) + count * sizeof(frame_t);
  848.     int max_order;
  849.  
  850.     max_order = fnzb(count);
  851.     size += buddy_conf_size(max_order);
  852.     return size;
  853. }
  854.  
  855. /** Create and add zone to system.
  856.  *
  857.  * @param start     First frame number (absolute).
  858.  * @param count     Size of zone in frames.
  859.  * @param confframe Where configuration frames are supposed to be.
  860.  *          Automatically checks, that we will not disturb the
  861.  *          kernel and possibly init.  If confframe is given
  862.  *          _outside_ this zone, it is expected, that the area is
  863.  *          already marked BUSY and big enough to contain
  864.  *          zone_conf_size() amount of data.  If the confframe is
  865.  *          inside the area, the zone free frame information is
  866.  *          modified not to include it.
  867.  *
  868.  * @return      Zone number or -1 on error.
  869.  */
  870. int zone_create(pfn_t start, count_t count, pfn_t confframe, int flags)
  871. {
  872.     zone_t *z;
  873.     uintptr_t addr;
  874.     count_t confcount;
  875.     unsigned int i;
  876.     int znum;
  877.  
  878.     /* Theoretically we could have here 0, practically make sure
  879.      * nobody tries to do that. If some platform requires, remove
  880.      * the assert
  881.      */
  882.     ASSERT(confframe);
  883.     /* If conframe is supposed to be inside our zone, then make sure
  884.      * it does not span kernel & init
  885.      */
  886.     confcount = SIZE2FRAMES(zone_conf_size(count));
  887.     if (confframe >= start && confframe < start+count) {
  888.         for (;confframe < start + count; confframe++) {
  889.             addr = PFN2ADDR(confframe);
  890.             if (overlaps(addr, PFN2ADDR(confcount),
  891.                 KA2PA(config.base), config.kernel_size))
  892.                 continue;
  893.            
  894.             if (overlaps(addr, PFN2ADDR(confcount),
  895.                 KA2PA(config.stack_base), config.stack_size))
  896.                 continue;
  897.            
  898.             bool overlap = false;
  899.             count_t i;
  900.             for (i = 0; i < init.cnt; i++)
  901.                 if (overlaps(addr, PFN2ADDR(confcount),
  902.                     KA2PA(init.tasks[i].addr),
  903.                     init.tasks[i].size)) {
  904.                     overlap = true;
  905.                     break;
  906.                 }
  907.             if (overlap)
  908.                 continue;
  909.            
  910.             break;
  911.         }
  912.         if (confframe >= start + count)
  913.             panic("Cannot find configuration data for zone.");
  914.     }
  915.  
  916.     z = (zone_t *) PA2KA(PFN2ADDR(confframe));
  917.     zone_construct(start, count, z, flags);
  918.     znum = zones_add_zone(z);
  919.     if (znum == -1)
  920.         return -1;
  921.  
  922.     /* If confdata in zone, mark as unavailable */
  923.     if (confframe >= start && confframe < start + count)
  924.         for (i = confframe; i < confframe + confcount; i++) {
  925.             zone_mark_unavailable(z, i - z->base);
  926.         }
  927.     return znum;
  928. }
  929.  
  930. /***************************************/
  931. /* Frame functions */
  932.  
  933. /** Set parent of frame. */
  934. void frame_set_parent(pfn_t pfn, void *data, unsigned int hint)
  935. {
  936.     zone_t *zone = find_zone_and_lock(pfn, &hint);
  937.  
  938.     ASSERT(zone);
  939.  
  940.     zone_get_frame(zone, pfn-zone->base)->parent = data;
  941.     spinlock_unlock(&zone->lock);
  942. }
  943.  
  944. void *frame_get_parent(pfn_t pfn, unsigned int hint)
  945. {
  946.     zone_t *zone = find_zone_and_lock(pfn, &hint);
  947.     void *res;
  948.  
  949.     ASSERT(zone);
  950.     res = zone_get_frame(zone, pfn - zone->base)->parent;
  951.    
  952.     spinlock_unlock(&zone->lock);
  953.     return res;
  954. }
  955.  
  956. /** Allocate power-of-two frames of physical memory.
  957.  *
  958.  * @param order     Allocate exactly 2^order frames.
  959.  * @param flags     Flags for host zone selection and address processing.
  960.  * @param pzone     Preferred zone.
  961.  *
  962.  * @return      Physical address of the allocated frame.
  963.  *
  964.  */
  965. void *frame_alloc_generic(uint8_t order, int flags, unsigned int *pzone)
  966. {
  967.     ipl_t ipl;
  968.     int freed;
  969.     pfn_t v;
  970.     zone_t *zone;
  971.    
  972. loop:
  973.     ipl = interrupts_disable();
  974.    
  975.     /*
  976.      * First, find suitable frame zone.
  977.      */
  978.     zone = find_free_zone_and_lock(order, pzone);
  979.    
  980.     /* If no memory, reclaim some slab memory,
  981.        if it does not help, reclaim all */
  982.     if (!zone && !(flags & FRAME_NO_RECLAIM)) {
  983.         freed = slab_reclaim(0);
  984.         if (freed)
  985.             zone = find_free_zone_and_lock(order, pzone);
  986.         if (!zone) {
  987.             freed = slab_reclaim(SLAB_RECLAIM_ALL);
  988.             if (freed)
  989.                 zone = find_free_zone_and_lock(order, pzone);
  990.         }
  991.     }
  992.     if (!zone) {
  993.         /*
  994.          * TODO: Sleep until frames are available again.
  995.          */
  996.         interrupts_restore(ipl);
  997.  
  998.         if (flags & FRAME_ATOMIC)
  999.             return 0;
  1000.        
  1001.         panic("Sleep not implemented.\n");
  1002.         goto loop;
  1003.     }
  1004.    
  1005.     v = zone_frame_alloc(zone, order);
  1006.     v += zone->base;
  1007.  
  1008.     spinlock_unlock(&zone->lock);
  1009.     interrupts_restore(ipl);
  1010.  
  1011.     if (flags & FRAME_KA)
  1012.         return (void *)PA2KA(PFN2ADDR(v));
  1013.     return (void *)PFN2ADDR(v);
  1014. }
  1015.  
  1016. /** Free a frame.
  1017.  *
  1018.  * Find respective frame structure for supplied physical frame address.
  1019.  * Decrement frame reference count.
  1020.  * If it drops to zero, move the frame structure to free list.
  1021.  *
  1022.  * @param frame     Physical Address of of the frame to be freed.
  1023.  */
  1024. void frame_free(uintptr_t frame)
  1025. {
  1026.     ipl_t ipl;
  1027.     zone_t *zone;
  1028.     pfn_t pfn = ADDR2PFN(frame);
  1029.  
  1030.     ipl = interrupts_disable();
  1031.    
  1032.     /*
  1033.      * First, find host frame zone for addr.
  1034.      */
  1035.     zone = find_zone_and_lock(pfn,NULL);
  1036.     ASSERT(zone);
  1037.    
  1038.     zone_frame_free(zone, pfn-zone->base);
  1039.    
  1040.     spinlock_unlock(&zone->lock);
  1041.     interrupts_restore(ipl);
  1042. }
  1043.  
  1044. /** Add reference to frame.
  1045.  *
  1046.  * Find respective frame structure for supplied PFN and
  1047.  * increment frame reference count.
  1048.  *
  1049.  * @param pfn       Frame number of the frame to be freed.
  1050.  */
  1051. void frame_reference_add(pfn_t pfn)
  1052. {
  1053.     ipl_t ipl;
  1054.     zone_t *zone;
  1055.     frame_t *frame;
  1056.  
  1057.     ipl = interrupts_disable();
  1058.    
  1059.     /*
  1060.      * First, find host frame zone for addr.
  1061.      */
  1062.     zone = find_zone_and_lock(pfn, NULL);
  1063.     ASSERT(zone);
  1064.    
  1065.     frame = &zone->frames[pfn - zone->base];
  1066.     frame->refcount++;
  1067.    
  1068.     spinlock_unlock(&zone->lock);
  1069.     interrupts_restore(ipl);
  1070. }
  1071.  
  1072. /** Mark given range unavailable in frame zones. */
  1073. void frame_mark_unavailable(pfn_t start, count_t count)
  1074. {
  1075.     unsigned int i;
  1076.     zone_t *zone;
  1077.     unsigned int prefzone = 0;
  1078.    
  1079.     for (i = 0; i < count; i++) {
  1080.         zone = find_zone_and_lock(start + i, &prefzone);
  1081.         if (!zone) /* PFN not found */
  1082.             continue;
  1083.         zone_mark_unavailable(zone, start + i - zone->base);
  1084.  
  1085.         spinlock_unlock(&zone->lock);
  1086.     }
  1087. }
  1088.  
  1089. /** Initialize physical memory management. */
  1090. void frame_init(void)
  1091. {
  1092.     if (config.cpu_active == 1) {
  1093.         zones.count = 0;
  1094.         spinlock_initialize(&zones.lock, "zones.lock");
  1095.     }
  1096.     /* Tell the architecture to create some memory */
  1097.     frame_arch_init();
  1098.     if (config.cpu_active == 1) {
  1099.         frame_mark_unavailable(ADDR2PFN(KA2PA(config.base)),
  1100.             SIZE2FRAMES(config.kernel_size));
  1101.         frame_mark_unavailable(ADDR2PFN(KA2PA(config.stack_base)),
  1102.             SIZE2FRAMES(config.stack_size));
  1103.        
  1104.         count_t i;
  1105.         for (i = 0; i < init.cnt; i++) {
  1106.             pfn_t pfn = ADDR2PFN(KA2PA(init.tasks[i].addr));
  1107.             frame_mark_unavailable(pfn,
  1108.                 SIZE2FRAMES(init.tasks[i].size));
  1109.         }
  1110.  
  1111.         if (ballocs.size)
  1112.             frame_mark_unavailable(ADDR2PFN(KA2PA(ballocs.base)),
  1113.                 SIZE2FRAMES(ballocs.size));
  1114.  
  1115.         /* Black list first frame, as allocating NULL would
  1116.          * fail in some places */
  1117.         frame_mark_unavailable(0, 1);
  1118.     }
  1119. }
  1120.  
  1121.  
  1122. /** Return total size of all zones. */
  1123. uint64_t zone_total_size(void)
  1124. {
  1125.     zone_t *zone = NULL;
  1126.     unsigned int i;
  1127.     ipl_t ipl;
  1128.     uint64_t total = 0;
  1129.  
  1130.     ipl = interrupts_disable();
  1131.     spinlock_lock(&zones.lock);
  1132.    
  1133.     for (i = 0; i < zones.count; i++) {
  1134.         zone = zones.info[i];
  1135.         spinlock_lock(&zone->lock);
  1136.         total += (uint64_t) FRAMES2SIZE(zone->count);
  1137.         spinlock_unlock(&zone->lock);
  1138.     }
  1139.    
  1140.     spinlock_unlock(&zones.lock);
  1141.     interrupts_restore(ipl);
  1142.    
  1143.     return total;
  1144. }
  1145.  
  1146. /** Prints list of zones. */
  1147. void zone_print_list(void)
  1148. {
  1149.     zone_t *zone = NULL;
  1150.     unsigned int i;
  1151.     ipl_t ipl;
  1152.  
  1153.     ipl = interrupts_disable();
  1154.     spinlock_lock(&zones.lock);
  1155.  
  1156. #ifdef __32_BITS__ 
  1157.     printf("#  base address free frames  busy frames\n");
  1158.     printf("-- ------------ ------------ ------------\n");
  1159. #endif
  1160.  
  1161. #ifdef __64_BITS__
  1162.     printf("#  base address         free frames  busy frames\n");
  1163.     printf("-- -------------------- ------------ ------------\n");
  1164. #endif
  1165.    
  1166.     for (i = 0; i < zones.count; i++) {
  1167.         zone = zones.info[i];
  1168.         spinlock_lock(&zone->lock);
  1169.  
  1170. #ifdef __32_BITS__
  1171.         printf("%-2u   %10p %12" PRIc " %12" PRIc "\n",
  1172.             i, PFN2ADDR(zone->base), zone->free_count,
  1173.             zone->busy_count);
  1174. #endif
  1175.  
  1176. #ifdef __64_BITS__
  1177.         printf("%-2u   %18p %12" PRIc " %12" PRIc "\n", i,
  1178.             PFN2ADDR(zone->base), zone->free_count, zone->busy_count);
  1179. #endif
  1180.        
  1181.         spinlock_unlock(&zone->lock);
  1182.     }
  1183.    
  1184.     spinlock_unlock(&zones.lock);
  1185.     interrupts_restore(ipl);
  1186. }
  1187.  
  1188. /** Prints zone details.
  1189.  *
  1190.  * @param num       Zone base address or zone number.
  1191.  */
  1192. void zone_print_one(unsigned int num)
  1193. {
  1194.     zone_t *zone = NULL;
  1195.     ipl_t ipl;
  1196.     unsigned int i;
  1197.  
  1198.     ipl = interrupts_disable();
  1199.     spinlock_lock(&zones.lock);
  1200.  
  1201.     for (i = 0; i < zones.count; i++) {
  1202.         if ((i == num) || (PFN2ADDR(zones.info[i]->base) == num)) {
  1203.             zone = zones.info[i];
  1204.             break;
  1205.         }
  1206.     }
  1207.     if (!zone) {
  1208.         printf("Zone not found.\n");
  1209.         goto out;
  1210.     }
  1211.    
  1212.     spinlock_lock(&zone->lock);
  1213.     printf("Memory zone information\n");
  1214.     printf("Zone base address: %p\n", PFN2ADDR(zone->base));
  1215.     printf("Zone size: %" PRIc " frames (%" PRIs " KB)\n", zone->count,
  1216.         SIZE2KB(FRAMES2SIZE(zone->count)));
  1217.     printf("Allocated space: %" PRIc " frames (%" PRIs " KB)\n",
  1218.         zone->busy_count, SIZE2KB(FRAMES2SIZE(zone->busy_count)));
  1219.     printf("Available space: %" PRIc " frames (%" PRIs " KB)\n",
  1220.         zone->free_count, SIZE2KB(FRAMES2SIZE(zone->free_count)));
  1221.     buddy_system_structure_print(zone->buddy_system, FRAME_SIZE);
  1222.     spinlock_unlock(&zone->lock);
  1223.    
  1224. out:
  1225.     spinlock_unlock(&zones.lock);
  1226.     interrupts_restore(ipl);
  1227. }
  1228.  
  1229. /** @}
  1230.  */
  1231.  
  1232.