Subversion Repositories HelenOS

Rev

Rev 3205 | Rev 3207 | Go to most recent revision | Only display areas with differences | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 3205 Rev 3206
1
/*
1
/*
2
 * Copyright (c) 2001-2005 Jakub Jermar
2
 * Copyright (c) 2001-2005 Jakub Jermar
3
 * Copyright (c) 2005 Sergey Bondari
3
 * Copyright (c) 2005 Sergey Bondari
4
 * All rights reserved.
4
 * All rights reserved.
5
 *
5
 *
6
 * Redistribution and use in source and binary forms, with or without
6
 * Redistribution and use in source and binary forms, with or without
7
 * modification, are permitted provided that the following conditions
7
 * modification, are permitted provided that the following conditions
8
 * are met:
8
 * are met:
9
 *
9
 *
10
 * - Redistributions of source code must retain the above copyright
10
 * - Redistributions of source code must retain the above copyright
11
 *   notice, this list of conditions and the following disclaimer.
11
 *   notice, this list of conditions and the following disclaimer.
12
 * - Redistributions in binary form must reproduce the above copyright
12
 * - Redistributions in binary form must reproduce the above copyright
13
 *   notice, this list of conditions and the following disclaimer in the
13
 *   notice, this list of conditions and the following disclaimer in the
14
 *   documentation and/or other materials provided with the distribution.
14
 *   documentation and/or other materials provided with the distribution.
15
 * - The name of the author may not be used to endorse or promote products
15
 * - The name of the author may not be used to endorse or promote products
16
 *   derived from this software without specific prior written permission.
16
 *   derived from this software without specific prior written permission.
17
 *
17
 *
18
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
22
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
23
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
27
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
 */
28
 */
29
 
29
 
30
/** @addtogroup genericmm
30
/** @addtogroup genericmm
31
 * @{
31
 * @{
32
 */
32
 */
33
 
33
 
34
/**
34
/**
35
 * @file
35
 * @file
36
 * @brief   Physical frame allocator.
36
 * @brief   Physical frame allocator.
37
 *
37
 *
38
 * This file contains the physical frame allocator and memory zone management.
38
 * This file contains the physical frame allocator and memory zone management.
39
 * The frame allocator is built on top of the buddy allocator.
39
 * The frame allocator is built on top of the buddy allocator.
40
 *
40
 *
41
 * @see buddy.c
41
 * @see buddy.c
42
 */
42
 */
43
 
43
 
44
/*
44
/*
45
 * Locking order
45
 * Locking order
46
 *
46
 *
47
 * In order to access particular zone, the process must first lock
47
 * In order to access particular zone, the process must first lock
48
 * the zones.lock, then lock the zone and then unlock the zones.lock.
48
 * the zones.lock, then lock the zone and then unlock the zones.lock.
49
 * This insures, that we can fiddle with the zones in runtime without
49
 * This insures, that we can fiddle with the zones in runtime without
50
 * affecting the processes.
50
 * affecting the processes.
51
 *
51
 *
52
 */
52
 */
53
 
53
 
54
#include <arch/types.h>
54
#include <arch/types.h>
55
#include <mm/frame.h>
55
#include <mm/frame.h>
56
#include <mm/as.h>
56
#include <mm/as.h>
57
#include <panic.h>
57
#include <panic.h>
58
#include <debug.h>
58
#include <debug.h>
59
#include <adt/list.h>
59
#include <adt/list.h>
60
#include <synch/spinlock.h>
60
#include <synch/spinlock.h>
61
#include <synch/mutex.h>
61
#include <synch/mutex.h>
62
#include <synch/condvar.h>
62
#include <synch/condvar.h>
63
#include <arch/asm.h>
63
#include <arch/asm.h>
64
#include <arch.h>
64
#include <arch.h>
65
#include <print.h>
65
#include <print.h>
66
#include <align.h>
66
#include <align.h>
67
#include <mm/slab.h>
67
#include <mm/slab.h>
68
#include <bitops.h>
68
#include <bitops.h>
69
#include <macros.h>
69
#include <macros.h>
70
#include <config.h>
70
#include <config.h>
71
 
71
 
72
typedef struct {
72
typedef struct {
73
    count_t refcount;   /**< tracking of shared frames  */
73
    count_t refcount;   /**< tracking of shared frames  */
74
    uint8_t buddy_order;    /**< buddy system block order */
74
    uint8_t buddy_order;    /**< buddy system block order */
75
    link_t buddy_link;  /**< link to the next free block inside one
75
    link_t buddy_link;  /**< link to the next free block inside one
76
                     order */
76
                     order */
77
    void *parent;           /**< If allocated by slab, this points there */
77
    void *parent;           /**< If allocated by slab, this points there */
78
} frame_t;
78
} frame_t;
79
 
79
 
80
typedef struct {
80
typedef struct {
81
    SPINLOCK_DECLARE(lock); /**< this lock protects everything below */
81
    SPINLOCK_DECLARE(lock); /**< this lock protects everything below */
82
    pfn_t base;     /**< frame_no of the first frame in the frames
82
    pfn_t base;     /**< frame_no of the first frame in the frames
83
                     array */
83
                     array */
84
    count_t count;          /**< Size of zone */
84
    count_t count;          /**< Size of zone */
85
 
85
 
86
    frame_t *frames;    /**< array of frame_t structures in this
86
    frame_t *frames;    /**< array of frame_t structures in this
87
                     zone */
87
                     zone */
88
    count_t free_count; /**< number of free frame_t structures */
88
    count_t free_count; /**< number of free frame_t structures */
89
    count_t busy_count; /**< number of busy frame_t structures */
89
    count_t busy_count; /**< number of busy frame_t structures */
90
   
90
   
91
    buddy_system_t *buddy_system; /**< buddy system for the zone */
91
    buddy_system_t *buddy_system; /**< buddy system for the zone */
92
    int flags;
92
    int flags;
93
} zone_t;
93
} zone_t;
94
 
94
 
95
/*
95
/*
96
 * The zoneinfo.lock must be locked when accessing zoneinfo structure.
96
 * The zoneinfo.lock must be locked when accessing zoneinfo structure.
97
 * Some of the attributes in zone_t structures are 'read-only'
97
 * Some of the attributes in zone_t structures are 'read-only'
98
 */
98
 */
99
 
99
 
100
typedef struct {
100
typedef struct {
101
    SPINLOCK_DECLARE(lock);
101
    SPINLOCK_DECLARE(lock);
102
    unsigned int count;
102
    unsigned int count;
103
    zone_t *info[ZONES_MAX];
103
    zone_t *info[ZONES_MAX];
104
} zones_t;
104
} zones_t;
105
 
105
 
106
static zones_t zones;
106
static zones_t zones;
107
 
107
 
108
/*
108
/*
109
 * Synchronization primitives used to sleep when there is no memory
109
 * Synchronization primitives used to sleep when there is no memory
110
 * available.
110
 * available.
111
 */
111
 */
112
mutex_t mem_avail_mtx;
112
mutex_t mem_avail_mtx;
113
condvar_t mem_avail_cv;
113
condvar_t mem_avail_cv;
114
unsigned long mem_avail_frames = 0; /**< Number of available frames. */
114
unsigned long mem_avail_frames = 0; /**< Number of available frames. */
115
unsigned long mem_avail_gen = 0;    /**< Generation counter. */
115
unsigned long mem_avail_gen = 0;    /**< Generation counter. */
116
 
116
 
117
/********************/
117
/********************/
118
/* Helper functions */
118
/* Helper functions */
119
/********************/
119
/********************/
120
 
120
 
121
static inline index_t frame_index(zone_t *zone, frame_t *frame)
121
static inline index_t frame_index(zone_t *zone, frame_t *frame)
122
{
122
{
123
    return (index_t) (frame - zone->frames);
123
    return (index_t) (frame - zone->frames);
124
}
124
}
125
 
125
 
126
static inline index_t frame_index_abs(zone_t *zone, frame_t *frame)
126
static inline index_t frame_index_abs(zone_t *zone, frame_t *frame)
127
{
127
{
128
    return (index_t) (frame - zone->frames) + zone->base;
128
    return (index_t) (frame - zone->frames) + zone->base;
129
}
129
}
130
 
130
 
131
static inline int frame_index_valid(zone_t *zone, index_t index)
131
static inline int frame_index_valid(zone_t *zone, index_t index)
132
{
132
{
133
    return (index < zone->count);
133
    return (index < zone->count);
134
}
134
}
135
 
135
 
136
/** Compute pfn_t from frame_t pointer & zone pointer */
136
/** Compute pfn_t from frame_t pointer & zone pointer */
137
static index_t make_frame_index(zone_t *zone, frame_t *frame)
137
static index_t make_frame_index(zone_t *zone, frame_t *frame)
138
{
138
{
139
    return (frame - zone->frames);
139
    return (frame - zone->frames);
140
}
140
}
141
 
141
 
142
/** Initialize frame structure.
142
/** Initialize frame structure.
143
 *
143
 *
144
 * @param frame     Frame structure to be initialized.
144
 * @param frame     Frame structure to be initialized.
145
 */
145
 */
146
static void frame_initialize(frame_t *frame)
146
static void frame_initialize(frame_t *frame)
147
{
147
{
148
    frame->refcount = 1;
148
    frame->refcount = 1;
149
    frame->buddy_order = 0;
149
    frame->buddy_order = 0;
150
}
150
}
151
 
151
 
152
/**********************/
152
/**********************/
153
/* Zoneinfo functions */
153
/* Zoneinfo functions */
154
/**********************/
154
/**********************/
155
 
155
 
156
/** Insert-sort zone into zones list.
156
/** Insert-sort zone into zones list.
157
 *
157
 *
158
 * @param newzone   New zone to be inserted into zone list.
158
 * @param newzone   New zone to be inserted into zone list.
159
 * @return      Zone number on success, -1 on error.
159
 * @return      Zone number on success, -1 on error.
160
 */
160
 */
161
static int zones_add_zone(zone_t *newzone)
161
static int zones_add_zone(zone_t *newzone)
162
{
162
{
163
    unsigned int i, j;
163
    unsigned int i, j;
164
    ipl_t ipl;
164
    ipl_t ipl;
165
    zone_t *z;
165
    zone_t *z;
166
 
166
 
167
    ipl = interrupts_disable();
167
    ipl = interrupts_disable();
168
    spinlock_lock(&zones.lock);
168
    spinlock_lock(&zones.lock);
169
   
169
   
170
    /* Try to merge */
170
    /* Try to merge */
171
    if (zones.count + 1 == ZONES_MAX) {
171
    if (zones.count + 1 == ZONES_MAX) {
172
        printf("Maximum zone count %u exceeded!\n", ZONES_MAX);
172
        printf("Maximum zone count %u exceeded!\n", ZONES_MAX);
173
        spinlock_unlock(&zones.lock);
173
        spinlock_unlock(&zones.lock);
174
        interrupts_restore(ipl);
174
        interrupts_restore(ipl);
175
        return -1;
175
        return -1;
176
    }
176
    }
177
   
177
   
178
    for (i = 0; i < zones.count; i++) {
178
    for (i = 0; i < zones.count; i++) {
179
        /* Check for overflow */
179
        /* Check for overflow */
180
        z = zones.info[i];
180
        z = zones.info[i];
181
        if (overlaps(newzone->base, newzone->count, z->base,
181
        if (overlaps(newzone->base, newzone->count, z->base,
182
            z->count)) {
182
            z->count)) {
183
            printf("Zones overlap!\n");
183
            printf("Zones overlap!\n");
184
            return -1;
184
            return -1;
185
        }
185
        }
186
        if (newzone->base < z->base)
186
        if (newzone->base < z->base)
187
            break;
187
            break;
188
    }
188
    }
189
   
189
   
190
    /* Move other zones up */
190
    /* Move other zones up */
191
    for (j = i; j < zones.count; j++)
191
    for (j = i; j < zones.count; j++)
192
        zones.info[j + 1] = zones.info[j];
192
        zones.info[j + 1] = zones.info[j];
193
   
193
   
194
    zones.info[i] = newzone;
194
    zones.info[i] = newzone;
195
    zones.count++;
195
    zones.count++;
196
   
196
   
197
    spinlock_unlock(&zones.lock);
197
    spinlock_unlock(&zones.lock);
198
    interrupts_restore(ipl);
198
    interrupts_restore(ipl);
199
 
199
 
200
    return i;
200
    return i;
201
}
201
}
202
 
202
 
203
/**
203
/**
204
 * Try to find a zone where can we find the frame.
204
 * Try to find a zone where can we find the frame.
205
 *
205
 *
206
 * Assume interrupts are disabled.
206
 * Assume interrupts are disabled.
207
 *
207
 *
208
 * @param frame     Frame number contained in zone.
208
 * @param frame     Frame number contained in zone.
209
 * @param pzone     If not null, it is used as zone hint. Zone index is
209
 * @param pzone     If not null, it is used as zone hint. Zone index is
210
 *          filled into the variable on success.
210
 *          filled into the variable on success.
211
 * @return      Pointer to locked zone containing frame.
211
 * @return      Pointer to locked zone containing frame.
212
 */
212
 */
213
static zone_t *find_zone_and_lock(pfn_t frame, unsigned int *pzone)
213
static zone_t *find_zone_and_lock(pfn_t frame, unsigned int *pzone)
214
{
214
{
215
    unsigned int i;
215
    unsigned int i;
216
    unsigned int hint = pzone ? *pzone : 0;
216
    unsigned int hint = pzone ? *pzone : 0;
217
    zone_t *z;
217
    zone_t *z;
218
   
218
   
219
    spinlock_lock(&zones.lock);
219
    spinlock_lock(&zones.lock);
220
 
220
 
221
    if (hint >= zones.count)
221
    if (hint >= zones.count)
222
        hint = 0;
222
        hint = 0;
223
   
223
   
224
    i = hint;
224
    i = hint;
225
    do {
225
    do {
226
        z = zones.info[i];
226
        z = zones.info[i];
227
        spinlock_lock(&z->lock);
227
        spinlock_lock(&z->lock);
228
        if (z->base <= frame && z->base + z->count > frame) {
228
        if (z->base <= frame && z->base + z->count > frame) {
229
            /* Unlock the global lock */
229
            /* Unlock the global lock */
230
            spinlock_unlock(&zones.lock);
230
            spinlock_unlock(&zones.lock);
231
            if (pzone)
231
            if (pzone)
232
                *pzone = i;
232
                *pzone = i;
233
            return z;
233
            return z;
234
        }
234
        }
235
        spinlock_unlock(&z->lock);
235
        spinlock_unlock(&z->lock);
236
 
236
 
237
        i++;
237
        i++;
238
        if (i >= zones.count)
238
        if (i >= zones.count)
239
            i = 0;
239
            i = 0;
240
    } while(i != hint);
240
    } while(i != hint);
241
 
241
 
242
    spinlock_unlock(&zones.lock);
242
    spinlock_unlock(&zones.lock);
243
    return NULL;
243
    return NULL;
244
}
244
}
245
 
245
 
246
/** @return True if zone can allocate specified order */
246
/** @return True if zone can allocate specified order */
247
static int zone_can_alloc(zone_t *z, uint8_t order)
247
static int zone_can_alloc(zone_t *z, uint8_t order)
248
{
248
{
249
    return buddy_system_can_alloc(z->buddy_system, order);
249
    return buddy_system_can_alloc(z->buddy_system, order);
250
}
250
}
251
 
251
 
252
/** Find and lock zone that can allocate order frames.
252
/** Find and lock zone that can allocate order frames.
253
 *
253
 *
254
 * Assume interrupts are disabled.
254
 * Assume interrupts are disabled.
255
 *
255
 *
256
 * @param order     Size (2^order) of free space we are trying to find.
256
 * @param order     Size (2^order) of free space we are trying to find.
257
 * @param pzone     Pointer to preferred zone or NULL, on return contains
257
 * @param pzone     Pointer to preferred zone or NULL, on return contains
258
 *          zone number.
258
 *          zone number.
259
 */
259
 */
260
static zone_t *find_free_zone_and_lock(uint8_t order, unsigned int *pzone)
260
static zone_t *find_free_zone_and_lock(uint8_t order, unsigned int *pzone)
261
{
261
{
262
    unsigned int i;
262
    unsigned int i;
263
    zone_t *z;
263
    zone_t *z;
264
    unsigned int hint = pzone ? *pzone : 0;
264
    unsigned int hint = pzone ? *pzone : 0;
265
   
265
   
266
    spinlock_lock(&zones.lock);
266
    spinlock_lock(&zones.lock);
267
    if (hint >= zones.count)
267
    if (hint >= zones.count)
268
        hint = 0;
268
        hint = 0;
269
    i = hint;
269
    i = hint;
270
    do {
270
    do {
271
        z = zones.info[i];
271
        z = zones.info[i];
272
       
272
       
273
        spinlock_lock(&z->lock);
273
        spinlock_lock(&z->lock);
274
 
274
 
275
        /* Check if the zone has 2^order frames area available  */
275
        /* Check if the zone has 2^order frames area available  */
276
        if (zone_can_alloc(z, order)) {
276
        if (zone_can_alloc(z, order)) {
277
            spinlock_unlock(&zones.lock);
277
            spinlock_unlock(&zones.lock);
278
            if (pzone)
278
            if (pzone)
279
                *pzone = i;
279
                *pzone = i;
280
            return z;
280
            return z;
281
        }
281
        }
282
        spinlock_unlock(&z->lock);
282
        spinlock_unlock(&z->lock);
283
        if (++i >= zones.count)
283
        if (++i >= zones.count)
284
            i = 0;
284
            i = 0;
285
    } while(i != hint);
285
    } while(i != hint);
286
    spinlock_unlock(&zones.lock);
286
    spinlock_unlock(&zones.lock);
287
    return NULL;
287
    return NULL;
288
}
288
}
289
 
289
 
290
/**************************/
290
/**************************/
291
/* Buddy system functions */
291
/* Buddy system functions */
292
/**************************/
292
/**************************/
293
 
293
 
294
/** Buddy system find_block implementation.
294
/** Buddy system find_block implementation.
295
 *
295
 *
296
 * Find block that is parent of current list.
296
 * Find block that is parent of current list.
297
 * That means go to lower addresses, until such block is found
297
 * That means go to lower addresses, until such block is found
298
 *
298
 *
299
 * @param order     Order of parent must be different then this
299
 * @param order     Order of parent must be different then this
300
 *          parameter!!
300
 *          parameter!!
301
 */
301
 */
302
static link_t *zone_buddy_find_block(buddy_system_t *b, link_t *child,
302
static link_t *zone_buddy_find_block(buddy_system_t *b, link_t *child,
303
    uint8_t order)
303
    uint8_t order)
304
{
304
{
305
    frame_t *frame;
305
    frame_t *frame;
306
    zone_t *zone;
306
    zone_t *zone;
307
    index_t index;
307
    index_t index;
308
   
308
   
309
    frame = list_get_instance(child, frame_t, buddy_link);
309
    frame = list_get_instance(child, frame_t, buddy_link);
310
    zone = (zone_t *) b->data;
310
    zone = (zone_t *) b->data;
311
 
311
 
312
    index = frame_index(zone, frame);
312
    index = frame_index(zone, frame);
313
    do {
313
    do {
314
        if (zone->frames[index].buddy_order != order) {
314
        if (zone->frames[index].buddy_order != order) {
315
            return &zone->frames[index].buddy_link;
315
            return &zone->frames[index].buddy_link;
316
        }
316
        }
317
    } while(index-- > 0);
317
    } while(index-- > 0);
318
    return NULL;
318
    return NULL;
319
}
319
}
320
 
320
 
321
static void zone_buddy_print_id(buddy_system_t *b, link_t *block)
-
 
322
{
-
 
323
    frame_t *frame;
-
 
324
    zone_t *zone;
-
 
325
    index_t index;
-
 
326
 
-
 
327
    frame = list_get_instance(block, frame_t, buddy_link);
-
 
328
    zone = (zone_t *) b->data;
-
 
329
    index = frame_index(zone, frame);
-
 
330
    printf("%" PRIi, index);
-
 
331
}                    
-
 
332
 
-
 
333
/** Buddy system find_buddy implementation.
321
/** Buddy system find_buddy implementation.
334
 *
322
 *
335
 * @param b     Buddy system.
323
 * @param b     Buddy system.
336
 * @param block     Block for which buddy should be found.
324
 * @param block     Block for which buddy should be found.
337
 *
325
 *
338
 * @return      Buddy for given block if found.
326
 * @return      Buddy for given block if found.
339
 */
327
 */
340
static link_t *zone_buddy_find_buddy(buddy_system_t *b, link_t *block)
328
static link_t *zone_buddy_find_buddy(buddy_system_t *b, link_t *block)
341
{
329
{
342
    frame_t *frame;
330
    frame_t *frame;
343
    zone_t *zone;
331
    zone_t *zone;
344
    index_t index;
332
    index_t index;
345
    bool is_left, is_right;
333
    bool is_left, is_right;
346
 
334
 
347
    frame = list_get_instance(block, frame_t, buddy_link);
335
    frame = list_get_instance(block, frame_t, buddy_link);
348
    zone = (zone_t *) b->data;
336
    zone = (zone_t *) b->data;
349
    ASSERT(IS_BUDDY_ORDER_OK(frame_index_abs(zone, frame),
337
    ASSERT(IS_BUDDY_ORDER_OK(frame_index_abs(zone, frame),
350
        frame->buddy_order));
338
        frame->buddy_order));
351
   
339
   
352
    is_left = IS_BUDDY_LEFT_BLOCK_ABS(zone, frame);
340
    is_left = IS_BUDDY_LEFT_BLOCK_ABS(zone, frame);
353
    is_right = IS_BUDDY_RIGHT_BLOCK_ABS(zone, frame);
341
    is_right = IS_BUDDY_RIGHT_BLOCK_ABS(zone, frame);
354
 
342
 
355
    ASSERT(is_left ^ is_right);
343
    ASSERT(is_left ^ is_right);
356
    if (is_left) {
344
    if (is_left) {
357
        index = (frame_index(zone, frame)) +
345
        index = (frame_index(zone, frame)) +
358
            (1 << frame->buddy_order);
346
            (1 << frame->buddy_order);
359
    } else {    /* if (is_right) */
347
    } else {    /* if (is_right) */
360
        index = (frame_index(zone, frame)) -
348
        index = (frame_index(zone, frame)) -
361
            (1 << frame->buddy_order);
349
            (1 << frame->buddy_order);
362
    }
350
    }
363
   
351
   
364
    if (frame_index_valid(zone, index)) {
352
    if (frame_index_valid(zone, index)) {
365
        if (zone->frames[index].buddy_order == frame->buddy_order &&
353
        if (zone->frames[index].buddy_order == frame->buddy_order &&
366
            zone->frames[index].refcount == 0) {
354
            zone->frames[index].refcount == 0) {
367
            return &zone->frames[index].buddy_link;
355
            return &zone->frames[index].buddy_link;
368
        }
356
        }
369
    }
357
    }
370
 
358
 
371
    return NULL;   
359
    return NULL;   
372
}
360
}
373
 
361
 
374
/** Buddy system bisect implementation.
362
/** Buddy system bisect implementation.
375
 *
363
 *
376
 * @param b     Buddy system.
364
 * @param b     Buddy system.
377
 * @param block     Block to bisect.
365
 * @param block     Block to bisect.
378
 *
366
 *
379
 * @return      Right block.
367
 * @return      Right block.
380
 */
368
 */
381
static link_t *zone_buddy_bisect(buddy_system_t *b, link_t *block)
369
static link_t *zone_buddy_bisect(buddy_system_t *b, link_t *block)
382
{
370
{
383
    frame_t *frame_l, *frame_r;
371
    frame_t *frame_l, *frame_r;
384
 
372
 
385
    frame_l = list_get_instance(block, frame_t, buddy_link);
373
    frame_l = list_get_instance(block, frame_t, buddy_link);
386
    frame_r = (frame_l + (1 << (frame_l->buddy_order - 1)));
374
    frame_r = (frame_l + (1 << (frame_l->buddy_order - 1)));
387
   
375
   
388
    return &frame_r->buddy_link;
376
    return &frame_r->buddy_link;
389
}
377
}
390
 
378
 
391
/** Buddy system coalesce implementation.
379
/** Buddy system coalesce implementation.
392
 *
380
 *
393
 * @param b     Buddy system.
381
 * @param b     Buddy system.
394
 * @param block_1   First block.
382
 * @param block_1   First block.
395
 * @param block_2   First block's buddy.
383
 * @param block_2   First block's buddy.
396
 *
384
 *
397
 * @return      Coalesced block (actually block that represents lower
385
 * @return      Coalesced block (actually block that represents lower
398
 *          address).
386
 *          address).
399
 */
387
 */
400
static link_t *zone_buddy_coalesce(buddy_system_t *b, link_t *block_1,
388
static link_t *zone_buddy_coalesce(buddy_system_t *b, link_t *block_1,
401
    link_t *block_2)
389
    link_t *block_2)
402
{
390
{
403
    frame_t *frame1, *frame2;
391
    frame_t *frame1, *frame2;
404
   
392
   
405
    frame1 = list_get_instance(block_1, frame_t, buddy_link);
393
    frame1 = list_get_instance(block_1, frame_t, buddy_link);
406
    frame2 = list_get_instance(block_2, frame_t, buddy_link);
394
    frame2 = list_get_instance(block_2, frame_t, buddy_link);
407
   
395
   
408
    return frame1 < frame2 ? block_1 : block_2;
396
    return frame1 < frame2 ? block_1 : block_2;
409
}
397
}
410
 
398
 
411
/** Buddy system set_order implementation.
399
/** Buddy system set_order implementation.
412
 *
400
 *
413
 * @param b     Buddy system.
401
 * @param b     Buddy system.
414
 * @param block     Buddy system block.
402
 * @param block     Buddy system block.
415
 * @param order     Order to set.
403
 * @param order     Order to set.
416
 */
404
 */
417
static void zone_buddy_set_order(buddy_system_t *b, link_t *block,
405
static void zone_buddy_set_order(buddy_system_t *b, link_t *block,
418
    uint8_t order)
406
    uint8_t order)
419
{
407
{
420
    frame_t *frame;
408
    frame_t *frame;
421
    frame = list_get_instance(block, frame_t, buddy_link);
409
    frame = list_get_instance(block, frame_t, buddy_link);
422
    frame->buddy_order = order;
410
    frame->buddy_order = order;
423
}
411
}
424
 
412
 
425
/** Buddy system get_order implementation.
413
/** Buddy system get_order implementation.
426
 *
414
 *
427
 * @param b     Buddy system.
415
 * @param b     Buddy system.
428
 * @param block     Buddy system block.
416
 * @param block     Buddy system block.
429
 *
417
 *
430
 * @return      Order of block.
418
 * @return      Order of block.
431
 */
419
 */
432
static uint8_t zone_buddy_get_order(buddy_system_t *b, link_t *block)
420
static uint8_t zone_buddy_get_order(buddy_system_t *b, link_t *block)
433
{
421
{
434
    frame_t *frame;
422
    frame_t *frame;
435
    frame = list_get_instance(block, frame_t, buddy_link);
423
    frame = list_get_instance(block, frame_t, buddy_link);
436
    return frame->buddy_order;
424
    return frame->buddy_order;
437
}
425
}
438
 
426
 
439
/** Buddy system mark_busy implementation.
427
/** Buddy system mark_busy implementation.
440
 *
428
 *
441
 * @param b     Buddy system.
429
 * @param b     Buddy system.
442
 * @param block     Buddy system block.
430
 * @param block     Buddy system block.
443
 */
431
 */
444
static void zone_buddy_mark_busy(buddy_system_t *b, link_t * block)
432
static void zone_buddy_mark_busy(buddy_system_t *b, link_t * block)
445
{
433
{
446
    frame_t * frame;
434
    frame_t * frame;
447
 
435
 
448
    frame = list_get_instance(block, frame_t, buddy_link);
436
    frame = list_get_instance(block, frame_t, buddy_link);
449
    frame->refcount = 1;
437
    frame->refcount = 1;
450
}
438
}
451
 
439
 
452
/** Buddy system mark_available implementation.
440
/** Buddy system mark_available implementation.
453
 *
441
 *
454
 * @param b     Buddy system.
442
 * @param b     Buddy system.
455
 * @param block     Buddy system block.
443
 * @param block     Buddy system block.
456
 */
444
 */
457
static void zone_buddy_mark_available(buddy_system_t *b, link_t *block)
445
static void zone_buddy_mark_available(buddy_system_t *b, link_t *block)
458
{
446
{
459
    frame_t *frame;
447
    frame_t *frame;
460
    frame = list_get_instance(block, frame_t, buddy_link);
448
    frame = list_get_instance(block, frame_t, buddy_link);
461
    frame->refcount = 0;
449
    frame->refcount = 0;
462
}
450
}
463
 
451
 
464
static buddy_system_operations_t zone_buddy_system_operations = {
452
static buddy_system_operations_t zone_buddy_system_operations = {
465
    .find_buddy = zone_buddy_find_buddy,
453
    .find_buddy = zone_buddy_find_buddy,
466
    .bisect = zone_buddy_bisect,
454
    .bisect = zone_buddy_bisect,
467
    .coalesce = zone_buddy_coalesce,
455
    .coalesce = zone_buddy_coalesce,
468
    .set_order = zone_buddy_set_order,
456
    .set_order = zone_buddy_set_order,
469
    .get_order = zone_buddy_get_order,
457
    .get_order = zone_buddy_get_order,
470
    .mark_busy = zone_buddy_mark_busy,
458
    .mark_busy = zone_buddy_mark_busy,
471
    .mark_available = zone_buddy_mark_available,
459
    .mark_available = zone_buddy_mark_available,
472
    .find_block = zone_buddy_find_block,
460
    .find_block = zone_buddy_find_block
473
    .print_id = zone_buddy_print_id
-
 
474
};
461
};
475
 
462
 
476
/******************/
463
/******************/
477
/* Zone functions */
464
/* Zone functions */
478
/******************/
465
/******************/
479
 
466
 
480
/** Allocate frame in particular zone.
467
/** Allocate frame in particular zone.
481
 *
468
 *
482
 * Assume zone is locked.
469
 * Assume zone is locked.
483
 * Panics if allocation is impossible.
470
 * Panics if allocation is impossible.
484
 *
471
 *
485
 * @param zone      Zone to allocate from.
472
 * @param zone      Zone to allocate from.
486
 * @param order     Allocate exactly 2^order frames.
473
 * @param order     Allocate exactly 2^order frames.
487
 *
474
 *
488
 * @return      Frame index in zone.
475
 * @return      Frame index in zone.
489
 *
476
 *
490
 */
477
 */
491
static pfn_t zone_frame_alloc(zone_t *zone, uint8_t order)
478
static pfn_t zone_frame_alloc(zone_t *zone, uint8_t order)
492
{
479
{
493
    pfn_t v;
480
    pfn_t v;
494
    link_t *tmp;
481
    link_t *tmp;
495
    frame_t *frame;
482
    frame_t *frame;
496
 
483
 
497
    /* Allocate frames from zone buddy system */
484
    /* Allocate frames from zone buddy system */
498
    tmp = buddy_system_alloc(zone->buddy_system, order);
485
    tmp = buddy_system_alloc(zone->buddy_system, order);
499
   
486
   
500
    ASSERT(tmp);
487
    ASSERT(tmp);
501
   
488
   
502
    /* Update zone information. */
489
    /* Update zone information. */
503
    zone->free_count -= (1 << order);
490
    zone->free_count -= (1 << order);
504
    zone->busy_count += (1 << order);
491
    zone->busy_count += (1 << order);
505
 
492
 
506
    /* Frame will be actually a first frame of the block. */
493
    /* Frame will be actually a first frame of the block. */
507
    frame = list_get_instance(tmp, frame_t, buddy_link);
494
    frame = list_get_instance(tmp, frame_t, buddy_link);
508
   
495
   
509
    /* get frame address */
496
    /* get frame address */
510
    v = make_frame_index(zone, frame);
497
    v = make_frame_index(zone, frame);
511
    return v;
498
    return v;
512
}
499
}
513
 
500
 
514
/** Free frame from zone.
501
/** Free frame from zone.
515
 *
502
 *
516
 * Assume zone is locked.
503
 * Assume zone is locked.
517
 *
504
 *
518
 * @param zone      Pointer to zone from which the frame is to be freed.
505
 * @param zone      Pointer to zone from which the frame is to be freed.
519
 * @param frame_idx Frame index relative to zone.
506
 * @param frame_idx Frame index relative to zone.
520
 */
507
 */
521
static void zone_frame_free(zone_t *zone, index_t frame_idx)
508
static void zone_frame_free(zone_t *zone, index_t frame_idx)
522
{
509
{
523
    frame_t *frame;
510
    frame_t *frame;
524
    uint8_t order;
511
    uint8_t order;
525
 
512
 
526
    frame = &zone->frames[frame_idx];
513
    frame = &zone->frames[frame_idx];
527
   
514
   
528
    /* remember frame order */
515
    /* remember frame order */
529
    order = frame->buddy_order;
516
    order = frame->buddy_order;
530
 
517
 
531
    ASSERT(frame->refcount);
518
    ASSERT(frame->refcount);
532
 
519
 
533
    if (!--frame->refcount) {
520
    if (!--frame->refcount) {
534
        buddy_system_free(zone->buddy_system, &frame->buddy_link);
521
        buddy_system_free(zone->buddy_system, &frame->buddy_link);
535
   
522
   
536
        /* Update zone information. */
523
        /* Update zone information. */
537
        zone->free_count += (1 << order);
524
        zone->free_count += (1 << order);
538
        zone->busy_count -= (1 << order);
525
        zone->busy_count -= (1 << order);
539
    }
526
    }
540
}
527
}
541
 
528
 
542
/** Return frame from zone. */
529
/** Return frame from zone. */
543
static frame_t *zone_get_frame(zone_t *zone, index_t frame_idx)
530
static frame_t *zone_get_frame(zone_t *zone, index_t frame_idx)
544
{
531
{
545
    ASSERT(frame_idx < zone->count);
532
    ASSERT(frame_idx < zone->count);
546
    return &zone->frames[frame_idx];
533
    return &zone->frames[frame_idx];
547
}
534
}
548
 
535
 
549
/** Mark frame in zone unavailable to allocation. */
536
/** Mark frame in zone unavailable to allocation. */
550
static void zone_mark_unavailable(zone_t *zone, index_t frame_idx)
537
static void zone_mark_unavailable(zone_t *zone, index_t frame_idx)
551
{
538
{
552
    frame_t *frame;
539
    frame_t *frame;
553
    link_t *link;
540
    link_t *link;
554
 
541
 
555
    frame = zone_get_frame(zone, frame_idx);
542
    frame = zone_get_frame(zone, frame_idx);
556
    if (frame->refcount)
543
    if (frame->refcount)
557
        return;
544
        return;
558
    link = buddy_system_alloc_block(zone->buddy_system,
545
    link = buddy_system_alloc_block(zone->buddy_system,
559
        &frame->buddy_link);
546
        &frame->buddy_link);
560
    ASSERT(link);
547
    ASSERT(link);
561
    zone->free_count--;
548
    zone->free_count--;
562
 
549
 
563
    mutex_lock(&mem_avail_mtx);
550
    mutex_lock(&mem_avail_mtx);
564
    mem_avail_frames--;
551
    mem_avail_frames--;
565
    mutex_unlock(&mem_avail_mtx);
552
    mutex_unlock(&mem_avail_mtx);
566
}
553
}
567
 
554
 
568
/** Join two zones.
555
/** Join two zones.
569
 *
556
 *
570
 * Expect zone_t *z to point to space at least zone_conf_size large.
557
 * Expect zone_t *z to point to space at least zone_conf_size large.
571
 *
558
 *
572
 * Assume z1 & z2 are locked.
559
 * Assume z1 & z2 are locked.
573
 *
560
 *
574
 * @param z     Target zone structure pointer.
561
 * @param z     Target zone structure pointer.
575
 * @param z1        Zone to merge.
562
 * @param z1        Zone to merge.
576
 * @param z2        Zone to merge.
563
 * @param z2        Zone to merge.
577
 */
564
 */
578
static void _zone_merge(zone_t *z, zone_t *z1, zone_t *z2)
565
static void _zone_merge(zone_t *z, zone_t *z1, zone_t *z2)
579
{
566
{
580
    uint8_t max_order;
567
    uint8_t max_order;
581
    unsigned int i;
568
    unsigned int i;
582
    int z2idx;
569
    int z2idx;
583
    pfn_t frame_idx;
570
    pfn_t frame_idx;
584
    frame_t *frame;
571
    frame_t *frame;
585
 
572
 
586
    ASSERT(!overlaps(z1->base, z1->count, z2->base, z2->count));
573
    ASSERT(!overlaps(z1->base, z1->count, z2->base, z2->count));
587
    ASSERT(z1->base < z2->base);
574
    ASSERT(z1->base < z2->base);
588
 
575
 
589
    spinlock_initialize(&z->lock, "zone_lock");
576
    spinlock_initialize(&z->lock, "zone_lock");
590
    z->base = z1->base;
577
    z->base = z1->base;
591
    z->count = z2->base + z2->count - z1->base;
578
    z->count = z2->base + z2->count - z1->base;
592
    z->flags = z1->flags & z2->flags;
579
    z->flags = z1->flags & z2->flags;
593
 
580
 
594
    z->free_count = z1->free_count + z2->free_count;
581
    z->free_count = z1->free_count + z2->free_count;
595
    z->busy_count = z1->busy_count + z2->busy_count;
582
    z->busy_count = z1->busy_count + z2->busy_count;
596
   
583
   
597
    max_order = fnzb(z->count);
584
    max_order = fnzb(z->count);
598
 
585
 
599
    z->buddy_system = (buddy_system_t *) &z[1];
586
    z->buddy_system = (buddy_system_t *) &z[1];
600
    buddy_system_create(z->buddy_system, max_order,
587
    buddy_system_create(z->buddy_system, max_order,
601
        &zone_buddy_system_operations, (void *) z);
588
        &zone_buddy_system_operations, (void *) z);
602
 
589
 
603
    z->frames = (frame_t *)((uint8_t *) z->buddy_system +
590
    z->frames = (frame_t *)((uint8_t *) z->buddy_system +
604
        buddy_conf_size(max_order));
591
        buddy_conf_size(max_order));
605
    for (i = 0; i < z->count; i++) {
592
    for (i = 0; i < z->count; i++) {
606
        /* This marks all frames busy */
593
        /* This marks all frames busy */
607
        frame_initialize(&z->frames[i]);
594
        frame_initialize(&z->frames[i]);
608
    }
595
    }
609
    /* Copy frames from both zones to preserve full frame orders,
596
    /* Copy frames from both zones to preserve full frame orders,
610
     * parents etc. Set all free frames with refcount=0 to 1, because
597
     * parents etc. Set all free frames with refcount=0 to 1, because
611
     * we add all free frames to buddy allocator later again, clear
598
     * we add all free frames to buddy allocator later again, clear
612
     * order to 0. Don't set busy frames with refcount=0, as they
599
     * order to 0. Don't set busy frames with refcount=0, as they
613
     * will not be reallocated during merge and it would make later
600
     * will not be reallocated during merge and it would make later
614
     * problems with allocation/free.
601
     * problems with allocation/free.
615
     */
602
     */
616
    for (i = 0; i < z1->count; i++)
603
    for (i = 0; i < z1->count; i++)
617
        z->frames[i] = z1->frames[i];
604
        z->frames[i] = z1->frames[i];
618
    for (i = 0; i < z2->count; i++) {
605
    for (i = 0; i < z2->count; i++) {
619
        z2idx = i + (z2->base - z1->base);
606
        z2idx = i + (z2->base - z1->base);
620
        z->frames[z2idx] = z2->frames[i];
607
        z->frames[z2idx] = z2->frames[i];
621
    }
608
    }
622
    i = 0;
609
    i = 0;
623
    while (i < z->count) {
610
    while (i < z->count) {
624
        if (z->frames[i].refcount) {
611
        if (z->frames[i].refcount) {
625
            /* skip busy frames */
612
            /* skip busy frames */
626
            i += 1 << z->frames[i].buddy_order;
613
            i += 1 << z->frames[i].buddy_order;
627
        } else { /* Free frames, set refcount=1 */
614
        } else { /* Free frames, set refcount=1 */
628
            /* All free frames have refcount=0, we need not
615
            /* All free frames have refcount=0, we need not
629
             * to check the order */
616
             * to check the order */
630
            z->frames[i].refcount = 1;
617
            z->frames[i].refcount = 1;
631
            z->frames[i].buddy_order = 0;
618
            z->frames[i].buddy_order = 0;
632
            i++;
619
            i++;
633
        }
620
        }
634
    }
621
    }
635
    /* Add free blocks from the 2 original zones */
622
    /* Add free blocks from the 2 original zones */
636
    while (zone_can_alloc(z1, 0)) {
623
    while (zone_can_alloc(z1, 0)) {
637
        frame_idx = zone_frame_alloc(z1, 0);
624
        frame_idx = zone_frame_alloc(z1, 0);
638
        frame = &z->frames[frame_idx];
625
        frame = &z->frames[frame_idx];
639
        frame->refcount = 0;
626
        frame->refcount = 0;
640
        buddy_system_free(z->buddy_system, &frame->buddy_link);
627
        buddy_system_free(z->buddy_system, &frame->buddy_link);
641
    }
628
    }
642
    while (zone_can_alloc(z2, 0)) {
629
    while (zone_can_alloc(z2, 0)) {
643
        frame_idx = zone_frame_alloc(z2, 0);
630
        frame_idx = zone_frame_alloc(z2, 0);
644
        frame = &z->frames[frame_idx + (z2->base - z1->base)];
631
        frame = &z->frames[frame_idx + (z2->base - z1->base)];
645
        frame->refcount = 0;
632
        frame->refcount = 0;
646
        buddy_system_free(z->buddy_system, &frame->buddy_link);
633
        buddy_system_free(z->buddy_system, &frame->buddy_link);
647
    }
634
    }
648
}
635
}
649
 
636
 
650
/** Return old configuration frames into the zone.
637
/** Return old configuration frames into the zone.
651
 *
638
 *
652
 * We have several cases
639
 * We have several cases
653
 * - the conf. data is outside of zone -> exit, shall we call frame_free??
640
 * - the conf. data is outside of zone -> exit, shall we call frame_free??
654
 * - the conf. data was created by zone_create or
641
 * - the conf. data was created by zone_create or
655
 *   updated with reduce_region -> free every frame
642
 *   updated with reduce_region -> free every frame
656
 *
643
 *
657
 * @param newzone   The actual zone where freeing should occur.
644
 * @param newzone   The actual zone where freeing should occur.
658
 * @param oldzone   Pointer to old zone configuration data that should
645
 * @param oldzone   Pointer to old zone configuration data that should
659
 *          be freed from new zone.
646
 *          be freed from new zone.
660
 */
647
 */
661
static void return_config_frames(zone_t *newzone, zone_t *oldzone)
648
static void return_config_frames(zone_t *newzone, zone_t *oldzone)
662
{
649
{
663
    pfn_t pfn;
650
    pfn_t pfn;
664
    frame_t *frame;
651
    frame_t *frame;
665
    count_t cframes;
652
    count_t cframes;
666
    unsigned int i;
653
    unsigned int i;
667
 
654
 
668
    pfn = ADDR2PFN((uintptr_t)KA2PA(oldzone));
655
    pfn = ADDR2PFN((uintptr_t)KA2PA(oldzone));
669
    cframes = SIZE2FRAMES(zone_conf_size(oldzone->count));
656
    cframes = SIZE2FRAMES(zone_conf_size(oldzone->count));
670
   
657
   
671
    if (pfn < newzone->base || pfn >= newzone->base + newzone->count)
658
    if (pfn < newzone->base || pfn >= newzone->base + newzone->count)
672
        return;
659
        return;
673
 
660
 
674
    frame = &newzone->frames[pfn - newzone->base];
661
    frame = &newzone->frames[pfn - newzone->base];
675
    ASSERT(!frame->buddy_order);
662
    ASSERT(!frame->buddy_order);
676
 
663
 
677
    for (i = 0; i < cframes; i++) {
664
    for (i = 0; i < cframes; i++) {
678
        newzone->busy_count++;
665
        newzone->busy_count++;
679
        zone_frame_free(newzone, pfn+i-newzone->base);
666
        zone_frame_free(newzone, pfn+i-newzone->base);
680
    }
667
    }
681
}
668
}
682
 
669
 
683
/** Reduce allocated block to count of order 0 frames.
670
/** Reduce allocated block to count of order 0 frames.
684
 *
671
 *
685
 * The allocated block need 2^order frames of space. Reduce all frames
672
 * The allocated block need 2^order frames of space. Reduce all frames
686
 * in block to order 0 and free the unneeded frames. This means, that
673
 * in block to order 0 and free the unneeded frames. This means, that
687
 * when freeing the previously allocated block starting with frame_idx,
674
 * when freeing the previously allocated block starting with frame_idx,
688
 * you have to free every frame.
675
 * you have to free every frame.
689
 *
676
 *
690
 * @param zone
677
 * @param zone
691
 * @param frame_idx     Index to block.
678
 * @param frame_idx     Index to block.
692
 * @param count         Allocated space in block.
679
 * @param count         Allocated space in block.
693
 */
680
 */
694
static void zone_reduce_region(zone_t *zone, pfn_t frame_idx, count_t count)
681
static void zone_reduce_region(zone_t *zone, pfn_t frame_idx, count_t count)
695
{
682
{
696
    count_t i;
683
    count_t i;
697
    uint8_t order;
684
    uint8_t order;
698
    frame_t *frame;
685
    frame_t *frame;
699
   
686
   
700
    ASSERT(frame_idx + count < zone->count);
687
    ASSERT(frame_idx + count < zone->count);
701
 
688
 
702
    order = zone->frames[frame_idx].buddy_order;
689
    order = zone->frames[frame_idx].buddy_order;
703
    ASSERT((count_t) (1 << order) >= count);
690
    ASSERT((count_t) (1 << order) >= count);
704
 
691
 
705
    /* Reduce all blocks to order 0 */
692
    /* Reduce all blocks to order 0 */
706
    for (i = 0; i < (count_t) (1 << order); i++) {
693
    for (i = 0; i < (count_t) (1 << order); i++) {
707
        frame = &zone->frames[i + frame_idx];
694
        frame = &zone->frames[i + frame_idx];
708
        frame->buddy_order = 0;
695
        frame->buddy_order = 0;
709
        if (!frame->refcount)
696
        if (!frame->refcount)
710
            frame->refcount = 1;
697
            frame->refcount = 1;
711
        ASSERT(frame->refcount == 1);
698
        ASSERT(frame->refcount == 1);
712
    }
699
    }
713
    /* Free unneeded frames */
700
    /* Free unneeded frames */
714
    for (i = count; i < (count_t) (1 << order); i++) {
701
    for (i = count; i < (count_t) (1 << order); i++) {
715
        zone_frame_free(zone, i + frame_idx);
702
        zone_frame_free(zone, i + frame_idx);
716
    }
703
    }
717
}
704
}
718
 
705
 
719
/** Merge zones z1 and z2.
706
/** Merge zones z1 and z2.
720
 *
707
 *
721
 * - the zones must be 2 zones with no zone existing in between,
708
 * - the zones must be 2 zones with no zone existing in between,
722
 *   which means that z2 = z1+1
709
 *   which means that z2 = z1+1
723
 *
710
 *
724
 * - When you create a new zone, the frame allocator configuration does
711
 * - When you create a new zone, the frame allocator configuration does
725
 *   not to be 2^order size. Once the allocator is running it is no longer
712
 *   not to be 2^order size. Once the allocator is running it is no longer
726
 *   possible, merged configuration data occupies more space :-/
713
 *   possible, merged configuration data occupies more space :-/
727
 */
714
 */
728
void zone_merge(unsigned int z1, unsigned int z2)
715
void zone_merge(unsigned int z1, unsigned int z2)
729
{
716
{
730
    ipl_t ipl;
717
    ipl_t ipl;
731
    zone_t *zone1, *zone2, *newzone;
718
    zone_t *zone1, *zone2, *newzone;
732
    unsigned int cframes;
719
    unsigned int cframes;
733
    uint8_t order;
720
    uint8_t order;
734
    unsigned int i;
721
    unsigned int i;
735
    pfn_t pfn;
722
    pfn_t pfn;
736
 
723
 
737
    ipl = interrupts_disable();
724
    ipl = interrupts_disable();
738
    spinlock_lock(&zones.lock);
725
    spinlock_lock(&zones.lock);
739
 
726
 
740
    if ((z1 >= zones.count) || (z2 >= zones.count))
727
    if ((z1 >= zones.count) || (z2 >= zones.count))
741
        goto errout;
728
        goto errout;
742
    /* We can join only 2 zones with none existing inbetween */
729
    /* We can join only 2 zones with none existing inbetween */
743
    if (z2 - z1 != 1)
730
    if (z2 - z1 != 1)
744
        goto errout;
731
        goto errout;
745
 
732
 
746
    zone1 = zones.info[z1];
733
    zone1 = zones.info[z1];
747
    zone2 = zones.info[z2];
734
    zone2 = zones.info[z2];
748
    spinlock_lock(&zone1->lock);
735
    spinlock_lock(&zone1->lock);
749
    spinlock_lock(&zone2->lock);
736
    spinlock_lock(&zone2->lock);
750
 
737
 
751
    cframes = SIZE2FRAMES(zone_conf_size(zone2->base + zone2->count -
738
    cframes = SIZE2FRAMES(zone_conf_size(zone2->base + zone2->count -
752
        zone1->base));
739
        zone1->base));
753
    if (cframes == 1)
740
    if (cframes == 1)
754
        order = 0;
741
        order = 0;
755
    else
742
    else
756
        order = fnzb(cframes - 1) + 1;
743
        order = fnzb(cframes - 1) + 1;
757
 
744
 
758
    /* Allocate zonedata inside one of the zones */
745
    /* Allocate zonedata inside one of the zones */
759
    if (zone_can_alloc(zone1, order))
746
    if (zone_can_alloc(zone1, order))
760
        pfn = zone1->base + zone_frame_alloc(zone1, order);
747
        pfn = zone1->base + zone_frame_alloc(zone1, order);
761
    else if (zone_can_alloc(zone2, order))
748
    else if (zone_can_alloc(zone2, order))
762
        pfn = zone2->base + zone_frame_alloc(zone2, order);
749
        pfn = zone2->base + zone_frame_alloc(zone2, order);
763
    else
750
    else
764
        goto errout2;
751
        goto errout2;
765
 
752
 
766
    newzone = (zone_t *) PA2KA(PFN2ADDR(pfn));
753
    newzone = (zone_t *) PA2KA(PFN2ADDR(pfn));
767
 
754
 
768
    _zone_merge(newzone, zone1, zone2);
755
    _zone_merge(newzone, zone1, zone2);
769
 
756
 
770
    /* Free unneeded config frames */
757
    /* Free unneeded config frames */
771
    zone_reduce_region(newzone, pfn - newzone->base,  cframes);
758
    zone_reduce_region(newzone, pfn - newzone->base,  cframes);
772
    /* Subtract zone information from busy frames */
759
    /* Subtract zone information from busy frames */
773
    newzone->busy_count -= cframes;
760
    newzone->busy_count -= cframes;
774
 
761
 
775
    /* Replace existing zones in zoneinfo list */
762
    /* Replace existing zones in zoneinfo list */
776
    zones.info[z1] = newzone;
763
    zones.info[z1] = newzone;
777
    for (i = z2 + 1; i < zones.count; i++)
764
    for (i = z2 + 1; i < zones.count; i++)
778
        zones.info[i - 1] = zones.info[i];
765
        zones.info[i - 1] = zones.info[i];
779
    zones.count--;
766
    zones.count--;
780
 
767
 
781
    /* Free old zone information */
768
    /* Free old zone information */
782
    return_config_frames(newzone, zone1);
769
    return_config_frames(newzone, zone1);
783
    return_config_frames(newzone, zone2);
770
    return_config_frames(newzone, zone2);
784
errout2:
771
errout2:
785
    /* Nobody is allowed to enter to zone, so we are safe
772
    /* Nobody is allowed to enter to zone, so we are safe
786
     * to touch the spinlocks last time */
773
     * to touch the spinlocks last time */
787
    spinlock_unlock(&zone1->lock);
774
    spinlock_unlock(&zone1->lock);
788
    spinlock_unlock(&zone2->lock);
775
    spinlock_unlock(&zone2->lock);
789
errout:
776
errout:
790
    spinlock_unlock(&zones.lock);
777
    spinlock_unlock(&zones.lock);
791
    interrupts_restore(ipl);
778
    interrupts_restore(ipl);
792
}
779
}
793
 
780
 
794
/** Merge all zones into one big zone.
781
/** Merge all zones into one big zone.
795
 *
782
 *
796
 * It is reasonable to do this on systems whose bios reports parts in chunks,
783
 * It is reasonable to do this on systems whose bios reports parts in chunks,
797
 * so that we could have 1 zone (it's faster).
784
 * so that we could have 1 zone (it's faster).
798
 */
785
 */
799
void zone_merge_all(void)
786
void zone_merge_all(void)
800
{
787
{
801
    int count = zones.count;
788
    int count = zones.count;
802
 
789
 
803
    while (zones.count > 1 && --count) {
790
    while (zones.count > 1 && --count) {
804
        zone_merge(0, 1);
791
        zone_merge(0, 1);
805
        break;
792
        break;
806
    }
793
    }
807
}
794
}
808
 
795
 
809
/** Create new frame zone.
796
/** Create new frame zone.
810
 *
797
 *
811
 * @param start     Physical address of the first frame within the zone.
798
 * @param start     Physical address of the first frame within the zone.
812
 * @param count     Count of frames in zone.
799
 * @param count     Count of frames in zone.
813
 * @param z     Address of configuration information of zone.
800
 * @param z     Address of configuration information of zone.
814
 * @param flags     Zone flags.
801
 * @param flags     Zone flags.
815
 *
802
 *
816
 * @return      Initialized zone.
803
 * @return      Initialized zone.
817
 */
804
 */
818
static void zone_construct(pfn_t start, count_t count, zone_t *z, int flags)
805
static void zone_construct(pfn_t start, count_t count, zone_t *z, int flags)
819
{
806
{
820
    unsigned int i;
807
    unsigned int i;
821
    uint8_t max_order;
808
    uint8_t max_order;
822
 
809
 
823
    spinlock_initialize(&z->lock, "zone_lock");
810
    spinlock_initialize(&z->lock, "zone_lock");
824
    z->base = start;
811
    z->base = start;
825
    z->count = count;
812
    z->count = count;
826
    z->flags = flags;
813
    z->flags = flags;
827
    z->free_count = count;
814
    z->free_count = count;
828
    z->busy_count = 0;
815
    z->busy_count = 0;
829
 
816
 
830
    /*
817
    /*
831
     * Compute order for buddy system, initialize
818
     * Compute order for buddy system, initialize
832
     */
819
     */
833
    max_order = fnzb(count);
820
    max_order = fnzb(count);
834
    z->buddy_system = (buddy_system_t *)&z[1];
821
    z->buddy_system = (buddy_system_t *)&z[1];
835
   
822
   
836
    buddy_system_create(z->buddy_system, max_order,
823
    buddy_system_create(z->buddy_system, max_order,
837
        &zone_buddy_system_operations, (void *) z);
824
        &zone_buddy_system_operations, (void *) z);
838
   
825
   
839
    /* Allocate frames _after_ the conframe */
826
    /* Allocate frames _after_ the conframe */
840
    /* Check sizes */
827
    /* Check sizes */
841
    z->frames = (frame_t *)((uint8_t *) z->buddy_system +
828
    z->frames = (frame_t *)((uint8_t *) z->buddy_system +
842
        buddy_conf_size(max_order));
829
        buddy_conf_size(max_order));
843
    for (i = 0; i < count; i++) {
830
    for (i = 0; i < count; i++) {
844
        frame_initialize(&z->frames[i]);
831
        frame_initialize(&z->frames[i]);
845
    }
832
    }
846
   
833
   
847
    /* Stuffing frames */
834
    /* Stuffing frames */
848
    for (i = 0; i < count; i++) {
835
    for (i = 0; i < count; i++) {
849
        z->frames[i].refcount = 0;
836
        z->frames[i].refcount = 0;
850
        buddy_system_free(z->buddy_system, &z->frames[i].buddy_link);
837
        buddy_system_free(z->buddy_system, &z->frames[i].buddy_link);
851
    }
838
    }
852
}
839
}
853
 
840
 
854
/** Compute configuration data size for zone.
841
/** Compute configuration data size for zone.
855
 *
842
 *
856
 * @param count     Size of zone in frames.
843
 * @param count     Size of zone in frames.
857
 * @return      Size of zone configuration info (in bytes).
844
 * @return      Size of zone configuration info (in bytes).
858
 */
845
 */
859
uintptr_t zone_conf_size(count_t count)
846
uintptr_t zone_conf_size(count_t count)
860
{
847
{
861
    int size = sizeof(zone_t) + count * sizeof(frame_t);
848
    int size = sizeof(zone_t) + count * sizeof(frame_t);
862
    int max_order;
849
    int max_order;
863
 
850
 
864
    max_order = fnzb(count);
851
    max_order = fnzb(count);
865
    size += buddy_conf_size(max_order);
852
    size += buddy_conf_size(max_order);
866
    return size;
853
    return size;
867
}
854
}
868
 
855
 
869
/** Create and add zone to system.
856
/** Create and add zone to system.
870
 *
857
 *
871
 * @param start     First frame number (absolute).
858
 * @param start     First frame number (absolute).
872
 * @param count     Size of zone in frames.
859
 * @param count     Size of zone in frames.
873
 * @param confframe Where configuration frames are supposed to be.
860
 * @param confframe Where configuration frames are supposed to be.
874
 *          Automatically checks, that we will not disturb the
861
 *          Automatically checks, that we will not disturb the
875
 *          kernel and possibly init.  If confframe is given
862
 *          kernel and possibly init.  If confframe is given
876
 *          _outside_ this zone, it is expected, that the area is
863
 *          _outside_ this zone, it is expected, that the area is
877
 *          already marked BUSY and big enough to contain
864
 *          already marked BUSY and big enough to contain
878
 *          zone_conf_size() amount of data.  If the confframe is
865
 *          zone_conf_size() amount of data.  If the confframe is
879
 *          inside the area, the zone free frame information is
866
 *          inside the area, the zone free frame information is
880
 *          modified not to include it.
867
 *          modified not to include it.
881
 *
868
 *
882
 * @return      Zone number or -1 on error.
869
 * @return      Zone number or -1 on error.
883
 */
870
 */
884
int zone_create(pfn_t start, count_t count, pfn_t confframe, int flags)
871
int zone_create(pfn_t start, count_t count, pfn_t confframe, int flags)
885
{
872
{
886
    zone_t *z;
873
    zone_t *z;
887
    uintptr_t addr;
874
    uintptr_t addr;
888
    count_t confcount;
875
    count_t confcount;
889
    unsigned int i;
876
    unsigned int i;
890
    int znum;
877
    int znum;
891
 
878
 
892
    /* Theoretically we could have here 0, practically make sure
879
    /* Theoretically we could have here 0, practically make sure
893
     * nobody tries to do that. If some platform requires, remove
880
     * nobody tries to do that. If some platform requires, remove
894
     * the assert
881
     * the assert
895
     */
882
     */
896
    ASSERT(confframe);
883
    ASSERT(confframe);
897
    /* If conframe is supposed to be inside our zone, then make sure
884
    /* If conframe is supposed to be inside our zone, then make sure
898
     * it does not span kernel & init
885
     * it does not span kernel & init
899
     */
886
     */
900
    confcount = SIZE2FRAMES(zone_conf_size(count));
887
    confcount = SIZE2FRAMES(zone_conf_size(count));
901
    if (confframe >= start && confframe < start + count) {
888
    if (confframe >= start && confframe < start + count) {
902
        for (; confframe < start + count; confframe++) {
889
        for (; confframe < start + count; confframe++) {
903
            addr = PFN2ADDR(confframe);
890
            addr = PFN2ADDR(confframe);
904
            if (overlaps(addr, PFN2ADDR(confcount),
891
            if (overlaps(addr, PFN2ADDR(confcount),
905
                KA2PA(config.base), config.kernel_size))
892
                KA2PA(config.base), config.kernel_size))
906
                continue;
893
                continue;
907
           
894
           
908
            if (overlaps(addr, PFN2ADDR(confcount),
895
            if (overlaps(addr, PFN2ADDR(confcount),
909
                KA2PA(config.stack_base), config.stack_size))
896
                KA2PA(config.stack_base), config.stack_size))
910
                continue;
897
                continue;
911
           
898
           
912
            bool overlap = false;
899
            bool overlap = false;
913
            count_t i;
900
            count_t i;
914
            for (i = 0; i < init.cnt; i++)
901
            for (i = 0; i < init.cnt; i++)
915
                if (overlaps(addr, PFN2ADDR(confcount),
902
                if (overlaps(addr, PFN2ADDR(confcount),
916
                    KA2PA(init.tasks[i].addr),
903
                    KA2PA(init.tasks[i].addr),
917
                    init.tasks[i].size)) {
904
                    init.tasks[i].size)) {
918
                    overlap = true;
905
                    overlap = true;
919
                    break;
906
                    break;
920
                }
907
                }
921
            if (overlap)
908
            if (overlap)
922
                continue;
909
                continue;
923
           
910
           
924
            break;
911
            break;
925
        }
912
        }
926
        if (confframe >= start + count)
913
        if (confframe >= start + count)
927
            panic("Cannot find configuration data for zone.");
914
            panic("Cannot find configuration data for zone.");
928
    }
915
    }
929
 
916
 
930
    z = (zone_t *) PA2KA(PFN2ADDR(confframe));
917
    z = (zone_t *) PA2KA(PFN2ADDR(confframe));
931
    zone_construct(start, count, z, flags);
918
    zone_construct(start, count, z, flags);
932
    znum = zones_add_zone(z);
919
    znum = zones_add_zone(z);
933
    if (znum == -1)
920
    if (znum == -1)
934
        return -1;
921
        return -1;
935
 
922
 
936
    mutex_lock(&mem_avail_mtx);
923
    mutex_lock(&mem_avail_mtx);
937
    mem_avail_frames += count;
924
    mem_avail_frames += count;
938
    mutex_unlock(&mem_avail_mtx);
925
    mutex_unlock(&mem_avail_mtx);
939
 
926
 
940
    /* If confdata in zone, mark as unavailable */
927
    /* If confdata in zone, mark as unavailable */
941
    if (confframe >= start && confframe < start + count)
928
    if (confframe >= start && confframe < start + count)
942
        for (i = confframe; i < confframe + confcount; i++) {
929
        for (i = confframe; i < confframe + confcount; i++) {
943
            zone_mark_unavailable(z, i - z->base);
930
            zone_mark_unavailable(z, i - z->base);
944
        }
931
        }
945
   
932
   
946
    return znum;
933
    return znum;
947
}
934
}
948
 
935
 
949
/***************************************/
936
/***************************************/
950
/* Frame functions */
937
/* Frame functions */
951
 
938
 
952
/** Set parent of frame. */
939
/** Set parent of frame. */
953
void frame_set_parent(pfn_t pfn, void *data, unsigned int hint)
940
void frame_set_parent(pfn_t pfn, void *data, unsigned int hint)
954
{
941
{
955
    zone_t *zone = find_zone_and_lock(pfn, &hint);
942
    zone_t *zone = find_zone_and_lock(pfn, &hint);
956
 
943
 
957
    ASSERT(zone);
944
    ASSERT(zone);
958
 
945
 
959
    zone_get_frame(zone, pfn - zone->base)->parent = data;
946
    zone_get_frame(zone, pfn - zone->base)->parent = data;
960
    spinlock_unlock(&zone->lock);
947
    spinlock_unlock(&zone->lock);
961
}
948
}
962
 
949
 
963
void *frame_get_parent(pfn_t pfn, unsigned int hint)
950
void *frame_get_parent(pfn_t pfn, unsigned int hint)
964
{
951
{
965
    zone_t *zone = find_zone_and_lock(pfn, &hint);
952
    zone_t *zone = find_zone_and_lock(pfn, &hint);
966
    void *res;
953
    void *res;
967
 
954
 
968
    ASSERT(zone);
955
    ASSERT(zone);
969
    res = zone_get_frame(zone, pfn - zone->base)->parent;
956
    res = zone_get_frame(zone, pfn - zone->base)->parent;
970
   
957
   
971
    spinlock_unlock(&zone->lock);
958
    spinlock_unlock(&zone->lock);
972
    return res;
959
    return res;
973
}
960
}
974
 
961
 
975
/** Allocate power-of-two frames of physical memory.
962
/** Allocate power-of-two frames of physical memory.
976
 *
963
 *
977
 * @param order     Allocate exactly 2^order frames.
964
 * @param order     Allocate exactly 2^order frames.
978
 * @param flags     Flags for host zone selection and address processing.
965
 * @param flags     Flags for host zone selection and address processing.
979
 * @param pzone     Preferred zone.
966
 * @param pzone     Preferred zone.
980
 *
967
 *
981
 * @return      Physical address of the allocated frame.
968
 * @return      Physical address of the allocated frame.
982
 *
969
 *
983
 */
970
 */
984
void *frame_alloc_generic(uint8_t order, int flags, unsigned int *pzone)
971
void *frame_alloc_generic(uint8_t order, int flags, unsigned int *pzone)
985
{
972
{
986
    ipl_t ipl;
973
    ipl_t ipl;
987
    int freed;
974
    int freed;
988
    pfn_t v;
975
    pfn_t v;
989
    zone_t *zone;
976
    zone_t *zone;
990
    unsigned long gen = 0;
977
    unsigned long gen = 0;
991
   
978
   
992
loop:
979
loop:
993
    ipl = interrupts_disable();
980
    ipl = interrupts_disable();
994
   
981
   
995
    /*
982
    /*
996
     * First, find suitable frame zone.
983
     * First, find suitable frame zone.
997
     */
984
     */
998
    zone = find_free_zone_and_lock(order, pzone);
985
    zone = find_free_zone_and_lock(order, pzone);
999
   
986
   
1000
    /* If no memory, reclaim some slab memory,
987
    /* If no memory, reclaim some slab memory,
1001
       if it does not help, reclaim all */
988
       if it does not help, reclaim all */
1002
    if (!zone && !(flags & FRAME_NO_RECLAIM)) {
989
    if (!zone && !(flags & FRAME_NO_RECLAIM)) {
1003
        freed = slab_reclaim(0);
990
        freed = slab_reclaim(0);
1004
        if (freed)
991
        if (freed)
1005
            zone = find_free_zone_and_lock(order, pzone);
992
            zone = find_free_zone_and_lock(order, pzone);
1006
        if (!zone) {
993
        if (!zone) {
1007
            freed = slab_reclaim(SLAB_RECLAIM_ALL);
994
            freed = slab_reclaim(SLAB_RECLAIM_ALL);
1008
            if (freed)
995
            if (freed)
1009
                zone = find_free_zone_and_lock(order, pzone);
996
                zone = find_free_zone_and_lock(order, pzone);
1010
        }
997
        }
1011
    }
998
    }
1012
    if (!zone) {
999
    if (!zone) {
1013
        /*
1000
        /*
1014
         * Sleep until some frames are available again.
1001
         * Sleep until some frames are available again.
1015
         */
1002
         */
1016
        if (flags & FRAME_ATOMIC) {
1003
        if (flags & FRAME_ATOMIC) {
1017
            interrupts_restore(ipl);
1004
            interrupts_restore(ipl);
1018
            return 0;
1005
            return 0;
1019
        }
1006
        }
1020
       
1007
       
1021
#ifdef CONFIG_DEBUG
1008
#ifdef CONFIG_DEBUG
1022
        unsigned long avail;
1009
        unsigned long avail;
1023
 
1010
 
1024
        mutex_lock(&mem_avail_mtx);
1011
        mutex_lock(&mem_avail_mtx);
1025
        avail = mem_avail_frames;
1012
        avail = mem_avail_frames;
1026
        mutex_unlock(&mem_avail_mtx);
1013
        mutex_unlock(&mem_avail_mtx);
1027
 
1014
 
1028
        printf("Thread %" PRIu64 " waiting for %u frames, "
1015
        printf("Thread %" PRIu64 " waiting for %u frames, "
1029
            "%u available.\n", THREAD->tid, 1ULL << order, avail);
1016
            "%u available.\n", THREAD->tid, 1ULL << order, avail);
1030
#endif
1017
#endif
1031
 
1018
 
1032
        mutex_lock(&mem_avail_mtx);
1019
        mutex_lock(&mem_avail_mtx);
1033
        while ((mem_avail_frames < (1ULL << order)) ||
1020
        while ((mem_avail_frames < (1ULL << order)) ||
1034
            gen == mem_avail_gen)
1021
            gen == mem_avail_gen)
1035
            condvar_wait(&mem_avail_cv, &mem_avail_mtx);
1022
            condvar_wait(&mem_avail_cv, &mem_avail_mtx);
1036
        gen = mem_avail_gen;
1023
        gen = mem_avail_gen;
1037
        mutex_unlock(&mem_avail_mtx);
1024
        mutex_unlock(&mem_avail_mtx);
1038
 
1025
 
1039
#ifdef CONFIG_DEBUG
1026
#ifdef CONFIG_DEBUG
1040
        mutex_lock(&mem_avail_mtx);
1027
        mutex_lock(&mem_avail_mtx);
1041
        avail = mem_avail_frames;
1028
        avail = mem_avail_frames;
1042
        mutex_unlock(&mem_avail_mtx);
1029
        mutex_unlock(&mem_avail_mtx);
1043
 
1030
 
1044
        printf("Thread %" PRIu64 " woken up, %u frames available.\n",
1031
        printf("Thread %" PRIu64 " woken up, %u frames available.\n",
1045
            THREAD->tid, avail);
1032
            THREAD->tid, avail);
1046
#endif
1033
#endif
1047
 
1034
 
1048
        interrupts_restore(ipl);
1035
        interrupts_restore(ipl);
1049
        goto loop;
1036
        goto loop;
1050
    }
1037
    }
1051
   
1038
   
1052
    v = zone_frame_alloc(zone, order);
1039
    v = zone_frame_alloc(zone, order);
1053
    v += zone->base;
1040
    v += zone->base;
1054
 
1041
 
1055
    spinlock_unlock(&zone->lock);
1042
    spinlock_unlock(&zone->lock);
1056
   
1043
   
1057
    mutex_lock(&mem_avail_mtx);
1044
    mutex_lock(&mem_avail_mtx);
1058
    mem_avail_frames -= (1ULL << order);
1045
    mem_avail_frames -= (1ULL << order);
1059
    mutex_unlock(&mem_avail_mtx);
1046
    mutex_unlock(&mem_avail_mtx);
1060
 
1047
 
1061
    interrupts_restore(ipl);
1048
    interrupts_restore(ipl);
1062
 
1049
 
1063
    if (flags & FRAME_KA)
1050
    if (flags & FRAME_KA)
1064
        return (void *)PA2KA(PFN2ADDR(v));
1051
        return (void *)PA2KA(PFN2ADDR(v));
1065
    return (void *)PFN2ADDR(v);
1052
    return (void *)PFN2ADDR(v);
1066
}
1053
}
1067
 
1054
 
1068
/** Free a frame.
1055
/** Free a frame.
1069
 *
1056
 *
1070
 * Find respective frame structure for supplied physical frame address.
1057
 * Find respective frame structure for supplied physical frame address.
1071
 * Decrement frame reference count.
1058
 * Decrement frame reference count.
1072
 * If it drops to zero, move the frame structure to free list.
1059
 * If it drops to zero, move the frame structure to free list.
1073
 *
1060
 *
1074
 * @param frame     Physical Address of of the frame to be freed.
1061
 * @param frame     Physical Address of of the frame to be freed.
1075
 */
1062
 */
1076
void frame_free(uintptr_t frame)
1063
void frame_free(uintptr_t frame)
1077
{
1064
{
1078
    ipl_t ipl;
1065
    ipl_t ipl;
1079
    zone_t *zone;
1066
    zone_t *zone;
1080
    pfn_t pfn = ADDR2PFN(frame);
1067
    pfn_t pfn = ADDR2PFN(frame);
1081
 
1068
 
1082
    ipl = interrupts_disable();
1069
    ipl = interrupts_disable();
1083
 
1070
 
1084
    /*
1071
    /*
1085
     * First, find host frame zone for addr.
1072
     * First, find host frame zone for addr.
1086
     */
1073
     */
1087
    zone = find_zone_and_lock(pfn, NULL);
1074
    zone = find_zone_and_lock(pfn, NULL);
1088
    ASSERT(zone);
1075
    ASSERT(zone);
1089
   
1076
   
1090
    zone_frame_free(zone, pfn - zone->base);
1077
    zone_frame_free(zone, pfn - zone->base);
1091
   
1078
   
1092
    spinlock_unlock(&zone->lock);
1079
    spinlock_unlock(&zone->lock);
1093
   
1080
   
1094
    /*
1081
    /*
1095
     * Signal that some memory has been freed.
1082
     * Signal that some memory has been freed.
1096
     */
1083
     */
1097
    mutex_lock(&mem_avail_mtx);
1084
    mutex_lock(&mem_avail_mtx);
1098
    mem_avail_frames++;
1085
    mem_avail_frames++;
1099
    mem_avail_gen++;
1086
    mem_avail_gen++;
1100
    condvar_broadcast(&mem_avail_cv);
1087
    condvar_broadcast(&mem_avail_cv);
1101
    mutex_unlock(&mem_avail_mtx);
1088
    mutex_unlock(&mem_avail_mtx);
1102
 
1089
 
1103
    interrupts_restore(ipl);
1090
    interrupts_restore(ipl);
1104
}
1091
}
1105
 
1092
 
1106
/** Add reference to frame.
1093
/** Add reference to frame.
1107
 *
1094
 *
1108
 * Find respective frame structure for supplied PFN and
1095
 * Find respective frame structure for supplied PFN and
1109
 * increment frame reference count.
1096
 * increment frame reference count.
1110
 *
1097
 *
1111
 * @param pfn       Frame number of the frame to be freed.
1098
 * @param pfn       Frame number of the frame to be freed.
1112
 */
1099
 */
1113
void frame_reference_add(pfn_t pfn)
1100
void frame_reference_add(pfn_t pfn)
1114
{
1101
{
1115
    ipl_t ipl;
1102
    ipl_t ipl;
1116
    zone_t *zone;
1103
    zone_t *zone;
1117
    frame_t *frame;
1104
    frame_t *frame;
1118
 
1105
 
1119
    ipl = interrupts_disable();
1106
    ipl = interrupts_disable();
1120
   
1107
   
1121
    /*
1108
    /*
1122
     * First, find host frame zone for addr.
1109
     * First, find host frame zone for addr.
1123
     */
1110
     */
1124
    zone = find_zone_and_lock(pfn, NULL);
1111
    zone = find_zone_and_lock(pfn, NULL);
1125
    ASSERT(zone);
1112
    ASSERT(zone);
1126
   
1113
   
1127
    frame = &zone->frames[pfn - zone->base];
1114
    frame = &zone->frames[pfn - zone->base];
1128
    frame->refcount++;
1115
    frame->refcount++;
1129
   
1116
   
1130
    spinlock_unlock(&zone->lock);
1117
    spinlock_unlock(&zone->lock);
1131
    interrupts_restore(ipl);
1118
    interrupts_restore(ipl);
1132
}
1119
}
1133
 
1120
 
1134
/** Mark given range unavailable in frame zones. */
1121
/** Mark given range unavailable in frame zones. */
1135
void frame_mark_unavailable(pfn_t start, count_t count)
1122
void frame_mark_unavailable(pfn_t start, count_t count)
1136
{
1123
{
1137
    unsigned int i;
1124
    unsigned int i;
1138
    zone_t *zone;
1125
    zone_t *zone;
1139
    unsigned int prefzone = 0;
1126
    unsigned int prefzone = 0;
1140
   
1127
   
1141
    for (i = 0; i < count; i++) {
1128
    for (i = 0; i < count; i++) {
1142
        zone = find_zone_and_lock(start + i, &prefzone);
1129
        zone = find_zone_and_lock(start + i, &prefzone);
1143
        if (!zone) /* PFN not found */
1130
        if (!zone) /* PFN not found */
1144
            continue;
1131
            continue;
1145
        zone_mark_unavailable(zone, start + i - zone->base);
1132
        zone_mark_unavailable(zone, start + i - zone->base);
1146
 
1133
 
1147
        spinlock_unlock(&zone->lock);
1134
        spinlock_unlock(&zone->lock);
1148
    }
1135
    }
1149
}
1136
}
1150
 
1137
 
1151
/** Initialize physical memory management. */
1138
/** Initialize physical memory management. */
1152
void frame_init(void)
1139
void frame_init(void)
1153
{
1140
{
1154
    if (config.cpu_active == 1) {
1141
    if (config.cpu_active == 1) {
1155
        zones.count = 0;
1142
        zones.count = 0;
1156
        spinlock_initialize(&zones.lock, "zones.lock");
1143
        spinlock_initialize(&zones.lock, "zones.lock");
1157
        mutex_initialize(&mem_avail_mtx, MUTEX_ACTIVE);
1144
        mutex_initialize(&mem_avail_mtx, MUTEX_ACTIVE);
1158
        condvar_initialize(&mem_avail_cv);
1145
        condvar_initialize(&mem_avail_cv);
1159
    }
1146
    }
1160
    /* Tell the architecture to create some memory */
1147
    /* Tell the architecture to create some memory */
1161
    frame_arch_init();
1148
    frame_arch_init();
1162
    if (config.cpu_active == 1) {
1149
    if (config.cpu_active == 1) {
1163
        frame_mark_unavailable(ADDR2PFN(KA2PA(config.base)),
1150
        frame_mark_unavailable(ADDR2PFN(KA2PA(config.base)),
1164
            SIZE2FRAMES(config.kernel_size));
1151
            SIZE2FRAMES(config.kernel_size));
1165
        frame_mark_unavailable(ADDR2PFN(KA2PA(config.stack_base)),
1152
        frame_mark_unavailable(ADDR2PFN(KA2PA(config.stack_base)),
1166
            SIZE2FRAMES(config.stack_size));
1153
            SIZE2FRAMES(config.stack_size));
1167
       
1154
       
1168
        count_t i;
1155
        count_t i;
1169
        for (i = 0; i < init.cnt; i++) {
1156
        for (i = 0; i < init.cnt; i++) {
1170
            pfn_t pfn = ADDR2PFN(KA2PA(init.tasks[i].addr));
1157
            pfn_t pfn = ADDR2PFN(KA2PA(init.tasks[i].addr));
1171
            frame_mark_unavailable(pfn,
1158
            frame_mark_unavailable(pfn,
1172
                SIZE2FRAMES(init.tasks[i].size));
1159
                SIZE2FRAMES(init.tasks[i].size));
1173
        }
1160
        }
1174
 
1161
 
1175
        if (ballocs.size)
1162
        if (ballocs.size)
1176
            frame_mark_unavailable(ADDR2PFN(KA2PA(ballocs.base)),
1163
            frame_mark_unavailable(ADDR2PFN(KA2PA(ballocs.base)),
1177
                SIZE2FRAMES(ballocs.size));
1164
                SIZE2FRAMES(ballocs.size));
1178
 
1165
 
1179
        /* Black list first frame, as allocating NULL would
1166
        /* Black list first frame, as allocating NULL would
1180
         * fail in some places */
1167
         * fail in some places */
1181
        frame_mark_unavailable(0, 1);
1168
        frame_mark_unavailable(0, 1);
1182
    }
1169
    }
1183
}
1170
}
1184
 
1171
 
1185
 
1172
 
1186
/** Return total size of all zones. */
1173
/** Return total size of all zones. */
1187
uint64_t zone_total_size(void)
1174
uint64_t zone_total_size(void)
1188
{
1175
{
1189
    zone_t *zone = NULL;
1176
    zone_t *zone = NULL;
1190
    unsigned int i;
1177
    unsigned int i;
1191
    ipl_t ipl;
1178
    ipl_t ipl;
1192
    uint64_t total = 0;
1179
    uint64_t total = 0;
1193
 
1180
 
1194
    ipl = interrupts_disable();
1181
    ipl = interrupts_disable();
1195
    spinlock_lock(&zones.lock);
1182
    spinlock_lock(&zones.lock);
1196
   
1183
   
1197
    for (i = 0; i < zones.count; i++) {
1184
    for (i = 0; i < zones.count; i++) {
1198
        zone = zones.info[i];
1185
        zone = zones.info[i];
1199
        spinlock_lock(&zone->lock);
1186
        spinlock_lock(&zone->lock);
1200
        total += (uint64_t) FRAMES2SIZE(zone->count);
1187
        total += (uint64_t) FRAMES2SIZE(zone->count);
1201
        spinlock_unlock(&zone->lock);
1188
        spinlock_unlock(&zone->lock);
1202
    }
1189
    }
1203
   
1190
   
1204
    spinlock_unlock(&zones.lock);
1191
    spinlock_unlock(&zones.lock);
1205
    interrupts_restore(ipl);
1192
    interrupts_restore(ipl);
1206
   
1193
   
1207
    return total;
1194
    return total;
1208
}
1195
}
1209
 
1196
 
1210
/** Prints list of zones. */
1197
/** Prints list of zones. */
1211
void zone_print_list(void)
1198
void zone_print_list(void)
1212
{
1199
{
1213
    zone_t *zone = NULL;
1200
    zone_t *zone = NULL;
1214
    unsigned int i;
1201
    unsigned int i;
1215
    ipl_t ipl;
1202
    ipl_t ipl;
1216
 
1203
 
1217
#ifdef __32_BITS__  
1204
#ifdef __32_BITS__  
1218
    printf("#  base address free frames  busy frames\n");
1205
    printf("#  base address free frames  busy frames\n");
1219
    printf("-- ------------ ------------ ------------\n");
1206
    printf("-- ------------ ------------ ------------\n");
1220
#endif
1207
#endif
1221
 
1208
 
1222
#ifdef __64_BITS__
1209
#ifdef __64_BITS__
1223
    printf("#  base address         free frames  busy frames\n");
1210
    printf("#  base address         free frames  busy frames\n");
1224
    printf("-- -------------------- ------------ ------------\n");
1211
    printf("-- -------------------- ------------ ------------\n");
1225
#endif
1212
#endif
1226
   
1213
   
1227
    /*
1214
    /*
1228
     * Because printing may require allocation of memory, we may not hold
1215
     * Because printing may require allocation of memory, we may not hold
1229
     * the frame allocator locks when printing zone statistics.  Therefore,
1216
     * the frame allocator locks when printing zone statistics.  Therefore,
1230
     * we simply gather the statistics under the protection of the locks and
1217
     * we simply gather the statistics under the protection of the locks and
1231
     * print the statistics when the locks have been released.
1218
     * print the statistics when the locks have been released.
1232
     *
1219
     *
1233
     * When someone adds/removes zones while we are printing the statistics,
1220
     * When someone adds/removes zones while we are printing the statistics,
1234
     * we may end up with inaccurate output (e.g. a zone being skipped from
1221
     * we may end up with inaccurate output (e.g. a zone being skipped from
1235
     * the listing).
1222
     * the listing).
1236
     */
1223
     */
1237
 
1224
 
1238
    for (i = 0; ; i++) {
1225
    for (i = 0; ; i++) {
1239
        uintptr_t base;
1226
        uintptr_t base;
1240
        count_t free_count;
1227
        count_t free_count;
1241
        count_t busy_count;
1228
        count_t busy_count;
1242
 
1229
 
1243
        ipl = interrupts_disable();
1230
        ipl = interrupts_disable();
1244
        spinlock_lock(&zones.lock);
1231
        spinlock_lock(&zones.lock);
1245
       
1232
       
1246
        if (i >= zones.count) {
1233
        if (i >= zones.count) {
1247
            spinlock_unlock(&zones.lock);
1234
            spinlock_unlock(&zones.lock);
1248
            interrupts_restore(ipl);
1235
            interrupts_restore(ipl);
1249
            break;
1236
            break;
1250
        }
1237
        }
1251
 
1238
 
1252
        zone = zones.info[i];
1239
        zone = zones.info[i];
1253
        spinlock_lock(&zone->lock);
1240
        spinlock_lock(&zone->lock);
1254
 
1241
 
1255
        base = PFN2ADDR(zone->base);
1242
        base = PFN2ADDR(zone->base);
1256
        free_count = zone->free_count;
1243
        free_count = zone->free_count;
1257
        busy_count = zone->busy_count;
1244
        busy_count = zone->busy_count;
1258
 
1245
 
1259
        spinlock_unlock(&zone->lock);
1246
        spinlock_unlock(&zone->lock);
1260
       
1247
       
1261
        spinlock_unlock(&zones.lock);
1248
        spinlock_unlock(&zones.lock);
1262
        interrupts_restore(ipl);
1249
        interrupts_restore(ipl);
1263
 
1250
 
1264
#ifdef __32_BITS__
1251
#ifdef __32_BITS__
1265
        printf("%-2u   %10p %12" PRIc " %12" PRIc "\n", i, base,
1252
        printf("%-2u   %10p %12" PRIc " %12" PRIc "\n", i, base,
1266
            free_count, busy_count);
1253
            free_count, busy_count);
1267
#endif
1254
#endif
1268
 
1255
 
1269
#ifdef __64_BITS__
1256
#ifdef __64_BITS__
1270
        printf("%-2u   %18p %12" PRIc " %12" PRIc "\n", i, base,
1257
        printf("%-2u   %18p %12" PRIc " %12" PRIc "\n", i, base,
1271
            free_count, busy_count);
1258
            free_count, busy_count);
1272
#endif
1259
#endif
1273
       
1260
       
1274
    }
1261
    }
1275
}
1262
}
1276
 
1263
 
1277
/** Prints zone details.
1264
/** Prints zone details.
1278
 *
1265
 *
1279
 * @param num       Zone base address or zone number.
1266
 * @param num       Zone base address or zone number.
1280
 */
1267
 */
1281
void zone_print_one(unsigned int num)
1268
void zone_print_one(unsigned int num)
1282
{
1269
{
1283
    zone_t *zone = NULL;
1270
    zone_t *zone = NULL;
1284
    ipl_t ipl;
1271
    ipl_t ipl;
1285
    unsigned int i;
1272
    unsigned int i;
-
 
1273
    uintptr_t base;
-
 
1274
    count_t count;
-
 
1275
    count_t busy_count;
-
 
1276
    count_t free_count;
1286
 
1277
 
1287
    ipl = interrupts_disable();
1278
    ipl = interrupts_disable();
1288
    spinlock_lock(&zones.lock);
1279
    spinlock_lock(&zones.lock);
1289
 
1280
 
1290
    for (i = 0; i < zones.count; i++) {
1281
    for (i = 0; i < zones.count; i++) {
1291
        if ((i == num) || (PFN2ADDR(zones.info[i]->base) == num)) {
1282
        if ((i == num) || (PFN2ADDR(zones.info[i]->base) == num)) {
1292
            zone = zones.info[i];
1283
            zone = zones.info[i];
1293
            break;
1284
            break;
1294
        }
1285
        }
1295
    }
1286
    }
1296
    if (!zone) {
1287
    if (!zone) {
-
 
1288
        spinlock_unlock(&zones.lock);
-
 
1289
        interrupts_restore(ipl);
1297
        printf("Zone not found.\n");
1290
        printf("Zone not found.\n");
1298
        goto out;
1291
        return;
1299
    }
1292
    }
1300
   
1293
   
1301
    spinlock_lock(&zone->lock);
1294
    spinlock_lock(&zone->lock);
1302
    printf("Memory zone information\n");
-
 
1303
    printf("Zone base address: %p\n", PFN2ADDR(zone->base));
1295
    base = PFN2ADDR(zone->base);
1304
    printf("Zone size: %" PRIc " frames (%" PRIs " KB)\n", zone->count,
-
 
1305
        SIZE2KB(FRAMES2SIZE(zone->count)));
1296
    count = zone->count;
1306
    printf("Allocated space: %" PRIc " frames (%" PRIs " KB)\n",
-
 
1307
        zone->busy_count, SIZE2KB(FRAMES2SIZE(zone->busy_count)));
1297
    busy_count = zone->busy_count;
1308
    printf("Available space: %" PRIc " frames (%" PRIs " KB)\n",
-
 
1309
        zone->free_count, SIZE2KB(FRAMES2SIZE(zone->free_count)));
1298
    free_count = zone->free_count;
1310
    buddy_system_structure_print(zone->buddy_system, FRAME_SIZE);
-
 
1311
    spinlock_unlock(&zone->lock);
1299
    spinlock_unlock(&zone->lock);
1312
   
-
 
1313
out:
-
 
1314
    spinlock_unlock(&zones.lock);
1300
    spinlock_unlock(&zones.lock);
1315
    interrupts_restore(ipl);
1301
    interrupts_restore(ipl);
-
 
1302
 
-
 
1303
    printf("Zone base address: %p\n", base);
-
 
1304
    printf("Zone size: %" PRIc " frames (%" PRIs " KiB)\n", count,
-
 
1305
        SIZE2KB(FRAMES2SIZE(count)));
-
 
1306
    printf("Allocated space: %" PRIc " frames (%" PRIs " KiB)\n",
-
 
1307
        busy_count, SIZE2KB(FRAMES2SIZE(busy_count)));
-
 
1308
    printf("Available space: %" PRIc " frames (%" PRIs " KiB)\n",
-
 
1309
        free_count, SIZE2KB(FRAMES2SIZE(free_count)));
1316
}
1310
}
1317
 
1311
 
1318
/** @}
1312
/** @}
1319
 */
1313
 */
1320
 
1314
 
1321
 
1315