2
* Copyright (c) 2001-2005 Jakub Jermar
3
* Copyright (c) 2005 Sergey Bondari
4
* Copyright (c) 2009 Martin Decky
7
* Redistribution and use in source and binary forms, with or without
8
* modification, are permitted provided that the following conditions
11
* - Redistributions of source code must retain the above copyright
12
* notice, this list of conditions and the following disclaimer.
13
* - Redistributions in binary form must reproduce the above copyright
14
* notice, this list of conditions and the following disclaimer in the
15
* documentation and/or other materials provided with the distribution.
16
* - The name of the author may not be used to endorse or promote products
17
* derived from this software without specific prior written permission.
19
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
20
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
21
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
22
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
23
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
24
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
28
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31
/** @addtogroup genericmm
37
* @brief Physical frame allocator.
39
* This file contains the physical frame allocator and memory zone management.
40
* The frame allocator is built on top of the buddy allocator.
45
#include <arch/types.h>
51
#include <synch/mutex.h>
52
#include <synch/condvar.h>
65
* Synchronization primitives used to sleep when there is no memory
68
mutex_t mem_avail_mtx;
69
condvar_t mem_avail_cv;
70
size_t mem_avail_req = 0; /**< Number of frames requested. */
71
size_t mem_avail_gen = 0; /**< Generation counter. */
73
/********************/
74
/* Helper functions */
75
/********************/
77
static inline size_t frame_index(zone_t *zone, frame_t *frame)
79
return (size_t) (frame - zone->frames);
82
static inline size_t frame_index_abs(zone_t *zone, frame_t *frame)
84
return (size_t) (frame - zone->frames) + zone->base;
87
static inline bool frame_index_valid(zone_t *zone, size_t index)
89
return (index < zone->count);
92
static inline size_t make_frame_index(zone_t *zone, frame_t *frame)
94
return (frame - zone->frames);
97
/** Initialize frame structure.
99
* @param frame Frame structure to be initialized.
102
static void frame_initialize(frame_t *frame)
105
frame->buddy_order = 0;
108
/*******************/
109
/* Zones functions */
110
/*******************/
112
/** Insert-sort zone into zones list.
114
* Assume interrupts are disabled and zones lock is
117
* @param base Base frame of the newly inserted zone.
118
* @param count Number of frames of the newly inserted zone.
120
* @return Zone number on success, -1 on error.
123
static size_t zones_insert_zone(pfn_t base, size_t count)
125
if (zones.count + 1 == ZONES_MAX) {
126
printf("Maximum zone count %u exceeded!\n", ZONES_MAX);
131
for (i = 0; i < zones.count; i++) {
132
/* Check for overlap */
133
if (overlaps(base, count,
134
zones.info[i].base, zones.info[i].count)) {
135
printf("Zones overlap!\n");
138
if (base < zones.info[i].base)
142
/* Move other zones up */
144
for (j = zones.count; j > i; j--) {
145
zones.info[j] = zones.info[j - 1];
146
zones.info[j].buddy_system->data =
147
(void *) &zones.info[j - 1];
155
/** Get total available frames.
157
* Assume interrupts are disabled and zones lock is
160
* @return Total number of available frames.
164
static size_t total_frames_free(void)
168
for (i = 0; i < zones.count; i++)
169
total += zones.info[i].free_count;
175
/** Find a zone with a given frames.
177
* Assume interrupts are disabled and zones lock is
180
* @param frame Frame number contained in zone.
181
* @param count Number of frames to look for.
182
* @param hint Used as zone hint.
184
* @return Zone index or -1 if not found.
187
size_t find_zone(pfn_t frame, size_t count, size_t hint)
189
if (hint >= zones.count)
194
if ((zones.info[i].base <= frame)
195
&& (zones.info[i].base + zones.info[i].count >= frame + count))
199
if (i >= zones.count)
206
/** @return True if zone can allocate specified order */
207
static bool zone_can_alloc(zone_t *zone, uint8_t order)
209
return (zone_flags_available(zone->flags)
210
&& buddy_system_can_alloc(zone->buddy_system, order));
213
/** Find a zone that can allocate order frames.
215
* Assume interrupts are disabled and zones lock is
218
* @param order Size (2^order) of free space we are trying to find.
219
* @param flags Required flags of the target zone.
220
* @param hind Preferred zone.
223
static size_t find_free_zone(uint8_t order, zone_flags_t flags, size_t hint)
225
if (hint >= zones.count)
231
* Check whether the zone meets the search criteria.
233
if ((zones.info[i].flags & flags) == flags) {
235
* Check if the zone has 2^order frames area available.
237
if (zone_can_alloc(&zones.info[i], order))
242
if (i >= zones.count)
249
/**************************/
250
/* Buddy system functions */
251
/**************************/
253
/** Buddy system find_block implementation.
255
* Find block that is parent of current list.
256
* That means go to lower addresses, until such block is found
258
* @param order Order of parent must be different then this
262
static link_t *zone_buddy_find_block(buddy_system_t *buddy, link_t *child,
265
frame_t *frame = list_get_instance(child, frame_t, buddy_link);
266
zone_t *zone = (zone_t *) buddy->data;
268
size_t index = frame_index(zone, frame);
270
if (zone->frames[index].buddy_order != order)
271
return &zone->frames[index].buddy_link;
272
} while (index-- > 0);
277
/** Buddy system find_buddy implementation.
279
* @param buddy Buddy system.
280
* @param block Block for which buddy should be found.
282
* @return Buddy for given block if found.
285
static link_t *zone_buddy_find_buddy(buddy_system_t *buddy, link_t *block)
287
frame_t *frame = list_get_instance(block, frame_t, buddy_link);
288
zone_t *zone = (zone_t *) buddy->data;
289
ASSERT(IS_BUDDY_ORDER_OK(frame_index_abs(zone, frame),
290
frame->buddy_order));
292
bool is_left = IS_BUDDY_LEFT_BLOCK_ABS(zone, frame);
296
index = (frame_index(zone, frame)) +
297
(1 << frame->buddy_order);
298
} else { /* is_right */
299
index = (frame_index(zone, frame)) -
300
(1 << frame->buddy_order);
303
if (frame_index_valid(zone, index)) {
304
if ((zone->frames[index].buddy_order == frame->buddy_order) &&
305
(zone->frames[index].refcount == 0)) {
306
return &zone->frames[index].buddy_link;
313
/** Buddy system bisect implementation.
315
* @param buddy Buddy system.
316
* @param block Block to bisect.
318
* @return Right block.
321
static link_t *zone_buddy_bisect(buddy_system_t *buddy, link_t *block)
323
frame_t *frame_l = list_get_instance(block, frame_t, buddy_link);
324
frame_t *frame_r = (frame_l + (1 << (frame_l->buddy_order - 1)));
326
return &frame_r->buddy_link;
329
/** Buddy system coalesce implementation.
331
* @param buddy Buddy system.
332
* @param block_1 First block.
333
* @param block_2 First block's buddy.
335
* @return Coalesced block (actually block that represents lower
339
static link_t *zone_buddy_coalesce(buddy_system_t *buddy, link_t *block_1,
342
frame_t *frame1 = list_get_instance(block_1, frame_t, buddy_link);
343
frame_t *frame2 = list_get_instance(block_2, frame_t, buddy_link);
345
return ((frame1 < frame2) ? block_1 : block_2);
348
/** Buddy system set_order implementation.
350
* @param buddy Buddy system.
351
* @param block Buddy system block.
352
* @param order Order to set.
355
static void zone_buddy_set_order(buddy_system_t *buddy, link_t *block,
358
list_get_instance(block, frame_t, buddy_link)->buddy_order = order;
361
/** Buddy system get_order implementation.
363
* @param buddy Buddy system.
364
* @param block Buddy system block.
366
* @return Order of block.
369
static uint8_t zone_buddy_get_order(buddy_system_t *buddy, link_t *block)
371
return list_get_instance(block, frame_t, buddy_link)->buddy_order;
374
/** Buddy system mark_busy implementation.
376
* @param buddy Buddy system.
377
* @param block Buddy system block.
380
static void zone_buddy_mark_busy(buddy_system_t *buddy, link_t * block)
382
list_get_instance(block, frame_t, buddy_link)->refcount = 1;
385
/** Buddy system mark_available implementation.
387
* @param buddy Buddy system.
388
* @param block Buddy system block.
390
static void zone_buddy_mark_available(buddy_system_t *buddy, link_t *block)
392
list_get_instance(block, frame_t, buddy_link)->refcount = 0;
395
static buddy_system_operations_t zone_buddy_system_operations = {
396
.find_buddy = zone_buddy_find_buddy,
397
.bisect = zone_buddy_bisect,
398
.coalesce = zone_buddy_coalesce,
399
.set_order = zone_buddy_set_order,
400
.get_order = zone_buddy_get_order,
401
.mark_busy = zone_buddy_mark_busy,
402
.mark_available = zone_buddy_mark_available,
403
.find_block = zone_buddy_find_block
410
/** Allocate frame in particular zone.
412
* Assume zone is locked and is available for allocation.
413
* Panics if allocation is impossible.
415
* @param zone Zone to allocate from.
416
* @param order Allocate exactly 2^order frames.
418
* @return Frame index in zone.
421
static pfn_t zone_frame_alloc(zone_t *zone, uint8_t order)
423
ASSERT(zone_flags_available(zone->flags));
425
/* Allocate frames from zone buddy system */
426
link_t *link = buddy_system_alloc(zone->buddy_system, order);
430
/* Update zone information. */
431
zone->free_count -= (1 << order);
432
zone->busy_count += (1 << order);
434
/* Frame will be actually a first frame of the block. */
435
frame_t *frame = list_get_instance(link, frame_t, buddy_link);
437
/* Get frame address */
438
return make_frame_index(zone, frame);
441
/** Free frame from zone.
443
* Assume zone is locked and is available for deallocation.
445
* @param zone Pointer to zone from which the frame is to be freed.
446
* @param frame_idx Frame index relative to zone.
449
static void zone_frame_free(zone_t *zone, size_t frame_idx)
451
ASSERT(zone_flags_available(zone->flags));
453
frame_t *frame = &zone->frames[frame_idx];
455
/* Remember frame order */
456
uint8_t order = frame->buddy_order;
458
ASSERT(frame->refcount);
460
if (!--frame->refcount) {
461
buddy_system_free(zone->buddy_system, &frame->buddy_link);
463
/* Update zone information. */
464
zone->free_count += (1 << order);
465
zone->busy_count -= (1 << order);
469
/** Return frame from zone. */
470
static frame_t *zone_get_frame(zone_t *zone, size_t frame_idx)
472
ASSERT(frame_idx < zone->count);
473
return &zone->frames[frame_idx];
476
/** Mark frame in zone unavailable to allocation. */
477
static void zone_mark_unavailable(zone_t *zone, size_t frame_idx)
479
ASSERT(zone_flags_available(zone->flags));
481
frame_t *frame = zone_get_frame(zone, frame_idx);
485
link_t *link __attribute__ ((unused));
487
link = buddy_system_alloc_block(zone->buddy_system,
496
* Expect buddy to point to space at least zone_conf_size large.
497
* Assume z1 & z2 are locked and compatible and zones lock is
500
* @param z1 First zone to merge.
501
* @param z2 Second zone to merge.
502
* @param old_z1 Original date of the first zone.
503
* @param buddy Merged zone buddy.
506
static void zone_merge_internal(size_t z1, size_t z2, zone_t *old_z1, buddy_system_t *buddy)
508
ASSERT(zone_flags_available(zones.info[z1].flags));
509
ASSERT(zone_flags_available(zones.info[z2].flags));
510
ASSERT(zones.info[z1].flags == zones.info[z2].flags);
511
ASSERT(zones.info[z1].base < zones.info[z2].base);
512
ASSERT(!overlaps(zones.info[z1].base, zones.info[z1].count,
513
zones.info[z2].base, zones.info[z2].count));
515
/* Difference between zone bases */
516
pfn_t base_diff = zones.info[z2].base - zones.info[z1].base;
518
zones.info[z1].count = base_diff + zones.info[z2].count;
519
zones.info[z1].free_count += zones.info[z2].free_count;
520
zones.info[z1].busy_count += zones.info[z2].busy_count;
521
zones.info[z1].buddy_system = buddy;
523
uint8_t order = fnzb(zones.info[z1].count);
524
buddy_system_create(zones.info[z1].buddy_system, order,
525
&zone_buddy_system_operations, (void *) &zones.info[z1]);
527
zones.info[z1].frames =
528
(frame_t *) ((uint8_t *) zones.info[z1].buddy_system
529
+ buddy_conf_size(order));
531
/* This marks all frames busy */
533
for (i = 0; i < zones.info[z1].count; i++)
534
frame_initialize(&zones.info[z1].frames[i]);
536
/* Copy frames from both zones to preserve full frame orders,
537
* parents etc. Set all free frames with refcount = 0 to 1, because
538
* we add all free frames to buddy allocator later again, clearing
539
* order to 0. Don't set busy frames with refcount = 0, as they
540
* will not be reallocated during merge and it would make later
541
* problems with allocation/free.
543
for (i = 0; i < old_z1->count; i++)
544
zones.info[z1].frames[i] = old_z1->frames[i];
546
for (i = 0; i < zones.info[z2].count; i++)
547
zones.info[z1].frames[base_diff + i]
548
= zones.info[z2].frames[i];
551
while (i < zones.info[z1].count) {
552
if (zones.info[z1].frames[i].refcount) {
553
/* Skip busy frames */
554
i += 1 << zones.info[z1].frames[i].buddy_order;
556
/* Free frames, set refcount = 1
557
* (all free frames have refcount == 0, we need not
558
* to check the order)
560
zones.info[z1].frames[i].refcount = 1;
561
zones.info[z1].frames[i].buddy_order = 0;
566
/* Add free blocks from the original zone z1 */
567
while (zone_can_alloc(old_z1, 0)) {
568
/* Allocate from the original zone */
569
pfn_t frame_idx = zone_frame_alloc(old_z1, 0);
571
/* Free the frame from the merged zone */
572
frame_t *frame = &zones.info[z1].frames[frame_idx];
574
buddy_system_free(zones.info[z1].buddy_system, &frame->buddy_link);
577
/* Add free blocks from the original zone z2 */
578
while (zone_can_alloc(&zones.info[z2], 0)) {
579
/* Allocate from the original zone */
580
pfn_t frame_idx = zone_frame_alloc(&zones.info[z2], 0);
582
/* Free the frame from the merged zone */
583
frame_t *frame = &zones.info[z1].frames[base_diff + frame_idx];
585
buddy_system_free(zones.info[z1].buddy_system, &frame->buddy_link);
589
/** Return old configuration frames into the zone.
592
* - The configuration data is outside the zone
593
* -> do nothing (perhaps call frame_free?)
594
* - The configuration data was created by zone_create
595
* or updated by reduce_region -> free every frame
597
* @param znum The actual zone where freeing should occur.
598
* @param pfn Old zone configuration frame.
599
* @param count Old zone frame count.
602
static void return_config_frames(size_t znum, pfn_t pfn, size_t count)
604
ASSERT(zone_flags_available(zones.info[znum].flags));
606
size_t cframes = SIZE2FRAMES(zone_conf_size(count));
608
if ((pfn < zones.info[znum].base)
609
|| (pfn >= zones.info[znum].base + zones.info[znum].count))
612
frame_t *frame __attribute__ ((unused));
614
frame = &zones.info[znum].frames[pfn - zones.info[znum].base];
615
ASSERT(!frame->buddy_order);
618
for (i = 0; i < cframes; i++) {
619
zones.info[znum].busy_count++;
620
zone_frame_free(&zones.info[znum],
621
pfn - zones.info[znum].base + i);
625
/** Reduce allocated block to count of order 0 frames.
627
* The allocated block needs 2^order frames. Reduce all frames
628
* in the block to order 0 and free the unneeded frames. This means that
629
* when freeing the previously allocated block starting with frame_idx,
630
* you have to free every frame.
633
* @param frame_idx Index the first frame of the block.
634
* @param count Allocated frames in block.
637
static void zone_reduce_region(size_t znum, pfn_t frame_idx, size_t count)
639
ASSERT(zone_flags_available(zones.info[znum].flags));
640
ASSERT(frame_idx + count < zones.info[znum].count);
642
uint8_t order = zones.info[znum].frames[frame_idx].buddy_order;
643
ASSERT((size_t) (1 << order) >= count);
645
/* Reduce all blocks to order 0 */
647
for (i = 0; i < (size_t) (1 << order); i++) {
648
frame_t *frame = &zones.info[znum].frames[i + frame_idx];
649
frame->buddy_order = 0;
650
if (!frame->refcount)
652
ASSERT(frame->refcount == 1);
655
/* Free unneeded frames */
656
for (i = count; i < (size_t) (1 << order); i++)
657
zone_frame_free(&zones.info[znum], i + frame_idx);
660
/** Merge zones z1 and z2.
662
* The merged zones must be 2 zones with no zone existing in between
663
* (which means that z2 = z1 + 1). Both zones must be available zones
664
* with the same flags.
666
* When you create a new zone, the frame allocator configuration does
667
* not to be 2^order size. Once the allocator is running it is no longer
668
* possible, merged configuration data occupies more space :-/
673
bool zone_merge(size_t z1, size_t z2)
675
ipl_t ipl = interrupts_disable();
676
spinlock_lock(&zones.lock);
680
/* We can join only 2 zones with none existing inbetween,
681
* the zones have to be available and with the same
684
if ((z1 >= zones.count) || (z2 >= zones.count)
686
|| (!zone_flags_available(zones.info[z1].flags))
687
|| (!zone_flags_available(zones.info[z2].flags))
688
|| (zones.info[z1].flags != zones.info[z2].flags)) {
693
pfn_t cframes = SIZE2FRAMES(zone_conf_size(
694
zones.info[z2].base - zones.info[z1].base
695
+ zones.info[z2].count));
701
order = fnzb(cframes - 1) + 1;
703
/* Allocate merged zone data inside one of the zones */
705
if (zone_can_alloc(&zones.info[z1], order)) {
706
pfn = zones.info[z1].base + zone_frame_alloc(&zones.info[z1], order);
707
} else if (zone_can_alloc(&zones.info[z2], order)) {
708
pfn = zones.info[z2].base + zone_frame_alloc(&zones.info[z2], order);
714
/* Preserve original data from z1 */
715
zone_t old_z1 = zones.info[z1];
716
old_z1.buddy_system->data = (void *) &old_z1;
718
/* Do zone merging */
719
buddy_system_t *buddy = (buddy_system_t *) PA2KA(PFN2ADDR(pfn));
720
zone_merge_internal(z1, z2, &old_z1, buddy);
722
/* Free unneeded config frames */
723
zone_reduce_region(z1, pfn - zones.info[z1].base, cframes);
725
/* Subtract zone information from busy frames */
726
zones.info[z1].busy_count -= cframes;
728
/* Free old zone information */
729
return_config_frames(z1,
730
ADDR2PFN(KA2PA((uintptr_t) old_z1.frames)), old_z1.count);
731
return_config_frames(z1,
732
ADDR2PFN(KA2PA((uintptr_t) zones.info[z2].frames)),
733
zones.info[z2].count);
735
/* Move zones down */
737
for (i = z2 + 1; i < zones.count; i++) {
738
zones.info[i - 1] = zones.info[i];
739
zones.info[i - 1].buddy_system->data =
740
(void *) &zones.info[i - 1];
746
spinlock_unlock(&zones.lock);
747
interrupts_restore(ipl);
752
/** Merge all mergeable zones into one big zone.
754
* It is reasonable to do this on systems where
755
* BIOS reports parts in chunks, so that we could
756
* have 1 zone (it's faster).
759
void zone_merge_all(void)
762
while (i < zones.count) {
763
if (!zone_merge(i, i + 1))
768
/** Create new frame zone.
770
* @param zone Zone to construct.
771
* @param buddy Address of buddy system configuration information.
772
* @param start Physical address of the first frame within the zone.
773
* @param count Count of frames in zone.
774
* @param flags Zone flags.
776
* @return Initialized zone.
779
static void zone_construct(zone_t *zone, buddy_system_t *buddy, pfn_t start, size_t count, zone_flags_t flags)
784
zone->free_count = count;
785
zone->busy_count = 0;
786
zone->buddy_system = buddy;
788
if (zone_flags_available(flags)) {
790
* Compute order for buddy system and initialize
792
uint8_t order = fnzb(count);
793
buddy_system_create(zone->buddy_system, order,
794
&zone_buddy_system_operations, (void *) zone);
796
/* Allocate frames _after_ the confframe */
799
zone->frames = (frame_t *) ((uint8_t *) zone->buddy_system +
800
buddy_conf_size(order));
803
for (i = 0; i < count; i++)
804
frame_initialize(&zone->frames[i]);
806
/* Stuffing frames */
807
for (i = 0; i < count; i++) {
808
zone->frames[i].refcount = 0;
809
buddy_system_free(zone->buddy_system, &zone->frames[i].buddy_link);
815
/** Compute configuration data size for zone.
817
* @param count Size of zone in frames.
819
* @return Size of zone configuration info (in bytes).
822
uintptr_t zone_conf_size(size_t count)
824
return (count * sizeof(frame_t) + buddy_conf_size(fnzb(count)));
827
/** Create and add zone to system.
829
* @param start First frame number (absolute).
830
* @param count Size of zone in frames.
831
* @param confframe Where configuration frames are supposed to be.
832
* Automatically checks, that we will not disturb the
833
* kernel and possibly init. If confframe is given
834
* _outside_ this zone, it is expected, that the area is
835
* already marked BUSY and big enough to contain
836
* zone_conf_size() amount of data. If the confframe is
837
* inside the area, the zone free frame information is
838
* modified not to include it.
840
* @return Zone number or -1 on error.
843
size_t zone_create(pfn_t start, size_t count, pfn_t confframe, zone_flags_t flags)
845
ipl_t ipl = interrupts_disable();
846
spinlock_lock(&zones.lock);
848
if (zone_flags_available(flags)) { /* Create available zone */
849
/* Theoretically we could have NULL here, practically make sure
850
* nobody tries to do that. If some platform requires, remove
853
ASSERT(confframe != NULL);
855
/* If confframe is supposed to be inside our zone, then make sure
856
* it does not span kernel & init
858
size_t confcount = SIZE2FRAMES(zone_conf_size(count));
859
if ((confframe >= start) && (confframe < start + count)) {
860
for (; confframe < start + count; confframe++) {
861
uintptr_t addr = PFN2ADDR(confframe);
862
if (overlaps(addr, PFN2ADDR(confcount),
863
KA2PA(config.base), config.kernel_size))
866
if (overlaps(addr, PFN2ADDR(confcount),
867
KA2PA(config.stack_base), config.stack_size))
870
bool overlap = false;
872
for (i = 0; i < init.cnt; i++)
873
if (overlaps(addr, PFN2ADDR(confcount),
874
KA2PA(init.tasks[i].addr),
875
init.tasks[i].size)) {
885
if (confframe >= start + count)
886
panic("Cannot find configuration data for zone.");
889
size_t znum = zones_insert_zone(start, count);
890
if (znum == (size_t) -1) {
891
spinlock_unlock(&zones.lock);
892
interrupts_restore(ipl);
896
buddy_system_t *buddy = (buddy_system_t *) PA2KA(PFN2ADDR(confframe));
897
zone_construct(&zones.info[znum], buddy, start, count, flags);
899
/* If confdata in zone, mark as unavailable */
900
if ((confframe >= start) && (confframe < start + count)) {
902
for (i = confframe; i < confframe + confcount; i++)
903
zone_mark_unavailable(&zones.info[znum],
904
i - zones.info[znum].base);
907
spinlock_unlock(&zones.lock);
908
interrupts_restore(ipl);
913
/* Non-available zone */
914
size_t znum = zones_insert_zone(start, count);
915
if (znum == (size_t) -1) {
916
spinlock_unlock(&zones.lock);
917
interrupts_restore(ipl);
920
zone_construct(&zones.info[znum], NULL, start, count, flags);
922
spinlock_unlock(&zones.lock);
923
interrupts_restore(ipl);
928
/*******************/
929
/* Frame functions */
930
/*******************/
932
/** Set parent of frame. */
933
void frame_set_parent(pfn_t pfn, void *data, size_t hint)
935
ipl_t ipl = interrupts_disable();
936
spinlock_lock(&zones.lock);
938
size_t znum = find_zone(pfn, 1, hint);
940
ASSERT(znum != (size_t) -1);
942
zone_get_frame(&zones.info[znum],
943
pfn - zones.info[znum].base)->parent = data;
945
spinlock_unlock(&zones.lock);
946
interrupts_restore(ipl);
949
void *frame_get_parent(pfn_t pfn, size_t hint)
951
ipl_t ipl = interrupts_disable();
952
spinlock_lock(&zones.lock);
954
size_t znum = find_zone(pfn, 1, hint);
956
ASSERT(znum != (size_t) -1);
958
void *res = zone_get_frame(&zones.info[znum],
959
pfn - zones.info[znum].base)->parent;
961
spinlock_unlock(&zones.lock);
962
interrupts_restore(ipl);
967
/** Allocate power-of-two frames of physical memory.
969
* @param order Allocate exactly 2^order frames.
970
* @param flags Flags for host zone selection and address processing.
971
* @param pzone Preferred zone.
973
* @return Physical address of the allocated frame.
976
void *frame_alloc_generic(uint8_t order, frame_flags_t flags, size_t *pzone)
978
size_t size = ((size_t) 1) << order;
980
size_t hint = pzone ? (*pzone) : 0;
983
ipl = interrupts_disable();
984
spinlock_lock(&zones.lock);
987
* First, find suitable frame zone.
989
size_t znum = find_free_zone(order,
990
FRAME_TO_ZONE_FLAGS(flags), hint);
992
/* If no memory, reclaim some slab memory,
993
if it does not help, reclaim all */
994
if ((znum == (size_t) -1) && (!(flags & FRAME_NO_RECLAIM))) {
995
spinlock_unlock(&zones.lock);
996
interrupts_restore(ipl);
998
size_t freed = slab_reclaim(0);
1000
ipl = interrupts_disable();
1001
spinlock_lock(&zones.lock);
1004
znum = find_free_zone(order,
1005
FRAME_TO_ZONE_FLAGS(flags), hint);
1007
if (znum == (size_t) -1) {
1008
spinlock_unlock(&zones.lock);
1009
interrupts_restore(ipl);
1011
freed = slab_reclaim(SLAB_RECLAIM_ALL);
1013
ipl = interrupts_disable();
1014
spinlock_lock(&zones.lock);
1017
znum = find_free_zone(order,
1018
FRAME_TO_ZONE_FLAGS(flags), hint);
1022
if (znum == (size_t) -1) {
1023
if (flags & FRAME_ATOMIC) {
1024
spinlock_unlock(&zones.lock);
1025
interrupts_restore(ipl);
1030
size_t avail = total_frames_free();
1033
spinlock_unlock(&zones.lock);
1034
interrupts_restore(ipl);
1037
* Sleep until some frames are available again.
1041
printf("Thread %" PRIu64 " waiting for %" PRIs " frames, "
1042
"%" PRIs " available.\n", THREAD->tid, size, avail);
1045
mutex_lock(&mem_avail_mtx);
1047
if (mem_avail_req > 0)
1048
mem_avail_req = min(mem_avail_req, size);
1050
mem_avail_req = size;
1051
size_t gen = mem_avail_gen;
1053
while (gen == mem_avail_gen)
1054
condvar_wait(&mem_avail_cv, &mem_avail_mtx);
1056
mutex_unlock(&mem_avail_mtx);
1059
printf("Thread %" PRIu64 " woken up.\n", THREAD->tid);
1065
pfn_t pfn = zone_frame_alloc(&zones.info[znum], order)
1066
+ zones.info[znum].base;
1068
spinlock_unlock(&zones.lock);
1069
interrupts_restore(ipl);
1074
if (flags & FRAME_KA)
1075
return (void *) PA2KA(PFN2ADDR(pfn));
1077
return (void *) PFN2ADDR(pfn);
1082
* Find respective frame structure for supplied physical frame address.
1083
* Decrement frame reference count. If it drops to zero, move the frame
1084
* structure to free list.
1086
* @param frame Physical Address of of the frame to be freed.
1089
void frame_free(uintptr_t frame)
1091
ipl_t ipl = interrupts_disable();
1092
spinlock_lock(&zones.lock);
1095
* First, find host frame zone for addr.
1097
pfn_t pfn = ADDR2PFN(frame);
1098
size_t znum = find_zone(pfn, 1, NULL);
1100
ASSERT(znum != (size_t) -1);
1102
zone_frame_free(&zones.info[znum], pfn - zones.info[znum].base);
1104
spinlock_unlock(&zones.lock);
1105
interrupts_restore(ipl);
1108
* Signal that some memory has been freed.
1110
mutex_lock(&mem_avail_mtx);
1111
if (mem_avail_req > 0)
1114
if (mem_avail_req == 0) {
1116
condvar_broadcast(&mem_avail_cv);
1118
mutex_unlock(&mem_avail_mtx);
1121
/** Add reference to frame.
1123
* Find respective frame structure for supplied PFN and
1124
* increment frame reference count.
1126
* @param pfn Frame number of the frame to be freed.
1129
void frame_reference_add(pfn_t pfn)
1131
ipl_t ipl = interrupts_disable();
1132
spinlock_lock(&zones.lock);
1135
* First, find host frame zone for addr.
1137
size_t znum = find_zone(pfn, 1, NULL);
1139
ASSERT(znum != (size_t) -1);
1141
zones.info[znum].frames[pfn - zones.info[znum].base].refcount++;
1143
spinlock_unlock(&zones.lock);
1144
interrupts_restore(ipl);
1147
/** Mark given range unavailable in frame zones. */
1148
void frame_mark_unavailable(pfn_t start, size_t count)
1150
ipl_t ipl = interrupts_disable();
1151
spinlock_lock(&zones.lock);
1154
for (i = 0; i < count; i++) {
1155
size_t znum = find_zone(start + i, 1, 0);
1156
if (znum == (size_t) -1) /* PFN not found */
1159
zone_mark_unavailable(&zones.info[znum],
1160
start + i - zones.info[znum].base);
1163
spinlock_unlock(&zones.lock);
1164
interrupts_restore(ipl);
1167
/** Initialize physical memory management. */
1168
void frame_init(void)
1170
if (config.cpu_active == 1) {
1172
spinlock_initialize(&zones.lock, "zones.lock");
1173
mutex_initialize(&mem_avail_mtx, MUTEX_ACTIVE);
1174
condvar_initialize(&mem_avail_cv);
1177
/* Tell the architecture to create some memory */
1179
if (config.cpu_active == 1) {
1180
frame_mark_unavailable(ADDR2PFN(KA2PA(config.base)),
1181
SIZE2FRAMES(config.kernel_size));
1182
frame_mark_unavailable(ADDR2PFN(KA2PA(config.stack_base)),
1183
SIZE2FRAMES(config.stack_size));
1186
for (i = 0; i < init.cnt; i++) {
1187
pfn_t pfn = ADDR2PFN(KA2PA(init.tasks[i].addr));
1188
frame_mark_unavailable(pfn,
1189
SIZE2FRAMES(init.tasks[i].size));
1193
frame_mark_unavailable(ADDR2PFN(KA2PA(ballocs.base)),
1194
SIZE2FRAMES(ballocs.size));
1196
/* Black list first frame, as allocating NULL would
1197
* fail in some places
1199
frame_mark_unavailable(0, 1);
1203
/** Return total size of all zones. */
1204
uint64_t zone_total_size(void)
1206
ipl_t ipl = interrupts_disable();
1207
spinlock_lock(&zones.lock);
1211
for (i = 0; i < zones.count; i++)
1212
total += (uint64_t) FRAMES2SIZE(zones.info[i].count);
1214
spinlock_unlock(&zones.lock);
1215
interrupts_restore(ipl);
1220
/** Prints list of zones. */
1221
void zone_print_list(void)
1224
printf("# base address frames flags free frames busy frames\n");
1225
printf("-- ------------ ------------ -------- ------------ ------------\n");
1229
printf("# base address frames flags free frames busy frames\n");
1230
printf("-- -------------------- ------------ -------- ------------ ------------\n");
1234
* Because printing may require allocation of memory, we may not hold
1235
* the frame allocator locks when printing zone statistics. Therefore,
1236
* we simply gather the statistics under the protection of the locks and
1237
* print the statistics when the locks have been released.
1239
* When someone adds/removes zones while we are printing the statistics,
1240
* we may end up with inaccurate output (e.g. a zone being skipped from
1246
ipl_t ipl = interrupts_disable();
1247
spinlock_lock(&zones.lock);
1249
if (i >= zones.count) {
1250
spinlock_unlock(&zones.lock);
1251
interrupts_restore(ipl);
1255
uintptr_t base = PFN2ADDR(zones.info[i].base);
1256
size_t count = zones.info[i].count;
1257
zone_flags_t flags = zones.info[i].flags;
1258
size_t free_count = zones.info[i].free_count;
1259
size_t busy_count = zones.info[i].busy_count;
1261
spinlock_unlock(&zones.lock);
1262
interrupts_restore(ipl);
1264
bool available = zone_flags_available(flags);
1266
printf("%-2" PRIs, i);
1269
printf(" %10p", base);
1273
printf(" %18p", base);
1276
printf(" %12" PRIs " %c%c%c ", count,
1277
available ? 'A' : ' ',
1278
(flags & ZONE_RESERVED) ? 'R' : ' ',
1279
(flags & ZONE_FIRMWARE) ? 'F' : ' ');
1282
printf("%12" PRIs " %12" PRIs,
1283
free_count, busy_count);
1289
/** Prints zone details.
1291
* @param num Zone base address or zone number.
1294
void zone_print_one(size_t num)
1296
ipl_t ipl = interrupts_disable();
1297
spinlock_lock(&zones.lock);
1298
size_t znum = (size_t) -1;
1301
for (i = 0; i < zones.count; i++) {
1302
if ((i == num) || (PFN2ADDR(zones.info[i].base) == num)) {
1308
if (znum == (size_t) -1) {
1309
spinlock_unlock(&zones.lock);
1310
interrupts_restore(ipl);
1311
printf("Zone not found.\n");
1315
uintptr_t base = PFN2ADDR(zones.info[i].base);
1316
zone_flags_t flags = zones.info[i].flags;
1317
size_t count = zones.info[i].count;
1318
size_t free_count = zones.info[i].free_count;
1319
size_t busy_count = zones.info[i].busy_count;
1321
spinlock_unlock(&zones.lock);
1322
interrupts_restore(ipl);
1324
bool available = zone_flags_available(flags);
1326
printf("Zone number: %" PRIs "\n", znum);
1327
printf("Zone base address: %p\n", base);
1328
printf("Zone size: %" PRIs " frames (%" PRIs " KiB)\n", count,
1329
SIZE2KB(FRAMES2SIZE(count)));
1330
printf("Zone flags: %c%c%c\n",
1331
available ? 'A' : ' ',
1332
(flags & ZONE_RESERVED) ? 'R' : ' ',
1333
(flags & ZONE_FIRMWARE) ? 'F' : ' ');
1336
printf("Allocated space: %" PRIs " frames (%" PRIs " KiB)\n",
1337
busy_count, SIZE2KB(FRAMES2SIZE(busy_count)));
1338
printf("Available space: %" PRIs " frames (%" PRIs " KiB)\n",
1339
free_count, SIZE2KB(FRAMES2SIZE(free_count)));