1
/******************************************************************************/
2
#ifdef JEMALLOC_H_TYPES
4
/* Maximum bitmap bit count is 2^LG_BITMAP_MAXBITS. */
5
#define LG_BITMAP_MAXBITS LG_RUN_MAXREGS
7
typedef struct bitmap_level_s bitmap_level_t;
8
typedef struct bitmap_info_s bitmap_info_t;
9
typedef unsigned long bitmap_t;
10
#define LG_SIZEOF_BITMAP LG_SIZEOF_LONG
12
/* Number of bits per group. */
13
#define LG_BITMAP_GROUP_NBITS (LG_SIZEOF_BITMAP + 3)
14
#define BITMAP_GROUP_NBITS (ZU(1) << LG_BITMAP_GROUP_NBITS)
15
#define BITMAP_GROUP_NBITS_MASK (BITMAP_GROUP_NBITS-1)
17
/* Maximum number of levels possible. */
18
#define BITMAP_MAX_LEVELS \
19
(LG_BITMAP_MAXBITS / LG_SIZEOF_BITMAP) \
20
+ !!(LG_BITMAP_MAXBITS % LG_SIZEOF_BITMAP)
22
#endif /* JEMALLOC_H_TYPES */
23
/******************************************************************************/
24
#ifdef JEMALLOC_H_STRUCTS
26
struct bitmap_level_s {
27
/* Offset of this level's groups within the array of groups. */
31
struct bitmap_info_s {
32
/* Logical number of bits in bitmap (stored at bottom level). */
35
/* Number of levels necessary for nbits. */
39
* Only the first (nlevels+1) elements are used, and levels are ordered
40
* bottom to top (e.g. the bottom level is stored in levels[0]).
42
bitmap_level_t levels[BITMAP_MAX_LEVELS+1];
45
#endif /* JEMALLOC_H_STRUCTS */
46
/******************************************************************************/
47
#ifdef JEMALLOC_H_EXTERNS
49
void bitmap_info_init(bitmap_info_t *binfo, size_t nbits);
50
size_t bitmap_info_ngroups(const bitmap_info_t *binfo);
51
size_t bitmap_size(size_t nbits);
52
void bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo);
54
#endif /* JEMALLOC_H_EXTERNS */
55
/******************************************************************************/
56
#ifdef JEMALLOC_H_INLINES
58
#ifndef JEMALLOC_ENABLE_INLINE
59
bool bitmap_full(bitmap_t *bitmap, const bitmap_info_t *binfo);
60
bool bitmap_get(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit);
61
void bitmap_set(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit);
62
size_t bitmap_sfu(bitmap_t *bitmap, const bitmap_info_t *binfo);
63
void bitmap_unset(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit);
66
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_BITMAP_C_))
68
bitmap_full(bitmap_t *bitmap, const bitmap_info_t *binfo)
70
unsigned rgoff = binfo->levels[binfo->nlevels].group_offset - 1;
71
bitmap_t rg = bitmap[rgoff];
72
/* The bitmap is full iff the root group is 0. */
77
bitmap_get(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit)
82
assert(bit < binfo->nbits);
83
goff = bit >> LG_BITMAP_GROUP_NBITS;
85
return (!(g & (1LU << (bit & BITMAP_GROUP_NBITS_MASK))));
89
bitmap_set(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit)
95
assert(bit < binfo->nbits);
96
assert(bitmap_get(bitmap, binfo, bit) == false);
97
goff = bit >> LG_BITMAP_GROUP_NBITS;
100
assert(g & (1LU << (bit & BITMAP_GROUP_NBITS_MASK)));
101
g ^= 1LU << (bit & BITMAP_GROUP_NBITS_MASK);
103
assert(bitmap_get(bitmap, binfo, bit));
104
/* Propagate group state transitions up the tree. */
107
for (i = 1; i < binfo->nlevels; i++) {
109
goff = bit >> LG_BITMAP_GROUP_NBITS;
110
gp = &bitmap[binfo->levels[i].group_offset + goff];
112
assert(g & (1LU << (bit & BITMAP_GROUP_NBITS_MASK)));
113
g ^= 1LU << (bit & BITMAP_GROUP_NBITS_MASK);
121
/* sfu: set first unset. */
122
JEMALLOC_INLINE size_t
123
bitmap_sfu(bitmap_t *bitmap, const bitmap_info_t *binfo)
129
assert(bitmap_full(bitmap, binfo) == false);
131
i = binfo->nlevels - 1;
132
g = bitmap[binfo->levels[i].group_offset];
136
g = bitmap[binfo->levels[i].group_offset + bit];
137
bit = (bit << LG_BITMAP_GROUP_NBITS) + (ffsl(g) - 1);
140
bitmap_set(bitmap, binfo, bit);
145
bitmap_unset(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit)
152
assert(bit < binfo->nbits);
153
assert(bitmap_get(bitmap, binfo, bit));
154
goff = bit >> LG_BITMAP_GROUP_NBITS;
157
propagate = (g == 0);
158
assert((g & (1LU << (bit & BITMAP_GROUP_NBITS_MASK))) == 0);
159
g ^= 1LU << (bit & BITMAP_GROUP_NBITS_MASK);
161
assert(bitmap_get(bitmap, binfo, bit) == false);
162
/* Propagate group state transitions up the tree. */
165
for (i = 1; i < binfo->nlevels; i++) {
167
goff = bit >> LG_BITMAP_GROUP_NBITS;
168
gp = &bitmap[binfo->levels[i].group_offset + goff];
170
propagate = (g == 0);
171
assert((g & (1LU << (bit & BITMAP_GROUP_NBITS_MASK)))
173
g ^= 1LU << (bit & BITMAP_GROUP_NBITS_MASK);
175
if (propagate == false)
183
#endif /* JEMALLOC_H_INLINES */
184
/******************************************************************************/