← Back to branch summary

~contact-philipashmore/treedb/master

« back to all changes in this revision

Viewing changes to treedb/4-sombrero/treedb-malloc-storage-impl.c

  • Committer: Philip Ashmore
  • Date: 2012-04-01 18:32:33 UTC
  • Revision ID: git-v1:2739703677c401fc0e552603449a319403922a41
Version 1.3.0-01

* Added a long description for debian/control

* Added a v3c-comet man page

* Added a treedb-malloc-daemon man page

* Lots of fiddly changes to pass Debians "lintian" checks
  All the DEBIAN*_REQUIRES have changed to DEBIAN*_DEPENDS.
  Since they go into the Build-Depends and Depends fields of the
  debian/control(.in) file, this makes more sense.

* Refactoring
  The avl, list, array, varray, handle-array headers are c-templates.
  While c-templates provide ultimate control, they aren't the most
  intuitive to use.
  Now the c-template headers all have "template" in their name and they
  all have wrapper headers providing the "common use case".

  Take a look at the tests and comet for examples.

* Nested c-template wrappers wrinkle
  One of the side-effects of this refactoring exercise is that it's now
  impossible to use a wrapper from inside the same wrapper.

  Even though this can be done using the c-templates by themselves, the
  reduction in the number of lines of hand-written code is still a win.

  The way around this is to "sed" all the avl wrapper headers to produce
  versions with a "2" at the end of their names and macros, and use
  those in the AVL two-tree allocator.

  Note that this jumping through hoops is because of deficiencies in the
  design of the "C" preprocessor - "m4" and even "make" can define
  macros that define other macros easily.

* treedb-malloc library improvements
  Instead of allocating an insanely huge mamory map, a more modest one
  is created when memory is allocated, and the rest mmapped onto
  /dev/zero.
  The idea is to preserve the start address by changing the ratio of
  read+write pages to read-only pages so the total map is fixed.

* oprofile tests added for treedb-malloc daemon + tests
  FYI

Show diffs side-by-side

added added

removed removed

Lines of Context:
treedb/4-sombrero/treedb-malloc-storage-impl.c
 
1
/* Copyright (C) 2010  Philip Ashmore (contact@philipashmore.com)             */
 
2
/* License: LGPLv3.  See LICENSE.txt for the full license.                    */
 
3
 
 
4
/* THIS IS NOT A DISK BASED IMPLEMENTATION */
 
5
 
 
6
#include "treedb.h"
 
7
#include <memory.h>
 
8
#include <stdlib.h>
 
9
#include <stdio.h>
 
10
#include <errno.h>
 
11
#include <sys/mman.h>
 
12
#include <pthread.h>
 
13
 
 
14
#define TREEDB_SOMBRERO_ALLOC_OMIT_pack_unpack
 
15
#include "treedb-malloc-defines.h"
 
16
 
 
17
extern A16(runtime_t) A16(runtime_init);
 
18
extern A32(runtime_t) A32(runtime_init);
 
19
extern LG(runtime_t) LG(runtime_init);
 
20
extern treedb_malloc_allocator_runtime_t treedb_malloc_allocator_runtime_init;
 
21
 
 
22
treedb_allocator_runtime create_treedb_allocator(int free_emptied)
 
23
{
 
24
        treedb_allocator_runtime h =
 
25
                (treedb_allocator_runtime)malloc(sizeof(treedb_malloc_allocator_runtime_t));
 
26
        if(!h)
 
27
                return h;
 
28
        *h = treedb_malloc_allocator_runtime_init;
 
29
        h->free_emptied_heaps = free_emptied;
 
30
        return h;
 
31
}
 
32
TreedbFreeResult free_treedb_allocator(treedb_allocator_runtime h)
 
33
{
 
34
        if(!h)
 
35
                return eTreedbFreeBadHeap;
 
36
        /* The lists and the heap tree use the heap runtime wrappers to store their
 
37
        nodes so free the heaps and it all disappears. */
 
38
        treedb_malloc_heap_node_ref_ptr_t t;
 
39
        t.heap_node = (HEAP(node_ptr_t))(h->heap.m_head);
 
40
        TreedbFreeResult ret = eTreedbFreeSuccess, ret2;
 
41
        while(t.heap_node) {
 
42
                treedb_malloc_heap_node_ref_ptr_t t2;
 
43
                t2.heap_node = (HEAP(node_ptr_t))(t.heap_node->m_heap_next);
 
44
                switch(*t.type) {
 
45
                case eMallocHeapNodeTree16:
 
46
                        ret2 = A16(free_heap)(h, t.a16);
 
47
                        break;
 
48
                case eMallocHeapNodeTree32:
 
49
                        ret2 = A32(free_heap)(h, t.a32);
 
50
                        break;
 
51
                case eMallocHeapNodeLarge:
 
52
                        ret2 = LG(free_heap)(h, t.large);
 
53
                        break;
 
54
                default:
 
55
                        ret2 = eTreedbFreeBadHeap;
 
56
                        break;
 
57
                }
 
58
                t = t2;
 
59
                if(ret2 > ret)
 
60
                        ret = ret2;
 
61
        }
 
62
        free(h);
 
63
        return ret;
 
64
}
 
65
/******************************************************************************/
 
66
/* Allocator16                                                                */
 
67
/******************************************************************************/
 
68
A16(runtime_wrap_ptr_t) A16(new_heap)(treedb_allocator_runtime h)
 
69
{
 
70
        (void)h;
 
71
        A16(runtime_wrap_ptr_t) p = (A16(runtime_wrap_ptr_t))
 
72
                malloc(sizeof(A16(runtime_wrap_t)));
 
73
        if(!p)
 
74
                return p;
 
75
        p->type = eMallocHeapNodeTree16;
 
76
        p->runtime = A16(runtime_init);
 
77
        p->runtime.m_pBase = mmap(0, 64 * 1024
 
78
                        , PROT_READ | PROT_WRITE
 
79
                        , MAP_PRIVATE | MAP_ANONYMOUS
 
80
                        , 0
 
81
                        , 0);
 
82
        if(p->runtime.m_pBase == (char *)-1) {
 
83
                p->runtime.m_pBase = 0;
 
84
                free(p);
 
85
                return 0;
 
86
        }
 
87
        TreedbInitResult res = A16(init_memory)(& p->runtime, sizeof(A16(treedb_t)));
 
88
        if(res != eTreedbInitSuccess) {
 
89
                munmap((char *)(p->runtime.m_pBase), 64 * 1024);
 
90
                free(p);
 
91
                return 0;
 
92
        }
 
93
        /* Resize the free node so it reaches the end of this heap. */
 
94
        A16(AvlAllocator_node_ptr_t) node
 
95
                = (A16(AvlAllocator_node_ptr_t))(p->runtime.m_pBase
 
96
                        + p->runtime.m_pTreedb->m_allocator.m_free.m_head);
 
97
        /* It's the only free node so we don't have to remove it and re-insert it
 
98
        because of the key change. */
 
99
        node->m_uLength += 64 * 1024 - TREEDB_PAGE_SIZE;
 
100
        p->m_uSize = 64 * 1024;
 
101
        return p;
 
102
}
 
103
TreedbFreeResult A16(free_heap)(treedb_allocator_runtime h, A16(runtime_wrap_ptr_t) p)
 
104
{
 
105
        (void)h;
 
106
        TreedbFreeResult ret = eTreedbFreeSuccess;
 
107
        if(p->runtime.m_pBase)
 
108
                munmap((char *)(p->runtime.m_pBase), 64 * 1024);
 
109
        else
 
110
                ret = eTreedbFreeBadHeap;
 
111
        free(p);
 
112
        return ret;
 
113
}
 
114
A16(AvlAllocator_node_ptr_t) A16(alloc_node)
 
115
( A16(runtime_ptr_t) runtime
 
116
, size_t size
 
117
) {
 
118
        char * p = (char *)treedb_malloc_allocator16_malloc(runtime, size);
 
119
        return (A16(AvlAllocator_node_ptr_t))
 
120
                (p - V3C_ROUND_UP_T2(uintptr_t, sizeof(A16(AvlAllocator_node_t)), 2));
 
121
}
 
122
 
 
123
/******************************************************************************/
 
124
/* Allocator32                                                                */
 
125
/******************************************************************************/
 
126
A32(runtime_wrap_ptr_t) A32(new_heap)(treedb_allocator_runtime h)
 
127
{
 
128
        (void)h;
 
129
        A32(runtime_wrap_ptr_t) p = (A32(runtime_wrap_ptr_t))
 
130
                malloc(sizeof(A32(runtime_wrap_t)));
 
131
        if(!p)
 
132
                return p;
 
133
        p->type = eMallocHeapNodeTree32;
 
134
        p->runtime = A32(runtime_init);
 
135
        p->runtime.m_pBase = mmap(0, 4UL * 1024UL * 1024UL
 
136
                        , PROT_READ | PROT_WRITE
 
137
                        , MAP_PRIVATE | MAP_ANONYMOUS
 
138
                        , 0
 
139
                        , 0);
 
140
        if(p->runtime.m_pBase == (char *)-1) {
 
141
                switch(errno) {
 
142
                case EINVAL:
 
143
                        printf("A32(new_heap) : mmap failed : EINVAL.\n");
 
144
                        break;
 
145
                case EACCES:
 
146
                        printf("A32(new_heap) : mmap failed : EACCES.\n");
 
147
                        break;
 
148
                case ENOMEM:
 
149
                        printf("A32(new_heap) : mmap failed : ENOMEM.\n");
 
150
                        break;
 
151
                case ENODEV:
 
152
                        printf("A32(new_heap) : mmap failed : ENODEV.\n");
 
153
                        break;
 
154
                case ENOEXEC:
 
155
                        printf("A32(new_heap) : mmap failed : ENOEXEC.\n");
 
156
                        break;
 
157
                default:
 
158
                        printf("A32(new_heap) : mmap failed : <?>.\n");
 
159
                        break;
 
160
                }
 
161
                p->runtime.m_pBase = 0;
 
162
                free(p);
 
163
                return 0;
 
164
        }
 
165
        TreedbInitResult res = A32(init_memory)(& p->runtime, sizeof(A32(treedb_t)));
 
166
        if(res != eTreedbInitSuccess) {
 
167
                munmap((char *)(p->runtime.m_pBase), 4UL * 1024UL * 1024UL);
 
168
                free(p);
 
169
                return 0;
 
170
        }
 
171
        /* Resize the free node so it reaches the end of this heap. */
 
172
        A32(AvlAllocator_node_ptr_t) node
 
173
                = (A32(AvlAllocator_node_ptr_t))(p->runtime.m_pBase
 
174
                        + p->runtime.m_pTreedb->m_allocator.m_free.m_head);
 
175
        /* It's the only free node so we don't have to remove it and re-insert it
 
176
        because of the key change. */
 
177
        node->m_uLength
 
178
                        += 4UL * 1024UL * 1024UL - TREEDB_PAGE_SIZE;
 
179
        p->m_uSize = 4UL * 1024UL * 1024UL;
 
180
        return p;
 
181
}
 
182
TreedbFreeResult A32(free_heap)(treedb_allocator_runtime h, A32(runtime_wrap_ptr_t) p)
 
183
{
 
184
        (void)h;
 
185
        TreedbFreeResult ret = eTreedbFreeSuccess;
 
186
        if(p->runtime.m_pBase)
 
187
                munmap((char *)(p->runtime.m_pBase), 4UL * 1024UL * 1024UL);
 
188
        else
 
189
                ret = eTreedbFreeBadHeap;
 
190
        free(p);
 
191
        return ret;
 
192
}
 
193
A32(AvlAllocator_node_ptr_t) A32(alloc_node)
 
194
( A32(runtime_ptr_t) runtime
 
195
, size_t size
 
196
) {
 
197
        char * p = (char *)treedb_malloc_allocator32_malloc(runtime, size);
 
198
        return (A32(AvlAllocator_node_ptr_t))
 
199
                (p - V3C_ROUND_UP_T2(uintptr_t, sizeof(A32(AvlAllocator_node_t)), 4));
 
200
}
 
201
 
 
202
/******************************************************************************/
 
203
/* Large Allocation                                                           */
 
204
/******************************************************************************/
 
205
LG(runtime_wrap_ptr_t) LG(new_heap)(treedb_allocator_runtime h
 
206
, size_t size)
 
207
{
 
208
        (void)h;
 
209
        LG(runtime_wrap_ptr_t) p = (LG(runtime_wrap_ptr_t))
 
210
                malloc(sizeof(LG(runtime_wrap_t)));
 
211
        if(!p)
 
212
                return p;
 
213
        p->type = eMallocHeapNodeLarge;
 
214
        p->runtime = LG(runtime_init);
 
215
        p->runtime.m_pBase = mmap(0, size + sizeof(uint64_t)
 
216
                        , PROT_READ | PROT_WRITE
 
217
                        , MAP_PRIVATE | MAP_ANONYMOUS
 
218
                        , 0
 
219
                        , 0);
 
220
        if(p->runtime.m_pBase == (char *)-1) {
 
221
                p->runtime.m_pBase = 0;
 
222
                free(p);
 
223
                return 0;
 
224
        }
 
225
        /* This demo get's a bit scraggy here :) */
 
226
        p->runtime.m_pData->length = p->m_uSize = size;
 
227
        return p;
 
228
}
 
229
TreedbFreeResult LG(free_heap)(treedb_allocator_runtime h, LG(runtime_wrap_ptr_t) p)
 
230
{
 
231
        (void)h;
 
232
        TreedbFreeResult ret = eTreedbFreeSuccess;
 
233
        if(p->runtime.m_pBase)
 
234
                munmap((char *)(p->runtime.m_pBase), p->runtime.m_pData->length + sizeof(uint64_t));
 
235
        else
 
236
                ret = eTreedbFreeBadHeap;
 
237
        free(p);
 
238
        return ret;
 
239
}
 
240
void * LG(realloc)(treedb_allocator_runtime h, LG(runtime_wrap_ptr_t) p, size_t new_size)
 
241
{
 
242
        (void)h;
 
243
        char * q = mremap((char *)(p->runtime.m_pBase), p->runtime.m_pData->length
 
244
                , new_size + sizeof(uint64_t)
 
245
                , MREMAP_MAYMOVE);
 
246
        if(q == (char *)-1)
 
247
                return 0;
 
248
        p->runtime.m_pData->length = p->m_uSize = new_size;
 
249
        return q;
 
250
}