~lttng/urcu/trunk

« back to all changes in this revision

Viewing changes to src/urcu-bp.c

  • Committer: Mathieu Desnoyers
  • Author(s): Wang Jing
  • Date: 2023-09-06 13:20:24 UTC
  • Revision ID: git-v1:dc46a9c324ae94d89da41ea9a3f97503115df88e
Add LoongArch support

This commit completes LoongArch support.

LoongArch supports byte and short atomic operations,
and defines UATOMIC_HAS_ATOMIC_BYTE and UATOMIC_HAS_ATOMIC_SHORT.

Signed-off-by: Wang Jing <wangjing@loongson.cn>
Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Change-Id: I335e654939bfc90994275f2a4fad550c95f3eba4

Show diffs side-by-side

added added

removed removed

Lines of Context:
75
75
 
76
76
/* Sleep delay in ms */
77
77
#define RCU_SLEEP_DELAY_MS      10
78
 
#define INIT_READER_COUNT       8
 
78
#define INIT_NR_THREADS         8
 
79
#define ARENA_INIT_ALLOC                \
 
80
        sizeof(struct registry_chunk)   \
 
81
        + INIT_NR_THREADS * sizeof(struct urcu_bp_reader)
79
82
 
80
83
/*
81
84
 * Active attempts to check for reader Q.S. before calling sleep().
145
148
static CDS_LIST_HEAD(registry);
146
149
 
147
150
struct registry_chunk {
148
 
        size_t capacity;                /* capacity of this chunk (in elements) */
149
 
        size_t used;                    /* count of elements used */
 
151
        size_t data_len;                /* data length */
 
152
        size_t used;                    /* amount of data used */
150
153
        struct cds_list_head node;      /* chunk_list node */
151
 
        struct urcu_bp_reader readers[];
 
154
        char data[];
152
155
};
153
156
 
154
157
struct registry_arena {
198
201
        }
199
202
}
200
203
 
201
 
/* Get the size of a chunk's allocation from its capacity (an element count). */
202
 
static size_t chunk_allocation_size(size_t capacity)
203
 
{
204
 
        return (capacity * sizeof(struct urcu_bp_reader)) +
205
 
                sizeof(struct registry_chunk);
206
 
}
207
 
 
208
204
/*
209
205
 * Always called with rcu_registry lock held. Releases this lock between
210
206
 * iterations and grabs it again. Holds the lock when it returns.
379
375
void expand_arena(struct registry_arena *arena)
380
376
{
381
377
        struct registry_chunk *new_chunk, *last_chunk;
382
 
        size_t old_chunk_size_bytes, new_chunk_size_bytes, new_capacity;
 
378
        size_t old_chunk_len, new_chunk_len;
383
379
 
384
380
        /* No chunk. */
385
381
        if (cds_list_empty(&arena->chunk_list)) {
386
 
                new_chunk_size_bytes = chunk_allocation_size(INIT_READER_COUNT);
 
382
                urcu_posix_assert(ARENA_INIT_ALLOC >=
 
383
                        sizeof(struct registry_chunk)
 
384
                        + sizeof(struct rcu_reader));
 
385
                new_chunk_len = ARENA_INIT_ALLOC;
387
386
                new_chunk = (struct registry_chunk *) mmap(NULL,
388
 
                        new_chunk_size_bytes,
 
387
                        new_chunk_len,
389
388
                        PROT_READ | PROT_WRITE,
390
389
                        MAP_ANONYMOUS | MAP_PRIVATE,
391
390
                        -1, 0);
392
391
                if (new_chunk == MAP_FAILED)
393
392
                        abort();
394
 
                memset(new_chunk, 0, new_chunk_size_bytes);
395
 
                new_chunk->capacity = INIT_READER_COUNT;
 
393
                memset(new_chunk, 0, new_chunk_len);
 
394
                new_chunk->data_len =
 
395
                        new_chunk_len - sizeof(struct registry_chunk);
396
396
                cds_list_add_tail(&new_chunk->node, &arena->chunk_list);
397
397
                return;         /* We're done. */
398
398
        }
400
400
        /* Try expanding last chunk. */
401
401
        last_chunk = cds_list_entry(arena->chunk_list.prev,
402
402
                struct registry_chunk, node);
403
 
        old_chunk_size_bytes = chunk_allocation_size(last_chunk->capacity);
404
 
        new_capacity = last_chunk->capacity << 1;
405
 
        new_chunk_size_bytes = chunk_allocation_size(new_capacity);
 
403
        old_chunk_len =
 
404
                last_chunk->data_len + sizeof(struct registry_chunk);
 
405
        new_chunk_len = old_chunk_len << 1;
406
406
 
407
407
        /* Don't allow memory mapping to move, just expand. */
408
 
        new_chunk = mremap_wrapper(last_chunk, old_chunk_size_bytes,
409
 
                new_chunk_size_bytes, 0);
 
408
        new_chunk = mremap_wrapper(last_chunk, old_chunk_len,
 
409
                new_chunk_len, 0);
410
410
        if (new_chunk != MAP_FAILED) {
411
411
                /* Should not have moved. */
412
 
                assert(new_chunk == last_chunk);
413
 
                memset((char *) last_chunk + old_chunk_size_bytes, 0,
414
 
                        new_chunk_size_bytes - old_chunk_size_bytes);
415
 
                last_chunk->capacity = new_capacity;
 
412
                urcu_posix_assert(new_chunk == last_chunk);
 
413
                memset((char *) last_chunk + old_chunk_len, 0,
 
414
                        new_chunk_len - old_chunk_len);
 
415
                last_chunk->data_len =
 
416
                        new_chunk_len - sizeof(struct registry_chunk);
416
417
                return;         /* We're done. */
417
418
        }
418
419
 
419
420
        /* Remap did not succeed, we need to add a new chunk. */
420
421
        new_chunk = (struct registry_chunk *) mmap(NULL,
421
 
                new_chunk_size_bytes,
 
422
                new_chunk_len,
422
423
                PROT_READ | PROT_WRITE,
423
424
                MAP_ANONYMOUS | MAP_PRIVATE,
424
425
                -1, 0);
425
426
        if (new_chunk == MAP_FAILED)
426
427
                abort();
427
 
        memset(new_chunk, 0, new_chunk_size_bytes);
428
 
        new_chunk->capacity = new_capacity;
 
428
        memset(new_chunk, 0, new_chunk_len);
 
429
        new_chunk->data_len =
 
430
                new_chunk_len - sizeof(struct registry_chunk);
429
431
        cds_list_add_tail(&new_chunk->node, &arena->chunk_list);
430
432
}
431
433
 
433
435
struct rcu_reader *arena_alloc(struct registry_arena *arena)
434
436
{
435
437
        struct registry_chunk *chunk;
 
438
        struct rcu_reader *rcu_reader_reg;
436
439
        int expand_done = 0;    /* Only allow to expand once per alloc */
 
440
        size_t len = sizeof(struct rcu_reader);
437
441
 
438
442
retry:
439
443
        cds_list_for_each_entry(chunk, &arena->chunk_list, node) {
440
 
                size_t spot_idx;
441
 
 
442
 
                /* Skip fully used chunks. */
443
 
                if (chunk->used == chunk->capacity) {
 
444
                if (chunk->data_len - chunk->used < len)
444
445
                        continue;
445
 
                }
446
 
 
447
 
                /* Find a spot. */
448
 
                for (spot_idx = 0; spot_idx < chunk->capacity; spot_idx++) {
449
 
                        if (!chunk->readers[spot_idx].alloc) {
450
 
                                chunk->readers[spot_idx].alloc = 1;
451
 
                                chunk->used++;
452
 
                                return &chunk->readers[spot_idx];
 
446
                /* Find spot */
 
447
                for (rcu_reader_reg = (struct rcu_reader *) &chunk->data[0];
 
448
                                rcu_reader_reg < (struct rcu_reader *) &chunk->data[chunk->data_len];
 
449
                                rcu_reader_reg++) {
 
450
                        if (!rcu_reader_reg->alloc) {
 
451
                                rcu_reader_reg->alloc = 1;
 
452
                                chunk->used += len;
 
453
                                return rcu_reader_reg;
453
454
                        }
454
455
                }
455
456
        }
497
498
        cds_list_del(&rcu_reader_reg->node);
498
499
        rcu_reader_reg->tid = 0;
499
500
        rcu_reader_reg->alloc = 0;
500
 
        chunk->used--;
 
501
        chunk->used -= sizeof(struct rcu_reader);
501
502
}
502
503
 
503
504
static
506
507
        struct registry_chunk *chunk;
507
508
 
508
509
        cds_list_for_each_entry(chunk, &registry_arena.chunk_list, node) {
509
 
                if (rcu_reader_reg < (struct urcu_bp_reader *) &chunk->readers[0])
 
510
                if (rcu_reader_reg < (struct rcu_reader *) &chunk->data[0])
510
511
                        continue;
511
 
                if (rcu_reader_reg >= (struct urcu_bp_reader *) &chunk->readers[chunk->capacity])
 
512
                if (rcu_reader_reg >= (struct rcu_reader *) &chunk->data[chunk->data_len])
512
513
                        continue;
513
514
                return chunk;
514
515
        }
657
658
 
658
659
                cds_list_for_each_entry_safe(chunk, tmp,
659
660
                                &registry_arena.chunk_list, node) {
660
 
                        munmap((void *) chunk, chunk_allocation_size(chunk->capacity));
 
661
                        munmap((void *) chunk, chunk->data_len
 
662
                                        + sizeof(struct registry_chunk));
661
663
                }
662
664
                CDS_INIT_LIST_HEAD(&registry_arena.chunk_list);
663
665
                ret = pthread_key_delete(urcu_bp_key);
714
716
void urcu_bp_prune_registry(void)
715
717
{
716
718
        struct registry_chunk *chunk;
 
719
        struct urcu_bp_reader *rcu_reader_reg;
717
720
 
718
721
        cds_list_for_each_entry(chunk, &registry_arena.chunk_list, node) {
719
 
                size_t spot_idx;
720
 
 
721
 
                for (spot_idx = 0; spot_idx < chunk->capacity; spot_idx++) {
722
 
                        struct urcu_bp_reader *reader = &chunk->readers[spot_idx];
723
 
 
724
 
                        if (!reader->alloc)
725
 
                                continue;
726
 
                        if (reader->tid == pthread_self())
727
 
                                continue;
728
 
                        cleanup_thread(chunk, reader);
 
722
                for (rcu_reader_reg = (struct urcu_bp_reader *) &chunk->data[0];
 
723
                                rcu_reader_reg < (struct urcu_bp_reader *) &chunk->data[chunk->data_len];
 
724
                                rcu_reader_reg++) {
 
725
                        if (!rcu_reader_reg->alloc)
 
726
                                continue;
 
727
                        if (rcu_reader_reg->tid == pthread_self())
 
728
                                continue;
 
729
                        cleanup_thread(chunk, rcu_reader_reg);
729
730
                }
730
731
        }
731
732
}