2
* Copyright 2010 Tilera Corporation. All Rights Reserved.
4
* This program is free software; you can redistribute it and/or
5
* modify it under the terms of the GNU General Public License
6
* as published by the Free Software Foundation, version 2.
8
* This program is distributed in the hope that it will be useful, but
9
* WITHOUT ANY WARRANTY; without even the implied warranty of
10
* MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
11
* NON INFRINGEMENT. See the GNU General Public License for
15
#include <linux/cache.h>
16
#include <linux/delay.h>
17
#include <linux/uaccess.h>
18
#include <linux/module.h>
20
#include <linux/atomic.h>
21
#include <asm/futex.h>
22
#include <arch/chip.h>
24
/* See <asm/atomic_32.h> */
25
#if ATOMIC_LOCKS_FOUND_VIA_TABLE()
28
* A block of memory containing locks for atomic ops. Each instance of this
29
* struct will be homed on a different CPU.
31
struct atomic_locks_on_cpu {
32
int lock[ATOMIC_HASH_L2_SIZE];
33
} __attribute__((aligned(ATOMIC_HASH_L2_SIZE * 4)));
35
static DEFINE_PER_CPU(struct atomic_locks_on_cpu, atomic_lock_pool);
37
/* The locks we'll use until __init_atomic_per_cpu is called. */
38
static struct atomic_locks_on_cpu __initdata initial_atomic_locks;
40
/* Hash into this vector to get a pointer to lock for the given atomic. */
41
struct atomic_locks_on_cpu *atomic_lock_ptr[ATOMIC_HASH_L1_SIZE]
43
[0 ... ATOMIC_HASH_L1_SIZE-1] (&initial_atomic_locks)
46
#else /* ATOMIC_LOCKS_FOUND_VIA_TABLE() */
48
/* This page is remapped on startup to be hash-for-home. */
49
int atomic_locks[PAGE_SIZE / sizeof(int)] __page_aligned_bss;
51
#endif /* ATOMIC_LOCKS_FOUND_VIA_TABLE() */
53
static inline int *__atomic_hashed_lock(volatile void *v)
55
/* NOTE: this code must match "sys_cmpxchg" in kernel/intvec_32.S */
56
#if ATOMIC_LOCKS_FOUND_VIA_TABLE()
58
(unsigned long) v & ((PAGE_SIZE-1) & -sizeof(long long));
59
unsigned long n = __insn_crc32_32(0, i);
61
/* Grab high bits for L1 index. */
62
unsigned long l1_index = n >> ((sizeof(n) * 8) - ATOMIC_HASH_L1_SHIFT);
63
/* Grab low bits for L2 index. */
64
unsigned long l2_index = n & (ATOMIC_HASH_L2_SIZE - 1);
66
return &atomic_lock_ptr[l1_index]->lock[l2_index];
69
* Use bits [3, 3 + ATOMIC_HASH_SHIFT) as the lock index.
70
* Using mm works here because atomic_locks is page aligned.
72
unsigned long ptr = __insn_mm((unsigned long)v >> 1,
73
(unsigned long)atomic_locks,
74
2, (ATOMIC_HASH_SHIFT + 2) - 1);
80
/* Return whether the passed pointer is a valid atomic lock pointer. */
81
static int is_atomic_lock(int *p)
83
#if ATOMIC_LOCKS_FOUND_VIA_TABLE()
85
for (i = 0; i < ATOMIC_HASH_L1_SIZE; ++i) {
87
if (p >= &atomic_lock_ptr[i]->lock[0] &&
88
p < &atomic_lock_ptr[i]->lock[ATOMIC_HASH_L2_SIZE]) {
94
return p >= &atomic_locks[0] && p < &atomic_locks[ATOMIC_HASH_SIZE];
98
void __atomic_fault_unlock(int *irqlock_word)
100
BUG_ON(!is_atomic_lock(irqlock_word));
101
BUG_ON(*irqlock_word != 1);
105
#endif /* CONFIG_SMP */
107
static inline int *__atomic_setup(volatile void *v)
109
/* Issue a load to the target to bring it into cache. */
111
return __atomic_hashed_lock(v);
114
int _atomic_xchg(atomic_t *v, int n)
116
return __atomic_xchg(&v->counter, __atomic_setup(v), n).val;
118
EXPORT_SYMBOL(_atomic_xchg);
120
int _atomic_xchg_add(atomic_t *v, int i)
122
return __atomic_xchg_add(&v->counter, __atomic_setup(v), i).val;
124
EXPORT_SYMBOL(_atomic_xchg_add);
126
int _atomic_xchg_add_unless(atomic_t *v, int a, int u)
129
* Note: argument order is switched here since it is easier
130
* to use the first argument consistently as the "old value"
131
* in the assembly, as is done for _atomic_cmpxchg().
133
return __atomic_xchg_add_unless(&v->counter, __atomic_setup(v), u, a)
136
EXPORT_SYMBOL(_atomic_xchg_add_unless);
138
int _atomic_cmpxchg(atomic_t *v, int o, int n)
140
return __atomic_cmpxchg(&v->counter, __atomic_setup(v), o, n).val;
142
EXPORT_SYMBOL(_atomic_cmpxchg);
144
unsigned long _atomic_or(volatile unsigned long *p, unsigned long mask)
146
return __atomic_or((int *)p, __atomic_setup(p), mask).val;
148
EXPORT_SYMBOL(_atomic_or);
150
unsigned long _atomic_andn(volatile unsigned long *p, unsigned long mask)
152
return __atomic_andn((int *)p, __atomic_setup(p), mask).val;
154
EXPORT_SYMBOL(_atomic_andn);
156
unsigned long _atomic_xor(volatile unsigned long *p, unsigned long mask)
158
return __atomic_xor((int *)p, __atomic_setup(p), mask).val;
160
EXPORT_SYMBOL(_atomic_xor);
163
u64 _atomic64_xchg(atomic64_t *v, u64 n)
165
return __atomic64_xchg(&v->counter, __atomic_setup(v), n);
167
EXPORT_SYMBOL(_atomic64_xchg);
169
u64 _atomic64_xchg_add(atomic64_t *v, u64 i)
171
return __atomic64_xchg_add(&v->counter, __atomic_setup(v), i);
173
EXPORT_SYMBOL(_atomic64_xchg_add);
175
u64 _atomic64_xchg_add_unless(atomic64_t *v, u64 a, u64 u)
178
* Note: argument order is switched here since it is easier
179
* to use the first argument consistently as the "old value"
180
* in the assembly, as is done for _atomic_cmpxchg().
182
return __atomic64_xchg_add_unless(&v->counter, __atomic_setup(v),
185
EXPORT_SYMBOL(_atomic64_xchg_add_unless);
187
u64 _atomic64_cmpxchg(atomic64_t *v, u64 o, u64 n)
189
return __atomic64_cmpxchg(&v->counter, __atomic_setup(v), o, n);
191
EXPORT_SYMBOL(_atomic64_cmpxchg);
194
static inline int *__futex_setup(int __user *v)
197
* Issue a prefetch to the counter to bring it into cache.
198
* As for __atomic_setup, but we can't do a read into the L1
199
* since it might fault; instead we do a prefetch into the L2.
202
return __atomic_hashed_lock((int __force *)v);
205
struct __get_user futex_set(u32 __user *v, int i)
207
return __atomic_xchg((int __force *)v, __futex_setup(v), i);
210
struct __get_user futex_add(u32 __user *v, int n)
212
return __atomic_xchg_add((int __force *)v, __futex_setup(v), n);
215
struct __get_user futex_or(u32 __user *v, int n)
217
return __atomic_or((int __force *)v, __futex_setup(v), n);
220
struct __get_user futex_andn(u32 __user *v, int n)
222
return __atomic_andn((int __force *)v, __futex_setup(v), n);
225
struct __get_user futex_xor(u32 __user *v, int n)
227
return __atomic_xor((int __force *)v, __futex_setup(v), n);
230
struct __get_user futex_cmpxchg(u32 __user *v, int o, int n)
232
return __atomic_cmpxchg((int __force *)v, __futex_setup(v), o, n);
236
* If any of the atomic or futex routines hit a bad address (not in
237
* the page tables at kernel PL) this routine is called. The futex
238
* routines are never used on kernel space, and the normal atomics and
239
* bitops are never used on user space. So a fault on kernel space
240
* must be fatal, but a fault on userspace is a futex fault and we
241
* need to return -EFAULT. Note that the context this routine is
242
* invoked in is the context of the "_atomic_xxx()" routines called
243
* by the functions in this file.
245
struct __get_user __atomic_bad_address(int __user *addr)
247
if (unlikely(!access_ok(VERIFY_WRITE, addr, sizeof(int))))
248
panic("Bad address used for kernel atomic op: %p\n", addr);
249
return (struct __get_user) { .err = -EFAULT };
253
#if CHIP_HAS_CBOX_HOME_MAP()
254
static int __init noatomichash(char *str)
256
pr_warning("noatomichash is deprecated.\n");
259
__setup("noatomichash", noatomichash);
262
void __init __init_atomic_per_cpu(void)
264
#if ATOMIC_LOCKS_FOUND_VIA_TABLE()
270
* Before this is called from setup, we just have one lock for
271
* all atomic objects/operations. Here we replace the
272
* elements of atomic_lock_ptr so that they point at per_cpu
273
* integers. This seemingly over-complex approach stems from
274
* the fact that DEFINE_PER_CPU defines an entry for each cpu
275
* in the grid, not each cpu from 0..ATOMIC_HASH_SIZE-1. But
276
* for efficient hashing of atomics to their locks we want a
277
* compile time constant power of 2 for the size of this
278
* table, so we use ATOMIC_HASH_SIZE.
280
* Here we populate atomic_lock_ptr from the per cpu
281
* atomic_lock_pool, interspersing by actual cpu so that
282
* subsequent elements are homed on consecutive cpus.
285
actual_cpu = cpumask_first(cpu_possible_mask);
287
for (i = 0; i < ATOMIC_HASH_L1_SIZE; ++i) {
289
* Preincrement to slightly bias against using cpu 0,
290
* which has plenty of stuff homed on it already.
292
actual_cpu = cpumask_next(actual_cpu, cpu_possible_mask);
293
if (actual_cpu >= nr_cpu_ids)
294
actual_cpu = cpumask_first(cpu_possible_mask);
296
atomic_lock_ptr[i] = &per_cpu(atomic_lock_pool, actual_cpu);
299
#else /* ATOMIC_LOCKS_FOUND_VIA_TABLE() */
301
/* Validate power-of-two and "bigger than cpus" assumption */
302
BUILD_BUG_ON(ATOMIC_HASH_SIZE & (ATOMIC_HASH_SIZE-1));
303
BUG_ON(ATOMIC_HASH_SIZE < nr_cpu_ids);
306
* On TILEPro we prefer to use a single hash-for-home
307
* page, since this means atomic operations are less
308
* likely to encounter a TLB fault and thus should
309
* in general perform faster. You may wish to disable
310
* this in situations where few hash-for-home tiles
313
BUG_ON((unsigned long)atomic_locks % PAGE_SIZE != 0);
315
/* The locks must all fit on one page. */
316
BUILD_BUG_ON(ATOMIC_HASH_SIZE * sizeof(int) > PAGE_SIZE);
319
* We use the page offset of the atomic value's address as
320
* an index into atomic_locks, excluding the low 3 bits.
321
* That should not produce more indices than ATOMIC_HASH_SIZE.
323
BUILD_BUG_ON((PAGE_SIZE >> 3) > ATOMIC_HASH_SIZE);
325
#endif /* ATOMIC_LOCKS_FOUND_VIA_TABLE() */
327
/* The futex code makes this assumption, so we validate it here. */
328
BUILD_BUG_ON(sizeof(atomic_t) != sizeof(int));