← Back to branch summary

~ubuntu-branches/ubuntu/precise/linux-ti-omap4/precise

« back to all changes in this revision

Viewing changes to kernel/jump_label.c

  • Committer: Bazaar Package Importer
  • Author(s): Paolo Pisati
  • Date: 2011-06-29 15:23:51 UTC
  • mfrom: (26.1.1 natty-proposed)
  • Revision ID: james.westby@ubuntu.com-20110629152351-xs96tm303d95rpbk
Tags: 3.0.0-1200.2
* Rebased against 3.0.0-6.7
* BSP from TI based on 3.0.0

Show diffs side-by-side

added added

removed removed

Lines of Context:
kernel/jump_label.c
2
2
 * jump label support
3
3
 *
4
4
 * Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
 
5
 * Copyright (C) 2011 Peter Zijlstra <pzijlstr@redhat.com>
5
6
 *
6
7
 */
7
 
#include <linux/jump_label.h>
8
8
#include <linux/memory.h>
9
9
#include <linux/uaccess.h>
10
10
#include <linux/module.h>
11
11
#include <linux/list.h>
12
 
#include <linux/jhash.h>
13
12
#include <linux/slab.h>
14
13
#include <linux/sort.h>
15
14
#include <linux/err.h>
 
15
#include <linux/jump_label.h>
16
16
 
17
17
#ifdef HAVE_JUMP_LABEL
18
18
 
19
 
#define JUMP_LABEL_HASH_BITS 6
20
 
#define JUMP_LABEL_TABLE_SIZE (1 << JUMP_LABEL_HASH_BITS)
21
 
static struct hlist_head jump_label_table[JUMP_LABEL_TABLE_SIZE];
22
 
 
23
19
/* mutex to protect coming/going of the the jump_label table */
24
20
static DEFINE_MUTEX(jump_label_mutex);
25
21
 
26
 
struct jump_label_entry {
27
 
        struct hlist_node hlist;
28
 
        struct jump_entry *table;
29
 
        int nr_entries;
30
 
        /* hang modules off here */
31
 
        struct hlist_head modules;
32
 
        unsigned long key;
33
 
};
34
 
 
35
 
struct jump_label_module_entry {
36
 
        struct hlist_node hlist;
37
 
        struct jump_entry *table;
38
 
        int nr_entries;
39
 
        struct module *mod;
40
 
};
41
 
 
42
22
void jump_label_lock(void)
43
23
{
44
24
        mutex_lock(&jump_label_mutex);
49
29
        mutex_unlock(&jump_label_mutex);
50
30
}
51
31
 
 
32
bool jump_label_enabled(struct jump_label_key *key)
 
33
{
 
34
        return !!atomic_read(&key->enabled);
 
35
}
 
36
 
52
37
static int jump_label_cmp(const void *a, const void *b)
53
38
{
54
39
        const struct jump_entry *jea = a;
64
49
}
65
50
 
66
51
static void
67
 
sort_jump_label_entries(struct jump_entry *start, struct jump_entry *stop)
 
52
jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop)
68
53
{
69
54
        unsigned long size;
70
55
 
73
58
        sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL);
74
59
}
75
60
 
76
 
static struct jump_label_entry *get_jump_label_entry(jump_label_t key)
77
 
{
78
 
        struct hlist_head *head;
79
 
        struct hlist_node *node;
80
 
        struct jump_label_entry *e;
81
 
        u32 hash = jhash((void *)&key, sizeof(jump_label_t), 0);
82
 
 
83
 
        head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
84
 
        hlist_for_each_entry(e, node, head, hlist) {
85
 
                if (key == e->key)
86
 
                        return e;
87
 
        }
88
 
        return NULL;
89
 
}
90
 
 
91
 
static struct jump_label_entry *
92
 
add_jump_label_entry(jump_label_t key, int nr_entries, struct jump_entry *table)
93
 
{
94
 
        struct hlist_head *head;
95
 
        struct jump_label_entry *e;
96
 
        u32 hash;
97
 
 
98
 
        e = get_jump_label_entry(key);
99
 
        if (e)
100
 
                return ERR_PTR(-EEXIST);
101
 
 
102
 
        e = kmalloc(sizeof(struct jump_label_entry), GFP_KERNEL);
103
 
        if (!e)
104
 
                return ERR_PTR(-ENOMEM);
105
 
 
106
 
        hash = jhash((void *)&key, sizeof(jump_label_t), 0);
107
 
        head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
108
 
        e->key = key;
109
 
        e->table = table;
110
 
        e->nr_entries = nr_entries;
111
 
        INIT_HLIST_HEAD(&(e->modules));
112
 
        hlist_add_head(&e->hlist, head);
113
 
        return e;
114
 
}
115
 
 
116
 
static int
117
 
build_jump_label_hashtable(struct jump_entry *start, struct jump_entry *stop)
118
 
{
119
 
        struct jump_entry *iter, *iter_begin;
120
 
        struct jump_label_entry *entry;
121
 
        int count;
122
 
 
123
 
        sort_jump_label_entries(start, stop);
124
 
        iter = start;
125
 
        while (iter < stop) {
126
 
                entry = get_jump_label_entry(iter->key);
127
 
                if (!entry) {
128
 
                        iter_begin = iter;
129
 
                        count = 0;
130
 
                        while ((iter < stop) &&
131
 
                                (iter->key == iter_begin->key)) {
132
 
                                iter++;
133
 
                                count++;
134
 
                        }
135
 
                        entry = add_jump_label_entry(iter_begin->key,
136
 
                                                        count, iter_begin);
137
 
                        if (IS_ERR(entry))
138
 
                                return PTR_ERR(entry);
139
 
                 } else {
140
 
                        WARN_ONCE(1, KERN_ERR "build_jump_hashtable: unexpected entry!\n");
141
 
                        return -1;
142
 
                }
143
 
        }
144
 
        return 0;
145
 
}
146
 
 
147
 
/***
148
 
 * jump_label_update - update jump label text
149
 
 * @key -  key value associated with a a jump label
150
 
 * @type - enum set to JUMP_LABEL_ENABLE or JUMP_LABEL_DISABLE
151
 
 *
152
 
 * Will enable/disable the jump for jump label @key, depending on the
153
 
 * value of @type.
154
 
 *
155
 
 */
156
 
 
157
 
void jump_label_update(unsigned long key, enum jump_label_type type)
158
 
{
159
 
        struct jump_entry *iter;
160
 
        struct jump_label_entry *entry;
161
 
        struct hlist_node *module_node;
162
 
        struct jump_label_module_entry *e_module;
163
 
        int count;
 
61
static void jump_label_update(struct jump_label_key *key, int enable);
 
62
 
 
63
void jump_label_inc(struct jump_label_key *key)
 
64
{
 
65
        if (atomic_inc_not_zero(&key->enabled))
 
66
                return;
164
67
 
165
68
        jump_label_lock();
166
 
        entry = get_jump_label_entry((jump_label_t)key);
167
 
        if (entry) {
168
 
                count = entry->nr_entries;
169
 
                iter = entry->table;
170
 
                while (count--) {
171
 
                        if (kernel_text_address(iter->code))
172
 
                                arch_jump_label_transform(iter, type);
173
 
                        iter++;
174
 
                }
175
 
                /* eanble/disable jump labels in modules */
176
 
                hlist_for_each_entry(e_module, module_node, &(entry->modules),
177
 
                                                        hlist) {
178
 
                        count = e_module->nr_entries;
179
 
                        iter = e_module->table;
180
 
                        while (count--) {
181
 
                                if (iter->key &&
182
 
                                                kernel_text_address(iter->code))
183
 
                                        arch_jump_label_transform(iter, type);
184
 
                                iter++;
185
 
                        }
186
 
                }
187
 
        }
 
69
        if (atomic_add_return(1, &key->enabled) == 1)
 
70
                jump_label_update(key, JUMP_LABEL_ENABLE);
 
71
        jump_label_unlock();
 
72
}
 
73
 
 
74
void jump_label_dec(struct jump_label_key *key)
 
75
{
 
76
        if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex))
 
77
                return;
 
78
 
 
79
        jump_label_update(key, JUMP_LABEL_DISABLE);
188
80
        jump_label_unlock();
189
81
}
190
82
 
197
89
        return 0;
198
90
}
199
91
 
200
 
#ifdef CONFIG_MODULES
201
 
 
202
 
static int module_conflict(void *start, void *end)
203
 
{
204
 
        struct hlist_head *head;
205
 
        struct hlist_node *node, *node_next, *module_node, *module_node_next;
206
 
        struct jump_label_entry *e;
207
 
        struct jump_label_module_entry *e_module;
208
 
        struct jump_entry *iter;
209
 
        int i, count;
210
 
        int conflict = 0;
211
 
 
212
 
        for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {
213
 
                head = &jump_label_table[i];
214
 
                hlist_for_each_entry_safe(e, node, node_next, head, hlist) {
215
 
                        hlist_for_each_entry_safe(e_module, module_node,
216
 
                                                        module_node_next,
217
 
                                                        &(e->modules), hlist) {
218
 
                                count = e_module->nr_entries;
219
 
                                iter = e_module->table;
220
 
                                while (count--) {
221
 
                                        if (addr_conflict(iter, start, end)) {
222
 
                                                conflict = 1;
223
 
                                                goto out;
224
 
                                        }
225
 
                                        iter++;
226
 
                                }
227
 
                        }
228
 
                }
229
 
        }
230
 
out:
231
 
        return conflict;
232
 
}
233
 
 
234
 
#endif
235
 
 
236
 
/***
237
 
 * jump_label_text_reserved - check if addr range is reserved
238
 
 * @start: start text addr
239
 
 * @end: end text addr
240
 
 *
241
 
 * checks if the text addr located between @start and @end
242
 
 * overlaps with any of the jump label patch addresses. Code
243
 
 * that wants to modify kernel text should first verify that
244
 
 * it does not overlap with any of the jump label addresses.
245
 
 * Caller must hold jump_label_mutex.
246
 
 *
247
 
 * returns 1 if there is an overlap, 0 otherwise
248
 
 */
249
 
int jump_label_text_reserved(void *start, void *end)
250
 
{
251
 
        struct jump_entry *iter;
252
 
        struct jump_entry *iter_start = __start___jump_table;
253
 
        struct jump_entry *iter_stop = __start___jump_table;
254
 
        int conflict = 0;
 
92
static int __jump_label_text_reserved(struct jump_entry *iter_start,
 
93
                struct jump_entry *iter_stop, void *start, void *end)
 
94
{
 
95
        struct jump_entry *iter;
255
96
 
256
97
        iter = iter_start;
257
98
        while (iter < iter_stop) {
258
 
                if (addr_conflict(iter, start, end)) {
259
 
                        conflict = 1;
260
 
                        goto out;
261
 
                }
 
99
                if (addr_conflict(iter, start, end))
 
100
                        return 1;
262
101
                iter++;
263
102
        }
264
103
 
265
 
        /* now check modules */
266
 
#ifdef CONFIG_MODULES
267
 
        conflict = module_conflict(start, end);
268
 
#endif
269
 
out:
270
 
        return conflict;
 
104
        return 0;
 
105
}
 
106
 
 
107
static void __jump_label_update(struct jump_label_key *key,
 
108
                                struct jump_entry *entry,
 
109
                                struct jump_entry *stop, int enable)
 
110
{
 
111
        for (; (entry < stop) &&
 
112
              (entry->key == (jump_label_t)(unsigned long)key);
 
113
              entry++) {
 
114
                /*
 
115
                 * entry->code set to 0 invalidates module init text sections
 
116
                 * kernel_text_address() verifies we are not in core kernel
 
117
                 * init code, see jump_label_invalidate_module_init().
 
118
                 */
 
119
                if (entry->code && kernel_text_address(entry->code))
 
120
                        arch_jump_label_transform(entry, enable);
 
121
        }
271
122
}
272
123
 
273
124
/*
277
128
{
278
129
}
279
130
 
280
 
static __init int init_jump_label(void)
 
131
static __init int jump_label_init(void)
281
132
{
282
 
        int ret;
283
133
        struct jump_entry *iter_start = __start___jump_table;
284
134
        struct jump_entry *iter_stop = __stop___jump_table;
 
135
        struct jump_label_key *key = NULL;
285
136
        struct jump_entry *iter;
286
137
 
287
138
        jump_label_lock();
288
 
        ret = build_jump_label_hashtable(__start___jump_table,
289
 
                                         __stop___jump_table);
290
 
        iter = iter_start;
291
 
        while (iter < iter_stop) {
 
139
        jump_label_sort_entries(iter_start, iter_stop);
 
140
 
 
141
        for (iter = iter_start; iter < iter_stop; iter++) {
292
142
                arch_jump_label_text_poke_early(iter->code);
293
 
                iter++;
 
143
                if (iter->key == (jump_label_t)(unsigned long)key)
 
144
                        continue;
 
145
 
 
146
                key = (struct jump_label_key *)(unsigned long)iter->key;
 
147
                atomic_set(&key->enabled, 0);
 
148
                key->entries = iter;
 
149
#ifdef CONFIG_MODULES
 
150
                key->next = NULL;
 
151
#endif
294
152
        }
295
153
        jump_label_unlock();
296
 
        return ret;
 
154
 
 
155
        return 0;
297
156
}
298
 
early_initcall(init_jump_label);
 
157
early_initcall(jump_label_init);
299
158
 
300
159
#ifdef CONFIG_MODULES
301
160
 
302
 
static struct jump_label_module_entry *
303
 
add_jump_label_module_entry(struct jump_label_entry *entry,
304
 
                            struct jump_entry *iter_begin,
305
 
                            int count, struct module *mod)
306
 
{
307
 
        struct jump_label_module_entry *e;
308
 
 
309
 
        e = kmalloc(sizeof(struct jump_label_module_entry), GFP_KERNEL);
310
 
        if (!e)
311
 
                return ERR_PTR(-ENOMEM);
312
 
        e->mod = mod;
313
 
        e->nr_entries = count;
314
 
        e->table = iter_begin;
315
 
        hlist_add_head(&e->hlist, &entry->modules);
316
 
        return e;
317
 
}
318
 
 
319
 
static int add_jump_label_module(struct module *mod)
320
 
{
321
 
        struct jump_entry *iter, *iter_begin;
322
 
        struct jump_label_entry *entry;
323
 
        struct jump_label_module_entry *module_entry;
324
 
        int count;
325
 
 
326
 
        /* if the module doesn't have jump label entries, just return */
327
 
        if (!mod->num_jump_entries)
328
 
                return 0;
329
 
 
330
 
        sort_jump_label_entries(mod->jump_entries,
331
 
                                mod->jump_entries + mod->num_jump_entries);
332
 
        iter = mod->jump_entries;
333
 
        while (iter < mod->jump_entries + mod->num_jump_entries) {
334
 
                entry = get_jump_label_entry(iter->key);
335
 
                iter_begin = iter;
336
 
                count = 0;
337
 
                while ((iter < mod->jump_entries + mod->num_jump_entries) &&
338
 
                        (iter->key == iter_begin->key)) {
339
 
                                iter++;
340
 
                                count++;
341
 
                }
342
 
                if (!entry) {
343
 
                        entry = add_jump_label_entry(iter_begin->key, 0, NULL);
344
 
                        if (IS_ERR(entry))
345
 
                                return PTR_ERR(entry);
346
 
                }
347
 
                module_entry = add_jump_label_module_entry(entry, iter_begin,
348
 
                                                           count, mod);
349
 
                if (IS_ERR(module_entry))
350
 
                        return PTR_ERR(module_entry);
351
 
        }
 
161
struct jump_label_mod {
 
162
        struct jump_label_mod *next;
 
163
        struct jump_entry *entries;
 
164
        struct module *mod;
 
165
};
 
166
 
 
167
static int __jump_label_mod_text_reserved(void *start, void *end)
 
168
{
 
169
        struct module *mod;
 
170
 
 
171
        mod = __module_text_address((unsigned long)start);
 
172
        if (!mod)
 
173
                return 0;
 
174
 
 
175
        WARN_ON_ONCE(__module_text_address((unsigned long)end) != mod);
 
176
 
 
177
        return __jump_label_text_reserved(mod->jump_entries,
 
178
                                mod->jump_entries + mod->num_jump_entries,
 
179
                                start, end);
 
180
}
 
181
 
 
182
static void __jump_label_mod_update(struct jump_label_key *key, int enable)
 
183
{
 
184
        struct jump_label_mod *mod = key->next;
 
185
 
 
186
        while (mod) {
 
187
                struct module *m = mod->mod;
 
188
 
 
189
                __jump_label_update(key, mod->entries,
 
190
                                    m->jump_entries + m->num_jump_entries,
 
191
                                    enable);
 
192
                mod = mod->next;
 
193
        }
 
194
}
 
195
 
 
196
/***
 
197
 * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop()
 
198
 * @mod: module to patch
 
199
 *
 
200
 * Allow for run-time selection of the optimal nops. Before the module
 
201
 * loads patch these with arch_get_jump_label_nop(), which is specified by
 
202
 * the arch specific jump label code.
 
203
 */
 
204
void jump_label_apply_nops(struct module *mod)
 
205
{
 
206
        struct jump_entry *iter_start = mod->jump_entries;
 
207
        struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
 
208
        struct jump_entry *iter;
 
209
 
 
210
        /* if the module doesn't have jump label entries, just return */
 
211
        if (iter_start == iter_stop)
 
212
                return;
 
213
 
 
214
        for (iter = iter_start; iter < iter_stop; iter++)
 
215
                arch_jump_label_text_poke_early(iter->code);
 
216
}
 
217
 
 
218
static int jump_label_add_module(struct module *mod)
 
219
{
 
220
        struct jump_entry *iter_start = mod->jump_entries;
 
221
        struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
 
222
        struct jump_entry *iter;
 
223
        struct jump_label_key *key = NULL;
 
224
        struct jump_label_mod *jlm;
 
225
 
 
226
        /* if the module doesn't have jump label entries, just return */
 
227
        if (iter_start == iter_stop)
 
228
                return 0;
 
229
 
 
230
        jump_label_sort_entries(iter_start, iter_stop);
 
231
 
 
232
        for (iter = iter_start; iter < iter_stop; iter++) {
 
233
                if (iter->key == (jump_label_t)(unsigned long)key)
 
234
                        continue;
 
235
 
 
236
                key = (struct jump_label_key *)(unsigned long)iter->key;
 
237
 
 
238
                if (__module_address(iter->key) == mod) {
 
239
                        atomic_set(&key->enabled, 0);
 
240
                        key->entries = iter;
 
241
                        key->next = NULL;
 
242
                        continue;
 
243
                }
 
244
 
 
245
                jlm = kzalloc(sizeof(struct jump_label_mod), GFP_KERNEL);
 
246
                if (!jlm)
 
247
                        return -ENOMEM;
 
248
 
 
249
                jlm->mod = mod;
 
250
                jlm->entries = iter;
 
251
                jlm->next = key->next;
 
252
                key->next = jlm;
 
253
 
 
254
                if (jump_label_enabled(key))
 
255
                        __jump_label_update(key, iter, iter_stop,
 
256
                                            JUMP_LABEL_ENABLE);
 
257
        }
 
258
 
352
259
        return 0;
353
260
}
354
261
 
355
 
static void remove_jump_label_module(struct module *mod)
 
262
static void jump_label_del_module(struct module *mod)
356
263
{
357
 
        struct hlist_head *head;
358
 
        struct hlist_node *node, *node_next, *module_node, *module_node_next;
359
 
        struct jump_label_entry *e;
360
 
        struct jump_label_module_entry *e_module;
361
 
        int i;
362
 
 
363
 
        /* if the module doesn't have jump label entries, just return */
364
 
        if (!mod->num_jump_entries)
365
 
                return;
366
 
 
367
 
        for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {
368
 
                head = &jump_label_table[i];
369
 
                hlist_for_each_entry_safe(e, node, node_next, head, hlist) {
370
 
                        hlist_for_each_entry_safe(e_module, module_node,
371
 
                                                  module_node_next,
372
 
                                                  &(e->modules), hlist) {
373
 
                                if (e_module->mod == mod) {
374
 
                                        hlist_del(&e_module->hlist);
375
 
                                        kfree(e_module);
376
 
                                }
377
 
                        }
378
 
                        if (hlist_empty(&e->modules) && (e->nr_entries == 0)) {
379
 
                                hlist_del(&e->hlist);
380
 
                                kfree(e);
381
 
                        }
 
264
        struct jump_entry *iter_start = mod->jump_entries;
 
265
        struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
 
266
        struct jump_entry *iter;
 
267
        struct jump_label_key *key = NULL;
 
268
        struct jump_label_mod *jlm, **prev;
 
269
 
 
270
        for (iter = iter_start; iter < iter_stop; iter++) {
 
271
                if (iter->key == (jump_label_t)(unsigned long)key)
 
272
                        continue;
 
273
 
 
274
                key = (struct jump_label_key *)(unsigned long)iter->key;
 
275
 
 
276
                if (__module_address(iter->key) == mod)
 
277
                        continue;
 
278
 
 
279
                prev = &key->next;
 
280
                jlm = key->next;
 
281
 
 
282
                while (jlm && jlm->mod != mod) {
 
283
                        prev = &jlm->next;
 
284
                        jlm = jlm->next;
 
285
                }
 
286
 
 
287
                if (jlm) {
 
288
                        *prev = jlm->next;
 
289
                        kfree(jlm);
382
290
                }
383
291
        }
384
292
}
385
293
 
386
 
static void remove_jump_label_module_init(struct module *mod)
 
294
static void jump_label_invalidate_module_init(struct module *mod)
387
295
{
388
 
        struct hlist_head *head;
389
 
        struct hlist_node *node, *node_next, *module_node, *module_node_next;
390
 
        struct jump_label_entry *e;
391
 
        struct jump_label_module_entry *e_module;
 
296
        struct jump_entry *iter_start = mod->jump_entries;
 
297
        struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
392
298
        struct jump_entry *iter;
393
 
        int i, count;
394
 
 
395
 
        /* if the module doesn't have jump label entries, just return */
396
 
        if (!mod->num_jump_entries)
397
 
                return;
398
 
 
399
 
        for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {
400
 
                head = &jump_label_table[i];
401
 
                hlist_for_each_entry_safe(e, node, node_next, head, hlist) {
402
 
                        hlist_for_each_entry_safe(e_module, module_node,
403
 
                                                  module_node_next,
404
 
                                                  &(e->modules), hlist) {
405
 
                                if (e_module->mod != mod)
406
 
                                        continue;
407
 
                                count = e_module->nr_entries;
408
 
                                iter = e_module->table;
409
 
                                while (count--) {
410
 
                                        if (within_module_init(iter->code, mod))
411
 
                                                iter->key = 0;
412
 
                                        iter++;
413
 
                                }
414
 
                        }
415
 
                }
 
299
 
 
300
        for (iter = iter_start; iter < iter_stop; iter++) {
 
301
                if (within_module_init(iter->code, mod))
 
302
                        iter->code = 0;
416
303
        }
417
304
}
418
305
 
426
313
        switch (val) {
427
314
        case MODULE_STATE_COMING:
428
315
                jump_label_lock();
429
 
                ret = add_jump_label_module(mod);
 
316
                ret = jump_label_add_module(mod);
430
317
                if (ret)
431
 
                        remove_jump_label_module(mod);
 
318
                        jump_label_del_module(mod);
432
319
                jump_label_unlock();
433
320
                break;
434
321
        case MODULE_STATE_GOING:
435
322
                jump_label_lock();
436
 
                remove_jump_label_module(mod);
 
323
                jump_label_del_module(mod);
437
324
                jump_label_unlock();
438
325
                break;
439
326
        case MODULE_STATE_LIVE:
440
327
                jump_label_lock();
441
 
                remove_jump_label_module_init(mod);
 
328
                jump_label_invalidate_module_init(mod);
442
329
                jump_label_unlock();
443
330
                break;
444
331
        }
445
 
        return ret;
446
 
}
447
 
 
448
 
/***
449
 
 * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop()
450
 
 * @mod: module to patch
451
 
 *
452
 
 * Allow for run-time selection of the optimal nops. Before the module
453
 
 * loads patch these with arch_get_jump_label_nop(), which is specified by
454
 
 * the arch specific jump label code.
455
 
 */
456
 
void jump_label_apply_nops(struct module *mod)
457
 
{
458
 
        struct jump_entry *iter;
459
 
 
460
 
        /* if the module doesn't have jump label entries, just return */
461
 
        if (!mod->num_jump_entries)
462
 
                return;
463
 
 
464
 
        iter = mod->jump_entries;
465
 
        while (iter < mod->jump_entries + mod->num_jump_entries) {
466
 
                arch_jump_label_text_poke_early(iter->code);
467
 
                iter++;
468
 
        }
 
332
 
 
333
        return notifier_from_errno(ret);
469
334
}
470
335
 
471
336
struct notifier_block jump_label_module_nb = {
472
337
        .notifier_call = jump_label_module_notify,
473
 
        .priority = 0,
 
338
        .priority = 1, /* higher than tracepoints */
474
339
};
475
340
 
476
 
static __init int init_jump_label_module(void)
 
341
static __init int jump_label_init_module(void)
477
342
{
478
343
        return register_module_notifier(&jump_label_module_nb);
479
344
}
480
 
early_initcall(init_jump_label_module);
 
345
early_initcall(jump_label_init_module);
481
346
 
482
347
#endif /* CONFIG_MODULES */
483
348
 
 
349
/***
 
350
 * jump_label_text_reserved - check if addr range is reserved
 
351
 * @start: start text addr
 
352
 * @end: end text addr
 
353
 *
 
354
 * checks if the text addr located between @start and @end
 
355
 * overlaps with any of the jump label patch addresses. Code
 
356
 * that wants to modify kernel text should first verify that
 
357
 * it does not overlap with any of the jump label addresses.
 
358
 * Caller must hold jump_label_mutex.
 
359
 *
 
360
 * returns 1 if there is an overlap, 0 otherwise
 
361
 */
 
362
int jump_label_text_reserved(void *start, void *end)
 
363
{
 
364
        int ret = __jump_label_text_reserved(__start___jump_table,
 
365
                        __stop___jump_table, start, end);
 
366
 
 
367
        if (ret)
 
368
                return ret;
 
369
 
 
370
#ifdef CONFIG_MODULES
 
371
        ret = __jump_label_mod_text_reserved(start, end);
 
372
#endif
 
373
        return ret;
 
374
}
 
375
 
 
376
static void jump_label_update(struct jump_label_key *key, int enable)
 
377
{
 
378
        struct jump_entry *entry = key->entries, *stop = __stop___jump_table;
 
379
 
 
380
#ifdef CONFIG_MODULES
 
381
        struct module *mod = __module_address((jump_label_t)key);
 
382
 
 
383
        __jump_label_mod_update(key, enable);
 
384
 
 
385
        if (mod)
 
386
                stop = mod->jump_entries + mod->num_jump_entries;
 
387
#endif
 
388
        /* if there are no users, entry can be NULL */
 
389
        if (entry)
 
390
                __jump_label_update(key, entry, stop, enable);
 
391
}
 
392
 
484
393
#endif