2
* Copyright (c) 2005 Topspin Communications. All rights reserved.
3
* Copyright (c) 2005 Cisco Systems. All rights reserved.
4
* Copyright (c) 2005 Mellanox Technologies. All rights reserved.
6
* This software is available to you under a choice of one of two
7
* licenses. You may choose to be licensed under the terms of the GNU
8
* General Public License (GPL) Version 2, available from the file
9
* COPYING in the main directory of this source tree, or the
10
* OpenIB.org BSD license below:
12
* Redistribution and use in source and binary forms, with or
13
* without modification, are permitted provided that the following
16
* - Redistributions of source code must retain the above
17
* copyright notice, this list of conditions and the following
20
* - Redistributions in binary form must reproduce the above
21
* copyright notice, this list of conditions and the following
22
* disclaimer in the documentation and/or other materials
23
* provided with the distribution.
25
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36
#include <linux/dma-mapping.h>
37
#include <linux/sched.h>
38
#include <linux/export.h>
39
#include <linux/hugetlb.h>
40
#include <linux/dma-attrs.h>
41
#include <linux/slab.h>
45
#define IB_UMEM_MAX_PAGE_CHUNK \
46
((PAGE_SIZE - offsetof(struct ib_umem_chunk, page_list)) / \
47
((void *) &((struct ib_umem_chunk *) 0)->page_list[1] - \
48
(void *) &((struct ib_umem_chunk *) 0)->page_list[0]))
50
static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int dirty)
52
struct ib_umem_chunk *chunk, *tmp;
55
list_for_each_entry_safe(chunk, tmp, &umem->chunk_list, list) {
56
ib_dma_unmap_sg(dev, chunk->page_list,
57
chunk->nents, DMA_BIDIRECTIONAL);
58
for (i = 0; i < chunk->nents; ++i) {
59
struct page *page = sg_page(&chunk->page_list[i]);
61
if (umem->writable && dirty)
62
set_page_dirty_lock(page);
71
* ib_umem_get - Pin and DMA map userspace memory.
72
* @context: userspace context to pin memory for
73
* @addr: userspace virtual address to start at
74
* @size: length of region to pin
75
* @access: IB_ACCESS_xxx flags for memory being pinned
76
* @dmasync: flush in-flight DMA when the memory region is written
78
struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr,
79
size_t size, int access, int dmasync)
82
struct page **page_list;
83
struct vm_area_struct **vma_list;
84
struct ib_umem_chunk *chunk;
86
unsigned long lock_limit;
87
unsigned long cur_base;
92
DEFINE_DMA_ATTRS(attrs);
95
dma_set_attr(DMA_ATTR_WRITE_BARRIER, &attrs);
98
return ERR_PTR(-EPERM);
100
umem = kmalloc(sizeof *umem, GFP_KERNEL);
102
return ERR_PTR(-ENOMEM);
104
umem->context = context;
106
umem->offset = addr & ~PAGE_MASK;
107
umem->page_size = PAGE_SIZE;
109
* We ask for writable memory if any access flags other than
110
* "remote read" are set. "Local write" and "remote write"
111
* obviously require write access. "Remote atomic" can do
112
* things like fetch and add, which will modify memory, and
113
* "MW bind" can change permissions by binding a window.
115
umem->writable = !!(access & ~IB_ACCESS_REMOTE_READ);
117
/* We assume the memory is from hugetlb until proved otherwise */
120
INIT_LIST_HEAD(&umem->chunk_list);
122
page_list = (struct page **) __get_free_page(GFP_KERNEL);
125
return ERR_PTR(-ENOMEM);
129
* if we can't alloc the vma_list, it's not so bad;
130
* just assume the memory is not hugetlb memory
132
vma_list = (struct vm_area_struct **) __get_free_page(GFP_KERNEL);
136
npages = PAGE_ALIGN(size + umem->offset) >> PAGE_SHIFT;
138
down_write(¤t->mm->mmap_sem);
140
locked = npages + current->mm->pinned_vm;
141
lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
143
if ((locked > lock_limit) && !capable(CAP_IPC_LOCK)) {
148
cur_base = addr & PAGE_MASK;
152
ret = get_user_pages(current, current->mm, cur_base,
153
min_t(unsigned long, npages,
154
PAGE_SIZE / sizeof (struct page *)),
155
1, !umem->writable, page_list, vma_list);
160
cur_base += ret * PAGE_SIZE;
166
chunk = kmalloc(sizeof *chunk + sizeof (struct scatterlist) *
167
min_t(int, ret, IB_UMEM_MAX_PAGE_CHUNK),
174
chunk->nents = min_t(int, ret, IB_UMEM_MAX_PAGE_CHUNK);
175
sg_init_table(chunk->page_list, chunk->nents);
176
for (i = 0; i < chunk->nents; ++i) {
178
!is_vm_hugetlb_page(vma_list[i + off]))
180
sg_set_page(&chunk->page_list[i], page_list[i + off], PAGE_SIZE, 0);
183
chunk->nmap = ib_dma_map_sg_attrs(context->device,
184
&chunk->page_list[0],
188
if (chunk->nmap <= 0) {
189
for (i = 0; i < chunk->nents; ++i)
190
put_page(sg_page(&chunk->page_list[i]));
199
list_add_tail(&chunk->list, &umem->chunk_list);
207
__ib_umem_release(context->device, umem, 0);
210
current->mm->pinned_vm = locked;
212
up_write(¤t->mm->mmap_sem);
214
free_page((unsigned long) vma_list);
215
free_page((unsigned long) page_list);
217
return ret < 0 ? ERR_PTR(ret) : umem;
219
EXPORT_SYMBOL(ib_umem_get);
221
static void ib_umem_account(struct work_struct *work)
223
struct ib_umem *umem = container_of(work, struct ib_umem, work);
225
down_write(&umem->mm->mmap_sem);
226
umem->mm->pinned_vm -= umem->diff;
227
up_write(&umem->mm->mmap_sem);
233
* ib_umem_release - release memory pinned with ib_umem_get
234
* @umem: umem struct to release
236
void ib_umem_release(struct ib_umem *umem)
238
struct ib_ucontext *context = umem->context;
239
struct mm_struct *mm;
242
__ib_umem_release(umem->context->device, umem, 1);
244
mm = get_task_mm(current);
250
diff = PAGE_ALIGN(umem->length + umem->offset) >> PAGE_SHIFT;
253
* We may be called with the mm's mmap_sem already held. This
254
* can happen when a userspace munmap() is the call that drops
255
* the last reference to our file and calls our release
256
* method. If there are memory regions to destroy, we'll end
257
* up here and not be able to take the mmap_sem. In that case
258
* we defer the vm_locked accounting to the system workqueue.
260
if (context->closing) {
261
if (!down_write_trylock(&mm->mmap_sem)) {
262
INIT_WORK(&umem->work, ib_umem_account);
266
queue_work(ib_wq, &umem->work);
270
down_write(&mm->mmap_sem);
272
current->mm->locked_vm -= diff;
273
up_write(&mm->mmap_sem);
277
EXPORT_SYMBOL(ib_umem_release);
279
int ib_umem_page_count(struct ib_umem *umem)
281
struct ib_umem_chunk *chunk;
286
shift = ilog2(umem->page_size);
289
list_for_each_entry(chunk, &umem->chunk_list, list)
290
for (i = 0; i < chunk->nmap; ++i)
291
n += sg_dma_len(&chunk->page_list[i]) >> shift;
295
EXPORT_SYMBOL(ib_umem_page_count);