← Back to branch summary

~ubuntu-branches/ubuntu/precise/linux-ti-omap4/precise

« back to all changes in this revision

Viewing changes to fs/xfs/linux-2.6/xfs_buf.c

  • Committer: Bazaar Package Importer
  • Author(s): Paolo Pisati
  • Date: 2011-06-29 15:23:51 UTC
  • mfrom: (26.1.1 natty-proposed)
  • Revision ID: james.westby@ubuntu.com-20110629152351-xs96tm303d95rpbk
Tags: 3.0.0-1200.2
* Rebased against 3.0.0-6.7
* BSP from TI based on 3.0.0

Show diffs side-by-side

added added

removed removed

Lines of Context:
fs/xfs/linux-2.6/xfs_buf.c
33
33
#include <linux/migrate.h>
34
34
#include <linux/backing-dev.h>
35
35
#include <linux/freezer.h>
36
 
#include <linux/list_sort.h>
37
36
 
38
37
#include "xfs_sb.h"
39
38
#include "xfs_inum.h"
94
93
}
95
94
 
96
95
/*
97
 
 *      Page Region interfaces.
98
 
 *
99
 
 *      For pages in filesystems where the blocksize is smaller than the
100
 
 *      pagesize, we use the page->private field (long) to hold a bitmap
101
 
 *      of uptodate regions within the page.
102
 
 *
103
 
 *      Each such region is "bytes per page / bits per long" bytes long.
104
 
 *
105
 
 *      NBPPR == number-of-bytes-per-page-region
106
 
 *      BTOPR == bytes-to-page-region (rounded up)
107
 
 *      BTOPRT == bytes-to-page-region-truncated (rounded down)
108
 
 */
109
 
#if (BITS_PER_LONG == 32)
110
 
#define PRSHIFT         (PAGE_CACHE_SHIFT - 5)  /* (32 == 1<<5) */
111
 
#elif (BITS_PER_LONG == 64)
112
 
#define PRSHIFT         (PAGE_CACHE_SHIFT - 6)  /* (64 == 1<<6) */
113
 
#else
114
 
#error BITS_PER_LONG must be 32 or 64
115
 
#endif
116
 
#define NBPPR           (PAGE_CACHE_SIZE/BITS_PER_LONG)
117
 
#define BTOPR(b)        (((unsigned int)(b) + (NBPPR - 1)) >> PRSHIFT)
118
 
#define BTOPRT(b)       (((unsigned int)(b) >> PRSHIFT))
119
 
 
120
 
STATIC unsigned long
121
 
page_region_mask(
122
 
        size_t          offset,
123
 
        size_t          length)
124
 
{
125
 
        unsigned long   mask;
126
 
        int             first, final;
127
 
 
128
 
        first = BTOPR(offset);
129
 
        final = BTOPRT(offset + length - 1);
130
 
        first = min(first, final);
131
 
 
132
 
        mask = ~0UL;
133
 
        mask <<= BITS_PER_LONG - (final - first);
134
 
        mask >>= BITS_PER_LONG - (final);
135
 
 
136
 
        ASSERT(offset + length <= PAGE_CACHE_SIZE);
137
 
        ASSERT((final - first) < BITS_PER_LONG && (final - first) >= 0);
138
 
 
139
 
        return mask;
140
 
}
141
 
 
142
 
STATIC void
143
 
set_page_region(
144
 
        struct page     *page,
145
 
        size_t          offset,
146
 
        size_t          length)
147
 
{
148
 
        set_page_private(page,
149
 
                page_private(page) | page_region_mask(offset, length));
150
 
        if (page_private(page) == ~0UL)
151
 
                SetPageUptodate(page);
152
 
}
153
 
 
154
 
STATIC int
155
 
test_page_region(
156
 
        struct page     *page,
157
 
        size_t          offset,
158
 
        size_t          length)
159
 
{
160
 
        unsigned long   mask = page_region_mask(offset, length);
161
 
 
162
 
        return (mask && (page_private(page) & mask) == mask);
163
 
}
164
 
 
165
 
/*
166
96
 * xfs_buf_lru_add - add a buffer to the LRU.
167
97
 *
168
98
 * The LRU takes a new reference to the buffer so that it will only be freed
189
119
 * The unlocked check is safe here because it only occurs when there are not
190
120
 * b_lru_ref counts left on the inode under the pag->pag_buf_lock. it is there
191
121
 * to optimise the shrinker removing the buffer from the LRU and calling
192
 
 * xfs_buf_free(). i.e. it removes an unneccessary round trip on the
 
122
 * xfs_buf_free(). i.e. it removes an unnecessary round trip on the
193
123
 * bt_lru_lock.
194
124
 */
195
125
STATIC void
332
262
 
333
263
        ASSERT(list_empty(&bp->b_lru));
334
264
 
335
 
        if (bp->b_flags & (_XBF_PAGE_CACHE|_XBF_PAGES)) {
 
265
        if (bp->b_flags & _XBF_PAGES) {
336
266
                uint            i;
337
267
 
338
268
                if (xfs_buf_is_vmapped(bp))
342
272
                for (i = 0; i < bp->b_page_count; i++) {
343
273
                        struct page     *page = bp->b_pages[i];
344
274
 
345
 
                        if (bp->b_flags & _XBF_PAGE_CACHE)
346
 
                                ASSERT(!PagePrivate(page));
347
 
                        page_cache_release(page);
 
275
                        __free_page(page);
348
276
                }
349
 
        }
 
277
        } else if (bp->b_flags & _XBF_KMEM)
 
278
                kmem_free(bp->b_addr);
350
279
        _xfs_buf_free_pages(bp);
351
280
        xfs_buf_deallocate(bp);
352
281
}
353
282
 
354
283
/*
355
 
 *      Finds all pages for buffer in question and builds it's page list.
 
284
 * Allocates all the pages for buffer in question and builds it's page list.
356
285
 */
357
286
STATIC int
358
 
_xfs_buf_lookup_pages(
 
287
xfs_buf_allocate_memory(
359
288
        xfs_buf_t               *bp,
360
289
        uint                    flags)
361
290
{
362
 
        struct address_space    *mapping = bp->b_target->bt_mapping;
363
 
        size_t                  blocksize = bp->b_target->bt_bsize;
364
291
        size_t                  size = bp->b_count_desired;
365
292
        size_t                  nbytes, offset;
366
293
        gfp_t                   gfp_mask = xb_to_gfp(flags);
367
294
        unsigned short          page_count, i;
368
 
        pgoff_t                 first;
369
295
        xfs_off_t               end;
370
296
        int                     error;
371
297
 
 
298
        /*
 
299
         * for buffers that are contained within a single page, just allocate
 
300
         * the memory from the heap - there's no need for the complexity of
 
301
         * page arrays to keep allocation down to order 0.
 
302
         */
 
303
        if (bp->b_buffer_length < PAGE_SIZE) {
 
304
                bp->b_addr = kmem_alloc(bp->b_buffer_length, xb_to_km(flags));
 
305
                if (!bp->b_addr) {
 
306
                        /* low memory - use alloc_page loop instead */
 
307
                        goto use_alloc_page;
 
308
                }
 
309
 
 
310
                if (((unsigned long)(bp->b_addr + bp->b_buffer_length - 1) &
 
311
                                                                PAGE_MASK) !=
 
312
                    ((unsigned long)bp->b_addr & PAGE_MASK)) {
 
313
                        /* b_addr spans two pages - use alloc_page instead */
 
314
                        kmem_free(bp->b_addr);
 
315
                        bp->b_addr = NULL;
 
316
                        goto use_alloc_page;
 
317
                }
 
318
                bp->b_offset = offset_in_page(bp->b_addr);
 
319
                bp->b_pages = bp->b_page_array;
 
320
                bp->b_pages[0] = virt_to_page(bp->b_addr);
 
321
                bp->b_page_count = 1;
 
322
                bp->b_flags |= XBF_MAPPED | _XBF_KMEM;
 
323
                return 0;
 
324
        }
 
325
 
 
326
use_alloc_page:
372
327
        end = bp->b_file_offset + bp->b_buffer_length;
373
328
        page_count = xfs_buf_btoc(end) - xfs_buf_btoct(bp->b_file_offset);
374
 
 
375
329
        error = _xfs_buf_get_pages(bp, page_count, flags);
376
330
        if (unlikely(error))
377
331
                return error;
378
 
        bp->b_flags |= _XBF_PAGE_CACHE;
379
332
 
380
333
        offset = bp->b_offset;
381
 
        first = bp->b_file_offset >> PAGE_CACHE_SHIFT;
 
334
        bp->b_flags |= _XBF_PAGES;
382
335
 
383
336
        for (i = 0; i < bp->b_page_count; i++) {
384
337
                struct page     *page;
385
338
                uint            retries = 0;
386
 
 
387
 
              retry:
388
 
                page = find_or_create_page(mapping, first + i, gfp_mask);
 
339
retry:
 
340
                page = alloc_page(gfp_mask);
389
341
                if (unlikely(page == NULL)) {
390
342
                        if (flags & XBF_READ_AHEAD) {
391
343
                                bp->b_page_count = i;
392
 
                                for (i = 0; i < bp->b_page_count; i++)
393
 
                                        unlock_page(bp->b_pages[i]);
394
 
                                return -ENOMEM;
 
344
                                error = ENOMEM;
 
345
                                goto out_free_pages;
395
346
                        }
396
347
 
397
348
                        /*
401
352
                         * handle buffer allocation failures we can't do much.
402
353
                         */
403
354
                        if (!(++retries % 100))
404
 
                                printk(KERN_ERR
405
 
                                        "XFS: possible memory allocation "
406
 
                                        "deadlock in %s (mode:0x%x)\n",
 
355
                                xfs_err(NULL,
 
356
                "possible memory allocation deadlock in %s (mode:0x%x)",
407
357
                                        __func__, gfp_mask);
408
358
 
409
359
                        XFS_STATS_INC(xb_page_retries);
413
363
 
414
364
                XFS_STATS_INC(xb_page_found);
415
365
 
416
 
                nbytes = min_t(size_t, size, PAGE_CACHE_SIZE - offset);
 
366
                nbytes = min_t(size_t, size, PAGE_SIZE - offset);
417
367
                size -= nbytes;
418
 
 
419
 
                ASSERT(!PagePrivate(page));
420
 
                if (!PageUptodate(page)) {
421
 
                        page_count--;
422
 
                        if (blocksize >= PAGE_CACHE_SIZE) {
423
 
                                if (flags & XBF_READ)
424
 
                                        bp->b_flags |= _XBF_PAGE_LOCKED;
425
 
                        } else if (!PagePrivate(page)) {
426
 
                                if (test_page_region(page, offset, nbytes))
427
 
                                        page_count++;
428
 
                        }
429
 
                }
430
 
 
431
368
                bp->b_pages[i] = page;
432
369
                offset = 0;
433
370
        }
434
 
 
435
 
        if (!(bp->b_flags & _XBF_PAGE_LOCKED)) {
436
 
                for (i = 0; i < bp->b_page_count; i++)
437
 
                        unlock_page(bp->b_pages[i]);
438
 
        }
439
 
 
440
 
        if (page_count == bp->b_page_count)
441
 
                bp->b_flags |= XBF_DONE;
442
 
 
 
371
        return 0;
 
372
 
 
373
out_free_pages:
 
374
        for (i = 0; i < bp->b_page_count; i++)
 
375
                __free_page(bp->b_pages[i]);
443
376
        return error;
444
377
}
445
378
 
446
379
/*
447
 
 *      Map buffer into kernel address-space if nessecary.
 
380
 *      Map buffer into kernel address-space if necessary.
448
381
 */
449
382
STATIC int
450
383
_xfs_buf_map_pages(
451
384
        xfs_buf_t               *bp,
452
385
        uint                    flags)
453
386
{
454
 
        /* A single page buffer is always mappable */
 
387
        ASSERT(bp->b_flags & _XBF_PAGES);
455
388
        if (bp->b_page_count == 1) {
 
389
                /* A single page buffer is always mappable */
456
390
                bp->b_addr = page_address(bp->b_pages[0]) + bp->b_offset;
457
391
                bp->b_flags |= XBF_MAPPED;
458
392
        } else if (flags & XBF_MAPPED) {
459
 
                bp->b_addr = vm_map_ram(bp->b_pages, bp->b_page_count,
460
 
                                        -1, PAGE_KERNEL);
461
 
                if (unlikely(bp->b_addr == NULL))
 
393
                int retried = 0;
 
394
 
 
395
                do {
 
396
                        bp->b_addr = vm_map_ram(bp->b_pages, bp->b_page_count,
 
397
                                                -1, PAGE_KERNEL);
 
398
                        if (bp->b_addr)
 
399
                                break;
 
400
                        vm_unmap_aliases();
 
401
                } while (retried++ <= 1);
 
402
 
 
403
                if (!bp->b_addr)
462
404
                        return -ENOMEM;
463
405
                bp->b_addr += bp->b_offset;
464
406
                bp->b_flags |= XBF_MAPPED;
569
511
                }
570
512
        }
571
513
 
 
514
        /*
 
515
         * if the buffer is stale, clear all the external state associated with
 
516
         * it. We need to keep flags such as how we allocated the buffer memory
 
517
         * intact here.
 
518
         */
572
519
        if (bp->b_flags & XBF_STALE) {
573
520
                ASSERT((bp->b_flags & _XBF_DELWRI_Q) == 0);
574
 
                bp->b_flags &= XBF_MAPPED;
 
521
                bp->b_flags &= XBF_MAPPED | _XBF_KMEM | _XBF_PAGES;
575
522
        }
576
523
 
577
524
        trace_xfs_buf_find(bp, flags, _RET_IP_);
592
539
        xfs_buf_flags_t         flags)
593
540
{
594
541
        xfs_buf_t               *bp, *new_bp;
595
 
        int                     error = 0, i;
 
542
        int                     error = 0;
596
543
 
597
544
        new_bp = xfs_buf_allocate(flags);
598
545
        if (unlikely(!new_bp))
600
547
 
601
548
        bp = _xfs_buf_find(target, ioff, isize, flags, new_bp);
602
549
        if (bp == new_bp) {
603
 
                error = _xfs_buf_lookup_pages(bp, flags);
 
550
                error = xfs_buf_allocate_memory(bp, flags);
604
551
                if (error)
605
552
                        goto no_buffer;
606
553
        } else {
609
556
                        return NULL;
610
557
        }
611
558
 
612
 
        for (i = 0; i < bp->b_page_count; i++)
613
 
                mark_page_accessed(bp->b_pages[i]);
614
 
 
615
559
        if (!(bp->b_flags & XBF_MAPPED)) {
616
560
                error = _xfs_buf_map_pages(bp, flags);
617
561
                if (unlikely(error)) {
618
 
                        printk(KERN_WARNING "%s: failed to map pages\n",
619
 
                                        __func__);
 
562
                        xfs_warn(target->bt_mount,
 
563
                                "%s: failed to map pages\n", __func__);
620
564
                        goto no_buffer;
621
565
                }
622
566
        }
710
654
        xfs_off_t               ioff,
711
655
        size_t                  isize)
712
656
{
713
 
        struct backing_dev_info *bdi;
714
 
 
715
 
        bdi = target->bt_mapping->backing_dev_info;
716
 
        if (bdi_read_congested(bdi))
 
657
        if (bdi_read_congested(target->bt_bdi))
717
658
                return;
718
659
 
719
660
        xfs_buf_read(target, ioff, isize,
767
708
        return bp;
768
709
}
769
710
 
 
711
/*
 
712
 * Return a buffer allocated as an empty buffer and associated to external
 
713
 * memory via xfs_buf_associate_memory() back to it's empty state.
 
714
 */
 
715
void
 
716
xfs_buf_set_empty(
 
717
        struct xfs_buf          *bp,
 
718
        size_t                  len)
 
719
{
 
720
        if (bp->b_pages)
 
721
                _xfs_buf_free_pages(bp);
 
722
 
 
723
        bp->b_pages = NULL;
 
724
        bp->b_page_count = 0;
 
725
        bp->b_addr = NULL;
 
726
        bp->b_file_offset = 0;
 
727
        bp->b_buffer_length = bp->b_count_desired = len;
 
728
        bp->b_bn = XFS_BUF_DADDR_NULL;
 
729
        bp->b_flags &= ~XBF_MAPPED;
 
730
}
 
731
 
770
732
static inline struct page *
771
733
mem_to_page(
772
734
        void                    *addr)
791
753
        size_t                  buflen;
792
754
        int                     page_count;
793
755
 
794
 
        pageaddr = (unsigned long)mem & PAGE_CACHE_MASK;
 
756
        pageaddr = (unsigned long)mem & PAGE_MASK;
795
757
        offset = (unsigned long)mem - pageaddr;
796
 
        buflen = PAGE_CACHE_ALIGN(len + offset);
797
 
        page_count = buflen >> PAGE_CACHE_SHIFT;
 
758
        buflen = PAGE_ALIGN(len + offset);
 
759
        page_count = buflen >> PAGE_SHIFT;
798
760
 
799
761
        /* Free any previous set of page pointers */
800
762
        if (bp->b_pages)
811
773
 
812
774
        for (i = 0; i < bp->b_page_count; i++) {
813
775
                bp->b_pages[i] = mem_to_page((void *)pageaddr);
814
 
                pageaddr += PAGE_CACHE_SIZE;
 
776
                pageaddr += PAGE_SIZE;
815
777
        }
816
778
 
817
779
        bp->b_count_desired = len;
818
780
        bp->b_buffer_length = buflen;
819
781
        bp->b_flags |= XBF_MAPPED;
820
 
        bp->b_flags &= ~_XBF_PAGE_LOCKED;
821
782
 
822
783
        return 0;
823
784
}
850
811
 
851
812
        error = _xfs_buf_map_pages(bp, XBF_MAPPED);
852
813
        if (unlikely(error)) {
853
 
                printk(KERN_WARNING "%s: failed to map pages\n",
854
 
                                __func__);
 
814
                xfs_warn(target->bt_mount,
 
815
                        "%s: failed to map pages\n", __func__);
855
816
                goto fail_free_mem;
856
817
        }
857
818
 
924
885
 
925
886
 
926
887
/*
927
 
 *      Mutual exclusion on buffers.  Locking model:
928
 
 *
929
 
 *      Buffers associated with inodes for which buffer locking
930
 
 *      is not enabled are not protected by semaphores, and are
931
 
 *      assumed to be exclusively owned by the caller.  There is a
932
 
 *      spinlock in the buffer, used by the caller when concurrent
933
 
 *      access is possible.
934
 
 */
935
 
 
936
 
/*
937
 
 *      Locks a buffer object, if it is not already locked.  Note that this in
938
 
 *      no way locks the underlying pages, so it is only useful for
939
 
 *      synchronizing concurrent use of buffer objects, not for synchronizing
940
 
 *      independent access to the underlying pages.
 
888
 *      Lock a buffer object, if it is not already locked.
941
889
 *
942
890
 *      If we come across a stale, pinned, locked buffer, we know that we are
943
891
 *      being asked to lock a buffer that has been reallocated. Because it is
971
919
}
972
920
 
973
921
/*
974
 
 *      Locks a buffer object.
975
 
 *      Note that this in no way locks the underlying pages, so it is only
976
 
 *      useful for synchronizing concurrent use of buffer objects, not for
977
 
 *      synchronizing independent access to the underlying pages.
 
922
 *      Lock a buffer object.
978
923
 *
979
924
 *      If we come across a stale, pinned, locked buffer, we know that we
980
925
 *      are being asked to lock a buffer that has been reallocated. Because
990
935
 
991
936
        if (atomic_read(&bp->b_pin_count) && (bp->b_flags & XBF_STALE))
992
937
                xfs_log_force(bp->b_target->bt_mount, 0);
993
 
        if (atomic_read(&bp->b_io_remaining))
994
 
                blk_run_address_space(bp->b_target->bt_mapping);
995
938
        down(&bp->b_sema);
996
939
        XB_SET_OWNER(bp);
997
940
 
1035
978
                set_current_state(TASK_UNINTERRUPTIBLE);
1036
979
                if (atomic_read(&bp->b_pin_count) == 0)
1037
980
                        break;
1038
 
                if (atomic_read(&bp->b_io_remaining))
1039
 
                        blk_run_address_space(bp->b_target->bt_mapping);
1040
 
                schedule();
 
981
                io_schedule();
1041
982
        }
1042
983
        remove_wait_queue(&bp->b_waiters, &wait);
1043
984
        set_current_state(TASK_RUNNING);
1249
1190
        xfs_buf_t               *bp,
1250
1191
        int                     schedule)
1251
1192
{
1252
 
        if (atomic_dec_and_test(&bp->b_io_remaining) == 1) {
1253
 
                bp->b_flags &= ~_XBF_PAGE_LOCKED;
 
1193
        if (atomic_dec_and_test(&bp->b_io_remaining) == 1)
1254
1194
                xfs_buf_ioend(bp, schedule);
1255
 
        }
1256
1195
}
1257
1196
 
1258
1197
STATIC void
1261
1200
        int                     error)
1262
1201
{
1263
1202
        xfs_buf_t               *bp = (xfs_buf_t *)bio->bi_private;
1264
 
        unsigned int            blocksize = bp->b_target->bt_bsize;
1265
 
        struct bio_vec          *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
1266
1203
 
1267
1204
        xfs_buf_ioerror(bp, -error);
1268
1205
 
1269
1206
        if (!error && xfs_buf_is_vmapped(bp) && (bp->b_flags & XBF_READ))
1270
1207
                invalidate_kernel_vmap_range(bp->b_addr, xfs_buf_vmap_len(bp));
1271
1208
 
1272
 
        do {
1273
 
                struct page     *page = bvec->bv_page;
1274
 
 
1275
 
                ASSERT(!PagePrivate(page));
1276
 
                if (unlikely(bp->b_error)) {
1277
 
                        if (bp->b_flags & XBF_READ)
1278
 
                                ClearPageUptodate(page);
1279
 
                } else if (blocksize >= PAGE_CACHE_SIZE) {
1280
 
                        SetPageUptodate(page);
1281
 
                } else if (!PagePrivate(page) &&
1282
 
                                (bp->b_flags & _XBF_PAGE_CACHE)) {
1283
 
                        set_page_region(page, bvec->bv_offset, bvec->bv_len);
1284
 
                }
1285
 
 
1286
 
                if (--bvec >= bio->bi_io_vec)
1287
 
                        prefetchw(&bvec->bv_page->flags);
1288
 
 
1289
 
                if (bp->b_flags & _XBF_PAGE_LOCKED)
1290
 
                        unlock_page(page);
1291
 
        } while (bvec >= bio->bi_io_vec);
1292
 
 
1293
1209
        _xfs_buf_ioend(bp, 1);
1294
1210
        bio_put(bio);
1295
1211
}
1303
1219
        int                     offset = bp->b_offset;
1304
1220
        int                     size = bp->b_count_desired;
1305
1221
        sector_t                sector = bp->b_bn;
1306
 
        unsigned int            blocksize = bp->b_target->bt_bsize;
1307
1222
 
1308
1223
        total_nr_pages = bp->b_page_count;
1309
1224
        map_i = 0;
1324
1239
                     (bp->b_flags & XBF_READ_AHEAD) ? READA : READ;
1325
1240
        }
1326
1241
 
1327
 
        /* Special code path for reading a sub page size buffer in --
1328
 
         * we populate up the whole page, and hence the other metadata
1329
 
         * in the same page.  This optimization is only valid when the
1330
 
         * filesystem block size is not smaller than the page size.
1331
 
         */
1332
 
        if ((bp->b_buffer_length < PAGE_CACHE_SIZE) &&
1333
 
            ((bp->b_flags & (XBF_READ|_XBF_PAGE_LOCKED)) ==
1334
 
              (XBF_READ|_XBF_PAGE_LOCKED)) &&
1335
 
            (blocksize >= PAGE_CACHE_SIZE)) {
1336
 
                bio = bio_alloc(GFP_NOIO, 1);
1337
 
 
1338
 
                bio->bi_bdev = bp->b_target->bt_bdev;
1339
 
                bio->bi_sector = sector - (offset >> BBSHIFT);
1340
 
                bio->bi_end_io = xfs_buf_bio_end_io;
1341
 
                bio->bi_private = bp;
1342
 
 
1343
 
                bio_add_page(bio, bp->b_pages[0], PAGE_CACHE_SIZE, 0);
1344
 
                size = 0;
1345
 
 
1346
 
                atomic_inc(&bp->b_io_remaining);
1347
 
 
1348
 
                goto submit_io;
1349
 
        }
1350
1242
 
1351
1243
next_chunk:
1352
1244
        atomic_inc(&bp->b_io_remaining);
1360
1252
        bio->bi_end_io = xfs_buf_bio_end_io;
1361
1253
        bio->bi_private = bp;
1362
1254
 
 
1255
 
1363
1256
        for (; size && nr_pages; nr_pages--, map_i++) {
1364
 
                int     rbytes, nbytes = PAGE_CACHE_SIZE - offset;
 
1257
                int     rbytes, nbytes = PAGE_SIZE - offset;
1365
1258
 
1366
1259
                if (nbytes > size)
1367
1260
                        nbytes = size;
1376
1269
                total_nr_pages--;
1377
1270
        }
1378
1271
 
1379
 
submit_io:
1380
1272
        if (likely(bio->bi_size)) {
1381
1273
                if (xfs_buf_is_vmapped(bp)) {
1382
1274
                        flush_kernel_vmap_range(bp->b_addr,
1386
1278
                if (size)
1387
1279
                        goto next_chunk;
1388
1280
        } else {
1389
 
                /*
1390
 
                 * if we get here, no pages were added to the bio. However,
1391
 
                 * we can't just error out here - if the pages are locked then
1392
 
                 * we have to unlock them otherwise we can hang on a later
1393
 
                 * access to the page.
1394
 
                 */
1395
1281
                xfs_buf_ioerror(bp, EIO);
1396
 
                if (bp->b_flags & _XBF_PAGE_LOCKED) {
1397
 
                        int i;
1398
 
                        for (i = 0; i < bp->b_page_count; i++)
1399
 
                                unlock_page(bp->b_pages[i]);
1400
 
                }
1401
1282
                bio_put(bio);
1402
1283
        }
1403
1284
}
1442
1323
{
1443
1324
        trace_xfs_buf_iowait(bp, _RET_IP_);
1444
1325
 
1445
 
        if (atomic_read(&bp->b_io_remaining))
1446
 
                blk_run_address_space(bp->b_target->bt_mapping);
1447
1326
        wait_for_completion(&bp->b_iowait);
1448
1327
 
1449
1328
        trace_xfs_buf_iowait_done(bp, _RET_IP_);
1461
1340
                return XFS_BUF_PTR(bp) + offset;
1462
1341
 
1463
1342
        offset += bp->b_offset;
1464
 
        page = bp->b_pages[offset >> PAGE_CACHE_SHIFT];
1465
 
        return (xfs_caddr_t)page_address(page) + (offset & (PAGE_CACHE_SIZE-1));
 
1343
        page = bp->b_pages[offset >> PAGE_SHIFT];
 
1344
        return (xfs_caddr_t)page_address(page) + (offset & (PAGE_SIZE-1));
1466
1345
}
1467
1346
 
1468
1347
/*
1484
1363
                page = bp->b_pages[xfs_buf_btoct(boff + bp->b_offset)];
1485
1364
                cpoff = xfs_buf_poff(boff + bp->b_offset);
1486
1365
                csize = min_t(size_t,
1487
 
                              PAGE_CACHE_SIZE-cpoff, bp->b_count_desired-boff);
 
1366
                              PAGE_SIZE-cpoff, bp->b_count_desired-boff);
1488
1367
 
1489
 
                ASSERT(((csize + cpoff) <= PAGE_CACHE_SIZE));
 
1368
                ASSERT(((csize + cpoff) <= PAGE_SIZE));
1490
1369
 
1491
1370
                switch (mode) {
1492
1371
                case XBRW_ZERO:
1543
1422
int
1544
1423
xfs_buftarg_shrink(
1545
1424
        struct shrinker         *shrink,
1546
 
        int                     nr_to_scan,
1547
 
        gfp_t                   mask)
 
1425
        struct shrink_control   *sc)
1548
1426
{
1549
1427
        struct xfs_buftarg      *btp = container_of(shrink,
1550
1428
                                        struct xfs_buftarg, bt_shrinker);
1551
1429
        struct xfs_buf          *bp;
 
1430
        int nr_to_scan = sc->nr_to_scan;
1552
1431
        LIST_HEAD(dispose);
1553
1432
 
1554
1433
        if (!nr_to_scan)
1599
1478
        xfs_flush_buftarg(btp, 1);
1600
1479
        if (mp->m_flags & XFS_MOUNT_BARRIER)
1601
1480
                xfs_blkdev_issue_flush(btp);
1602
 
        iput(btp->bt_mapping->host);
1603
1481
 
1604
1482
        kthread_stop(btp->bt_task);
1605
1483
        kmem_free(btp);
1617
1495
        btp->bt_smask = sectorsize - 1;
1618
1496
 
1619
1497
        if (set_blocksize(btp->bt_bdev, sectorsize)) {
1620
 
                printk(KERN_WARNING
1621
 
                        "XFS: Cannot set_blocksize to %u on device %s\n",
 
1498
                xfs_warn(btp->bt_mount,
 
1499
                        "Cannot set_blocksize to %u on device %s\n",
1622
1500
                        sectorsize, XFS_BUFTARG_NAME(btp));
1623
1501
                return EINVAL;
1624
1502
        }
1625
1503
 
1626
 
        if (verbose &&
1627
 
            (PAGE_CACHE_SIZE / BITS_PER_LONG) > sectorsize) {
1628
 
                printk(KERN_WARNING
1629
 
                        "XFS: %u byte sectors in use on device %s.  "
1630
 
                        "This is suboptimal; %u or greater is ideal.\n",
1631
 
                        sectorsize, XFS_BUFTARG_NAME(btp),
1632
 
                        (unsigned int)PAGE_CACHE_SIZE / BITS_PER_LONG);
1633
 
        }
1634
 
 
1635
1504
        return 0;
1636
1505
}
1637
1506
 
1646
1515
        struct block_device     *bdev)
1647
1516
{
1648
1517
        return xfs_setsize_buftarg_flags(btp,
1649
 
                        PAGE_CACHE_SIZE, bdev_logical_block_size(bdev), 0);
 
1518
                        PAGE_SIZE, bdev_logical_block_size(bdev), 0);
1650
1519
}
1651
1520
 
1652
1521
int
1659
1528
}
1660
1529
 
1661
1530
STATIC int
1662
 
xfs_mapping_buftarg(
1663
 
        xfs_buftarg_t           *btp,
1664
 
        struct block_device     *bdev)
1665
 
{
1666
 
        struct backing_dev_info *bdi;
1667
 
        struct inode            *inode;
1668
 
        struct address_space    *mapping;
1669
 
        static const struct address_space_operations mapping_aops = {
1670
 
                .sync_page = block_sync_page,
1671
 
                .migratepage = fail_migrate_page,
1672
 
        };
1673
 
 
1674
 
        inode = new_inode(bdev->bd_inode->i_sb);
1675
 
        if (!inode) {
1676
 
                printk(KERN_WARNING
1677
 
                        "XFS: Cannot allocate mapping inode for device %s\n",
1678
 
                        XFS_BUFTARG_NAME(btp));
1679
 
                return ENOMEM;
1680
 
        }
1681
 
        inode->i_ino = get_next_ino();
1682
 
        inode->i_mode = S_IFBLK;
1683
 
        inode->i_bdev = bdev;
1684
 
        inode->i_rdev = bdev->bd_dev;
1685
 
        bdi = blk_get_backing_dev_info(bdev);
1686
 
        if (!bdi)
1687
 
                bdi = &default_backing_dev_info;
1688
 
        mapping = &inode->i_data;
1689
 
        mapping->a_ops = &mapping_aops;
1690
 
        mapping->backing_dev_info = bdi;
1691
 
        mapping_set_gfp_mask(mapping, GFP_NOFS);
1692
 
        btp->bt_mapping = mapping;
1693
 
        return 0;
1694
 
}
1695
 
 
1696
 
STATIC int
1697
1531
xfs_alloc_delwrite_queue(
1698
1532
        xfs_buftarg_t           *btp,
1699
1533
        const char              *fsname)
1721
1555
        btp->bt_mount = mp;
1722
1556
        btp->bt_dev =  bdev->bd_dev;
1723
1557
        btp->bt_bdev = bdev;
 
1558
        btp->bt_bdi = blk_get_backing_dev_info(bdev);
 
1559
        if (!btp->bt_bdi)
 
1560
                goto error;
 
1561
 
1724
1562
        INIT_LIST_HEAD(&btp->bt_lru);
1725
1563
        spin_lock_init(&btp->bt_lru_lock);
1726
1564
        if (xfs_setsize_buftarg_early(btp, bdev))
1727
1565
                goto error;
1728
 
        if (xfs_mapping_buftarg(btp, bdev))
1729
 
                goto error;
1730
1566
        if (xfs_alloc_delwrite_queue(btp, fsname))
1731
1567
                goto error;
1732
1568
        btp->bt_shrinker.shrink = xfs_buftarg_shrink;
1923
1759
        do {
1924
1760
                long    age = xfs_buf_age_centisecs * msecs_to_jiffies(10);
1925
1761
                long    tout = xfs_buf_timer_centisecs * msecs_to_jiffies(10);
1926
 
                int     count = 0;
1927
1762
                struct list_head tmp;
 
1763
                struct blk_plug plug;
1928
1764
 
1929
1765
                if (unlikely(freezing(current))) {
1930
1766
                        set_bit(XBT_FORCE_SLEEP, &target->bt_flags);
1940
1776
 
1941
1777
                xfs_buf_delwri_split(target, &tmp, age);
1942
1778
                list_sort(NULL, &tmp, xfs_buf_cmp);
 
1779
 
 
1780
                blk_start_plug(&plug);
1943
1781
                while (!list_empty(&tmp)) {
1944
1782
                        struct xfs_buf *bp;
1945
1783
                        bp = list_first_entry(&tmp, struct xfs_buf, b_list);
1946
1784
                        list_del_init(&bp->b_list);
1947
1785
                        xfs_bdstrat_cb(bp);
1948
 
                        count++;
1949
1786
                }
1950
 
                if (count)
1951
 
                        blk_run_address_space(target->bt_mapping);
1952
 
 
 
1787
                blk_finish_plug(&plug);
1953
1788
        } while (!kthread_should_stop());
1954
1789
 
1955
1790
        return 0;
1969
1804
        int             pincount = 0;
1970
1805
        LIST_HEAD(tmp_list);
1971
1806
        LIST_HEAD(wait_list);
 
1807
        struct blk_plug plug;
1972
1808
 
1973
1809
        xfs_buf_runall_queues(xfsconvertd_workqueue);
1974
1810
        xfs_buf_runall_queues(xfsdatad_workqueue);
1983
1819
         * we do that after issuing all the IO.
1984
1820
         */
1985
1821
        list_sort(NULL, &tmp_list, xfs_buf_cmp);
 
1822
 
 
1823
        blk_start_plug(&plug);
1986
1824
        while (!list_empty(&tmp_list)) {
1987
1825
                bp = list_first_entry(&tmp_list, struct xfs_buf, b_list);
1988
1826
                ASSERT(target == bp->b_target);
1993
1831
                }
1994
1832
                xfs_bdstrat_cb(bp);
1995
1833
        }
 
1834
        blk_finish_plug(&plug);
1996
1835
 
1997
1836
        if (wait) {
1998
 
                /* Expedite and wait for IO to complete. */
1999
 
                blk_run_address_space(target->bt_mapping);
 
1837
                /* Wait for IO to complete. */
2000
1838
                while (!list_empty(&wait_list)) {
2001
1839
                        bp = list_first_entry(&wait_list, struct xfs_buf, b_list);
2002
1840
 
2022
1860
        if (!xfslogd_workqueue)
2023
1861
                goto out_free_buf_zone;
2024
1862
 
2025
 
        xfsdatad_workqueue = create_workqueue("xfsdatad");
 
1863
        xfsdatad_workqueue = alloc_workqueue("xfsdatad", WQ_MEM_RECLAIM, 1);
2026
1864
        if (!xfsdatad_workqueue)
2027
1865
                goto out_destroy_xfslogd_workqueue;
2028
1866
 
2029
 
        xfsconvertd_workqueue = create_workqueue("xfsconvertd");
 
1867
        xfsconvertd_workqueue = alloc_workqueue("xfsconvertd",
 
1868
                                                WQ_MEM_RECLAIM, 1);
2030
1869
        if (!xfsconvertd_workqueue)
2031
1870
                goto out_destroy_xfsdatad_workqueue;
2032
1871