mm/highmem.c

   1 /*
   2  * High memory handling common code and variables.
   3  *
   4  * (C) 1999 Andrea Arcangeli, SuSE GmbH, andrea@suse.de
   5  *          Gerhard Wichert, Siemens AG, Gerhard.Wichert@pdb.siemens.de
   6  *
   7  *
   8  * Redesigned the x86 32-bit VM architecture to deal with
   9  * 64-bit physical space. With current x86 CPUs this
  10  * means up to 64 Gigabytes physical RAM.
  11  *
  12  * Rewrote high memory support to move the page cache into
  13  * high memory. Implemented permanent (schedulable) kmaps
  14  * based on Linus' idea.
  15  *
  16  * Copyright (C) 1999 Ingo Molnar <mingo@redhat.com>
  17  */
  18
  19 #include <linux/mm.h>
  20 #include <linux/pagemap.h>
  21 #include <linux/mempool.h>
  22 #include <linux/blkdev.h>
  23 #include <asm/pgalloc.h>
  24
  25 static mempool_t *page_pool, *isa_page_pool;
  26
  27 static void *page_pool_alloc(int gfp_mask, void *data)
  28 {
  29         int gfp = gfp_mask | (int) (long) data;
  30
  31         return alloc_page(gfp);
  32 }
  33
  34 static void page_pool_free(void *page, void *data)
  35 {
  36         __free_page(page);
  37 }
  38
  39 /*
  40  * Virtual_count is not a pure "count".
  41  *  0 means that it is not mapped, and has not been mapped
  42  *    since a TLB flush - it is usable.
  43  *  1 means that there are no users, but it has been mapped
  44  *    since the last TLB flush - so we can't use it.
  45  *  n means that there are (n-1) current users of it.
  46  */
  47 #ifdef CONFIG_HIGHMEM
  48 static int pkmap_count[LAST_PKMAP];
  49 static unsigned int last_pkmap_nr;
  50 static spinlock_t kmap_lock __cacheline_aligned_in_smp = SPIN_LOCK_UNLOCKED;
  51
  52 pte_t * pkmap_page_table;
  53
  54 static DECLARE_WAIT_QUEUE_HEAD(pkmap_map_wait);
  55
  56 static void flush_all_zero_pkmaps(void)
  57 {
  58         int i;
  59
  60         flush_cache_all();
  61
  62         for (i = 0; i < LAST_PKMAP; i++) {
  63                 struct page *page;
  64
  65                 /*
  66                  * zero means we don't have anything to do,
  67                  * >1 means that it is still in use. Only
  68                  * a count of 1 means that it is free but
  69                  * needs to be unmapped
  70                  */
  71                 if (pkmap_count[i] != 1)
  72                         continue;
  73                 pkmap_count[i] = 0;
  74
  75                 /* sanity check */
  76                 if (pte_none(pkmap_page_table[i]))
  77                         BUG();
  78
  79                 /*
  80                  * Don't need an atomic fetch-and-clear op here;
  81                  * no-one has the page mapped, and cannot get at
  82                  * its virtual address (and hence PTE) without first
  83                  * getting the kmap_lock (which is held here).
  84                  * So no dangers, even with speculative execution.
  85                  */
  86                 page = pte_page(pkmap_page_table[i]);
  87                 pte_clear(&pkmap_page_table[i]);
  88
  89                 page->virtual = NULL;
  90         }
  91         flush_tlb_kernel_range(PKMAP_ADDR(0), PKMAP_ADDR(LAST_PKMAP));
  92 }
  93
  94 static inline unsigned long map_new_virtual(struct page *page)
  95 {
  96         unsigned long vaddr;
  97         int count;
  98
  99 start:
 100         count = LAST_PKMAP;
 101         /* Find an empty entry */
 102         for (;;) {
 103                 last_pkmap_nr = (last_pkmap_nr + 1) & LAST_PKMAP_MASK;
 104                 if (!last_pkmap_nr) {
 105                         flush_all_zero_pkmaps();
 106                         count = LAST_PKMAP;
 107                 }
 108                 if (!pkmap_count[last_pkmap_nr])
 109                         break;  /* Found a usable entry */
 110                 if (--count)
 111                         continue;
 112
 113                 /*
 114                  * Sleep for somebody else to unmap their entries
 115                  */
 116                 {
 117                         DECLARE_WAITQUEUE(wait, current);
 118
 119                         current->state = TASK_UNINTERRUPTIBLE;
 120                         add_wait_queue(&pkmap_map_wait, &wait);
 121                         spin_unlock(&kmap_lock);
 122                         schedule();
 123                         remove_wait_queue(&pkmap_map_wait, &wait);
 124                         spin_lock(&kmap_lock);
 125
 126                         /* Somebody else might have mapped it while we slept */
 127                         if (page->virtual)
 128                                 return (unsigned long) page->virtual;
 129
 130                         /* Re-start */
 131                         goto start;
 132                 }
 133         }
 134         vaddr = PKMAP_ADDR(last_pkmap_nr);
 135         set_pte(&(pkmap_page_table[last_pkmap_nr]), mk_pte(page, kmap_prot));
 136
 137         pkmap_count[last_pkmap_nr] = 1;
 138         page->virtual = (void *) vaddr;
 139
 140         return vaddr;
 141 }
 142
 143 void *kmap_high(struct page *page)
 144 {
 145         unsigned long vaddr;
 146
 147         /*
 148          * For highmem pages, we can't trust "virtual" until
 149          * after we have the lock.
 150          *
 151          * We cannot call this from interrupts, as it may block
 152          */
 153         spin_lock(&kmap_lock);
 154         vaddr = (unsigned long) page->virtual;
 155         if (!vaddr)
 156                 vaddr = map_new_virtual(page);
 157         pkmap_count[PKMAP_NR(vaddr)]++;
 158         if (pkmap_count[PKMAP_NR(vaddr)] < 2)
 159                 BUG();
 160         spin_unlock(&kmap_lock);
 161         return (void*) vaddr;
 162 }
 163
 164 void kunmap_high(struct page *page)
 165 {
 166         unsigned long vaddr;
 167         unsigned long nr;
 168         int need_wakeup;
 169
 170         spin_lock(&kmap_lock);
 171         vaddr = (unsigned long) page->virtual;
 172         if (!vaddr)
 173                 BUG();
 174         nr = PKMAP_NR(vaddr);
 175
 176         /*
 177          * A count must never go down to zero
 178          * without a TLB flush!
 179          */
 180         need_wakeup = 0;
 181         switch (--pkmap_count[nr]) {
 182         case 0:
 183                 BUG();
 184         case 1:
 185                 /*
 186                  * Avoid an unnecessary wake_up() function call.
 187                  * The common case is pkmap_count[] == 1, but
 188                  * no waiters.
 189                  * The tasks queued in the wait-queue are guarded
 190                  * by both the lock in the wait-queue-head and by
 191                  * the kmap_lock.  As the kmap_lock is held here,
 192                  * no need for the wait-queue-head's lock.  Simply
 193                  * test if the queue is empty.
 194                  */
 195                 need_wakeup = waitqueue_active(&pkmap_map_wait);
 196         }
 197         spin_unlock(&kmap_lock);
 198
 199         /* do wake-up, if needed, race-free outside of the spin lock */
 200         if (need_wakeup)
 201                 wake_up(&pkmap_map_wait);
 202 }
 203
 204 #define POOL_SIZE       64
 205
 206 static __init int init_emergency_pool(void)
 207 {
 208         struct sysinfo i;
 209         si_meminfo(&i);
 210         si_swapinfo(&i);
 211
 212         if (!i.totalhigh)
 213                 return 0;
 214
 215         page_pool = mempool_create(POOL_SIZE, page_pool_alloc, page_pool_free, NULL);
 216         if (!page_pool)
 217                 BUG();
 218         printk("highmem bounce pool size: %d pages\n", POOL_SIZE);
 219
 220         return 0;
 221 }
 222
 223 __initcall(init_emergency_pool);
 224
 225 /*
 226  * highmem version, map in to vec
 227  */
 228 static inline void bounce_copy_vec(struct bio_vec *to, unsigned char *vfrom)
 229 {
 230         unsigned long flags;
 231         unsigned char *vto;
 232
 233         local_irq_save(flags);
 234         vto = kmap_atomic(to->bv_page, KM_BOUNCE_READ);
 235         memcpy(vto + to->bv_offset, vfrom, to->bv_len);
 236         kunmap_atomic(vto, KM_BOUNCE_READ);
 237         local_irq_restore(flags);
 238 }
 239
 240 #else /* CONFIG_HIGHMEM */
 241
 242 #define bounce_copy_vec(to, vfrom)      \
 243         memcpy(page_address((to)->bv_page) + (to)->bv_offset, vfrom, (to)->bv_len)
 244
 245 #endif
 246
 247 #define ISA_POOL_SIZE   16
 248
 249 /*
 250  * gets called "every" time someone init's a queue with BLK_BOUNCE_ISA
 251  * as the max address, so check if the pool has already been created.
 252  */
 253 int init_emergency_isa_pool(void)
 254 {
 255         if (isa_page_pool)
 256                 return 0;
 257
 258         isa_page_pool = mempool_create(ISA_POOL_SIZE, page_pool_alloc, page_pool_free, (void *) __GFP_DMA);
 259         if (!isa_page_pool)
 260                 BUG();
 261
 262         printk("isa bounce pool size: %d pages\n", ISA_POOL_SIZE);
 263         return 0;
 264 }
 265
 266 /*
 267  * Simple bounce buffer support for highmem pages. Depending on the
 268  * queue gfp mask set, *to may or may not be a highmem page. kmap it
 269  * always, it will do the Right Thing
 270  */
 271 static inline void copy_to_high_bio_irq(struct bio *to, struct bio *from)
 272 {
 273         unsigned char *vfrom;
 274         struct bio_vec *tovec, *fromvec;
 275         int i;
 276
 277         __bio_for_each_segment(tovec, to, i, 0) {
 278                 fromvec = from->bi_io_vec + i;
 279
 280                 /*
 281                  * not bounced
 282                  */
 283                 if (tovec->bv_page == fromvec->bv_page)
 284                         continue;
 285
 286                 vfrom = page_address(fromvec->bv_page) + fromvec->bv_offset;
 287
 288                 bounce_copy_vec(tovec, vfrom);
 289         }
 290 }
 291
 292 static inline void bounce_end_io(struct bio *bio, mempool_t *pool)
 293 {
 294         struct bio *bio_orig = bio->bi_private;
 295         struct bio_vec *bvec, *org_vec;
 296         int i;
 297
 298         if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
 299                 goto out_eio;
 300
 301         set_bit(BIO_UPTODATE, &bio_orig->bi_flags);
 302
 303         /*
 304          * free up bounce indirect pages used
 305          */
 306         __bio_for_each_segment(bvec, bio, i, 0) {
 307                 org_vec = bio_orig->bi_io_vec + i;
 308                 if (bvec->bv_page == org_vec->bv_page)
 309                         continue;
 310
 311                 mempool_free(bvec->bv_page, pool);
 312         }
 313
 314 out_eio:
 315         bio_orig->bi_end_io(bio_orig);
 316         bio_put(bio);
 317 }
 318
 319 static void bounce_end_io_write(struct bio *bio)
 320 {
 321         bounce_end_io(bio, page_pool);
 322 }
 323
 324 static void bounce_end_io_write_isa(struct bio *bio)
 325 {
 326         bounce_end_io(bio, isa_page_pool);
 327 }
 328
 329 static inline void __bounce_end_io_read(struct bio *bio, mempool_t *pool)
 330 {
 331         struct bio *bio_orig = bio->bi_private;
 332
 333         if (test_bit(BIO_UPTODATE, &bio->bi_flags))
 334                 copy_to_high_bio_irq(bio_orig, bio);
 335
 336         bounce_end_io(bio, pool);
 337 }
 338
 339 static void bounce_end_io_read(struct bio *bio)
 340 {
 341         __bounce_end_io_read(bio, page_pool);
 342 }
 343
 344 static void bounce_end_io_read_isa(struct bio *bio)
 345 {
 346         return __bounce_end_io_read(bio, isa_page_pool);
 347 }
 348
 349 void create_bounce(unsigned long pfn, int gfp, struct bio **bio_orig)
 350 {
 351         struct page *page;
 352         struct bio *bio = NULL;
 353         int i, rw = bio_data_dir(*bio_orig), bio_gfp;
 354         struct bio_vec *to, *from;
 355         mempool_t *pool;
 356
 357         BUG_ON((*bio_orig)->bi_idx);
 358
 359         /*
 360          * for non-isa bounce case, just check if the bounce pfn is equal
 361          * to or bigger than the highest pfn in the system -- in that case,
 362          * don't waste time iterating over bio segments
 363          */
 364         if (!(gfp & GFP_DMA)) {
 365                 if (pfn >= blk_max_pfn)
 366                         return;
 367
 368                 bio_gfp = GFP_NOHIGHIO;
 369                 pool = page_pool;
 370         } else {
 371                 BUG_ON(!isa_page_pool);
 372                 bio_gfp = GFP_NOIO;
 373                 pool = isa_page_pool;
 374         }
 375
 376         bio_for_each_segment(from, *bio_orig, i) {
 377                 page = from->bv_page;
 378
 379                 /*
 380                  * is destination page below bounce pfn?
 381                  */
 382                 if ((page - page_zone(page)->zone_mem_map) + (page_zone(page)->zone_start_paddr >> PAGE_SHIFT) < pfn)
 383                         continue;
 384
 385                 /*
 386                  * irk, bounce it
 387                  */
 388                 if (!bio)
 389                         bio = bio_alloc(bio_gfp, (*bio_orig)->bi_vcnt);
 390
 391                 to = bio->bi_io_vec + i;
 392
 393                 to->bv_page = mempool_alloc(pool, gfp);
 394                 to->bv_len = from->bv_len;
 395                 to->bv_offset = from->bv_offset;
 396
 397                 if (rw & WRITE) {
 398                         char *vto, *vfrom;
 399
 400                         vto = page_address(to->bv_page) + to->bv_offset;
 401                         vfrom = kmap(from->bv_page) + from->bv_offset;
 402                         memcpy(vto, vfrom, to->bv_len);
 403                         kunmap(from->bv_page);
 404                 }
 405         }
 406
 407         /*
 408          * no pages bounced
 409          */
 410         if (!bio)
 411                 return;
 412
 413         /*
 414          * at least one page was bounced, fill in possible non-highmem
 415          * pages
 416          */
 417         bio_for_each_segment(from, *bio_orig, i) {
 418                 to = &bio->bi_io_vec[i];
 419                 if (!to->bv_page) {
 420                         to->bv_page = from->bv_page;
 421                         to->bv_len = from->bv_len;
 422                         to->bv_offset = to->bv_offset;
 423                 }
 424         }
 425
 426         bio->bi_bdev = (*bio_orig)->bi_bdev;
 427         bio->bi_sector = (*bio_orig)->bi_sector;
 428         bio->bi_rw = (*bio_orig)->bi_rw;
 429
 430         bio->bi_vcnt = (*bio_orig)->bi_vcnt;
 431         bio->bi_idx = 0;
 432         bio->bi_size = (*bio_orig)->bi_size;
 433
 434         if (pool == page_pool) {
 435                 if (rw & WRITE)
 436                         bio->bi_end_io = bounce_end_io_write;
 437                 else
 438                         bio->bi_end_io = bounce_end_io_read;
 439         } else {
 440                 if (rw & WRITE)
 441                         bio->bi_end_io = bounce_end_io_write_isa;
 442                 else
 443                         bio->bi_end_io = bounce_end_io_read_isa;
 444         }
 445
 446         bio->bi_private = *bio_orig;
 447         *bio_orig = bio;
 448 }
 449
 450 #if CONFIG_DEBUG_HIGHMEM
 451 void check_highmem_ptes(void)
 452 {
 453         int idx, type;
 454
 455         for (type = 0; type < KM_TYPE_NR; type++) {
 456                 idx = type + KM_TYPE_NR*smp_processor_id();
 457                 if (!pte_none(*(kmap_pte-idx))) {
 458                         printk("scheduling with KM_TYPE %d held!\n", type);
 459                         BUG();
 460                 }
 461         }
 462 }
 463 #endif
 464