mm/highmem.c

   1 /*
   2  * High memory handling common code and variables.
   3  *
   4  * (C) 1999 Andrea Arcangeli, SuSE GmbH, andrea@suse.de
   5  *          Gerhard Wichert, Siemens AG, Gerhard.Wichert@pdb.siemens.de
   6  *
   7  *
   8  * Redesigned the x86 32-bit VM architecture to deal with
   9  * 64-bit physical space. With current x86 CPUs this
  10  * means up to 64 Gigabytes physical RAM.
  11  *
  12  * Rewrote high memory support to move the page cache into
  13  * high memory. Implemented permanent (schedulable) kmaps
  14  * based on Linus' idea.
  15  *
  16  * Copyright (C) 1999 Ingo Molnar <mingo@redhat.com>
  17  */
  18
  19 #include <linux/mm.h>
  20 #include <linux/pagemap.h>
  21 #include <linux/highmem.h>
  22 #include <linux/swap.h>
  23 #include <linux/slab.h>
  24 #include <linux/compiler.h>
  25
  26 #include <linux/kernel_stat.h>
  27
  28 /*
  29  * Virtual_count is not a pure "count".
  30  *  0 means that it is not mapped, and has not been mapped
  31  *    since a TLB flush - it is usable.
  32  *  1 means that there are no users, but it has been mapped
  33  *    since the last TLB flush - so we can't use it.
  34  *  n means that there are (n-1) current users of it.
  35  */
  36 static int pkmap_count[LAST_PKMAP];
  37 static unsigned int last_pkmap_nr;
  38 static spinlock_t kmap_lock = SPIN_LOCK_UNLOCKED;
  39
  40 pte_t * pkmap_page_table;
  41
  42 static DECLARE_WAIT_QUEUE_HEAD(pkmap_map_wait);
  43
  44 static void flush_all_zero_pkmaps(void)
  45 {
  46         int i;
  47
  48         flush_cache_all();
  49
  50         for (i = 0; i < LAST_PKMAP; i++) {
  51                 struct page *page;
  52
  53                 /*
  54                  * zero means we don't have anything to do,
  55                  * >1 means that it is still in use. Only
  56                  * a count of 1 means that it is free but
  57                  * needs to be unmapped
  58                  */
  59                 if (pkmap_count[i] != 1)
  60                         continue;
  61                 pkmap_count[i] = 0;
  62
  63                 /* sanity check */
  64                 if (pte_none(pkmap_page_table[i]))
  65                         BUG();
  66
  67                 /*
  68                  * Don't need an atomic fetch-and-clear op here;
  69                  * no-one has the page mapped, and cannot get at
  70                  * its virtual address (and hence PTE) without first
  71                  * getting the kmap_lock (which is held here).
  72                  * So no dangers, even with speculative execution.
  73                  */
  74                 page = pte_page(pkmap_page_table[i]);
  75                 pte_clear(&pkmap_page_table[i]);
  76
  77                 page->virtual = NULL;
  78         }
  79         flush_tlb_all();
  80 }
  81
  82 static inline unsigned long map_new_virtual(struct page *page)
  83 {
  84         unsigned long vaddr;
  85         int count;
  86
  87 start:
  88         count = LAST_PKMAP;
  89         /* Find an empty entry */
  90         for (;;) {
  91                 last_pkmap_nr = (last_pkmap_nr + 1) & LAST_PKMAP_MASK;
  92                 if (!last_pkmap_nr) {
  93                         flush_all_zero_pkmaps();
  94                         count = LAST_PKMAP;
  95                 }
  96                 if (!pkmap_count[last_pkmap_nr])
  97                         break;  /* Found a usable entry */
  98                 if (--count)
  99                         continue;
 100
 101                 /*
 102                  * Sleep for somebody else to unmap their entries
 103                  */
 104                 {
 105                         DECLARE_WAITQUEUE(wait, current);
 106
 107                         current->state = TASK_UNINTERRUPTIBLE;
 108                         add_wait_queue(&pkmap_map_wait, &wait);
 109                         spin_unlock(&kmap_lock);
 110                         schedule();
 111                         remove_wait_queue(&pkmap_map_wait, &wait);
 112                         spin_lock(&kmap_lock);
 113
 114                         /* Somebody else might have mapped it while we slept */
 115                         if (page->virtual)
 116                                 return (unsigned long) page->virtual;
 117
 118                         /* Re-start */
 119                         goto start;
 120                 }
 121         }
 122         vaddr = PKMAP_ADDR(last_pkmap_nr);
 123         set_pte(&(pkmap_page_table[last_pkmap_nr]), mk_pte(page, kmap_prot));
 124
 125         pkmap_count[last_pkmap_nr] = 1;
 126         page->virtual = (void *) vaddr;
 127
 128         return vaddr;
 129 }
 130
 131 void *kmap_high(struct page *page)
 132 {
 133         unsigned long vaddr;
 134
 135         /*
 136          * For highmem pages, we can't trust "virtual" until
 137          * after we have the lock.
 138          *
 139          * We cannot call this from interrupts, as it may block
 140          */
 141         spin_lock(&kmap_lock);
 142         vaddr = (unsigned long) page->virtual;
 143         if (!vaddr)
 144                 vaddr = map_new_virtual(page);
 145         pkmap_count[PKMAP_NR(vaddr)]++;
 146         if (pkmap_count[PKMAP_NR(vaddr)] < 2)
 147                 BUG();
 148         spin_unlock(&kmap_lock);
 149         return (void*) vaddr;
 150 }
 151
 152 void kunmap_high(struct page *page)
 153 {
 154         unsigned long vaddr;
 155         unsigned long nr;
 156         int need_wakeup;
 157
 158         spin_lock(&kmap_lock);
 159         vaddr = (unsigned long) page->virtual;
 160         if (!vaddr)
 161                 BUG();
 162         nr = PKMAP_NR(vaddr);
 163
 164         /*
 165          * A count must never go down to zero
 166          * without a TLB flush!
 167          */
 168         need_wakeup = 0;
 169         switch (--pkmap_count[nr]) {
 170         case 0:
 171                 BUG();
 172         case 1:
 173                 /*
 174                  * Avoid an unnecessary wake_up() function call.
 175                  * The common case is pkmap_count[] == 1, but
 176                  * no waiters.
 177                  * The tasks queued in the wait-queue are guarded
 178                  * by both the lock in the wait-queue-head and by
 179                  * the kmap_lock.  As the kmap_lock is held here,
 180                  * no need for the wait-queue-head's lock.  Simply
 181                  * test if the queue is empty.
 182                  */
 183                 need_wakeup = waitqueue_active(&pkmap_map_wait);
 184         }
 185         spin_unlock(&kmap_lock);
 186
 187         /* do wake-up, if needed, race-free outside of the spin lock */
 188         if (need_wakeup)
 189                 wake_up(&pkmap_map_wait);
 190 }
 191
 192 #define POOL_SIZE 64
 193
 194 /*
 195  * This lock gets no contention at all, normally.
 196  */
 197 static spinlock_t emergency_lock = SPIN_LOCK_UNLOCKED;
 198
 199 int nr_emergency_pages;
 200 static LIST_HEAD(emergency_pages);
 201
 202 int nr_emergency_bhs;
 203 static LIST_HEAD(emergency_bhs);
 204
 205 /*
 206  * Simple bounce buffer support for highmem pages. Depending on the
 207  * queue gfp mask set, *to may or may not be a highmem page. kmap it
 208  * always, it will do the Right Thing
 209  */
 210 static inline void copy_to_high_bio_irq(struct bio *to, struct bio *from)
 211 {
 212         unsigned char *vto, *vfrom;
 213         unsigned long flags;
 214         struct bio_vec *tovec, *fromvec;
 215         int i;
 216
 217         __bio_for_each_segment(tovec, to, i, 0) {
 218                 fromvec = &from->bi_io_vec[i];
 219
 220                 /*
 221                  * not bounced
 222                  */
 223                 if (tovec->bv_page == fromvec->bv_page)
 224                         continue;
 225
 226                 vfrom = page_address(fromvec->bv_page) + fromvec->bv_offset;
 227
 228                 local_irq_save(flags);
 229                 vto = kmap_atomic(tovec->bv_page, KM_BOUNCE_READ);
 230                 memcpy(vto + tovec->bv_offset, vfrom, tovec->bv_len);
 231                 kunmap_atomic(vto, KM_BOUNCE_READ);
 232                 local_irq_restore(flags);
 233         }
 234 }
 235
 236 static __init int init_emergency_pool(void)
 237 {
 238         struct sysinfo i;
 239         si_meminfo(&i);
 240         si_swapinfo(&i);
 241
 242         if (!i.totalhigh)
 243                 return 0;
 244
 245         spin_lock_irq(&emergency_lock);
 246         while (nr_emergency_pages < POOL_SIZE) {
 247                 struct page * page = alloc_page(GFP_ATOMIC);
 248                 if (!page) {
 249                         printk("couldn't refill highmem emergency pages");
 250                         break;
 251                 }
 252                 list_add(&page->list, &emergency_pages);
 253                 nr_emergency_pages++;
 254         }
 255         spin_unlock_irq(&emergency_lock);
 256         printk("allocated %d pages reserved for the highmem bounces\n", nr_emergency_pages);
 257         return 0;
 258 }
 259
 260 __initcall(init_emergency_pool);
 261
 262 static inline int bounce_end_io (struct bio *bio, int nr_sectors)
 263 {
 264         struct bio *bio_orig = bio->bi_private;
 265         struct bio_vec *bvec, *org_vec;
 266         unsigned long flags;
 267         int ret, i;
 268
 269         if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
 270                 goto out_eio;
 271
 272         set_bit(BIO_UPTODATE, &bio_orig->bi_flags);
 273
 274         /*
 275          * free up bounce indirect pages used
 276          */
 277         spin_lock_irqsave(&emergency_lock, flags);
 278         __bio_for_each_segment(bvec, bio, i, 0) {
 279                 org_vec = &bio_orig->bi_io_vec[i];
 280                 if (bvec->bv_page == org_vec->bv_page)
 281                         continue;
 282
 283                 if (nr_emergency_pages >= POOL_SIZE)
 284                         __free_page(bvec->bv_page);
 285                 else {
 286                         /*
 287                          * We are abusing page->list to manage
 288                          * the highmem emergency pool:
 289                          */
 290                         list_add(&bvec->bv_page->list, &emergency_pages);
 291                         nr_emergency_pages++;
 292                 }
 293         }
 294         spin_unlock_irqrestore(&emergency_lock, flags);
 295
 296 out_eio:
 297         ret = bio_orig->bi_end_io(bio_orig, nr_sectors);
 298
 299         bio_put(bio);
 300         return ret;
 301 }
 302
 303 static int bounce_end_io_write(struct bio *bio, int nr_sectors)
 304 {
 305         return bounce_end_io(bio, nr_sectors);
 306 }
 307
 308 static int bounce_end_io_read (struct bio *bio, int nr_sectors)
 309 {
 310         struct bio *bio_orig = bio->bi_private;
 311
 312         if (test_bit(BIO_UPTODATE, &bio->bi_flags))
 313                 copy_to_high_bio_irq(bio_orig, bio);
 314
 315         return bounce_end_io(bio, nr_sectors);
 316 }
 317
 318 struct page *alloc_bounce_page(int gfp_mask)
 319 {
 320         struct list_head *tmp;
 321         struct page *page;
 322
 323         page = alloc_page(gfp_mask);
 324         if (page)
 325                 return page;
 326         /*
 327          * No luck. First, kick the VM so it doesnt idle around while
 328          * we are using up our emergency rations.
 329          */
 330         wakeup_bdflush();
 331
 332 repeat_alloc:
 333         /*
 334          * Try to allocate from the emergency pool.
 335          */
 336         tmp = &emergency_pages;
 337         spin_lock_irq(&emergency_lock);
 338         if (!list_empty(tmp)) {
 339                 page = list_entry(tmp->next, struct page, list);
 340                 list_del(tmp->next);
 341                 nr_emergency_pages--;
 342         }
 343         spin_unlock_irq(&emergency_lock);
 344         if (page)
 345                 return page;
 346
 347         /* we need to wait I/O completion */
 348         run_task_queue(&tq_disk);
 349
 350         current->policy |= SCHED_YIELD;
 351         __set_current_state(TASK_RUNNING);
 352         schedule();
 353         goto repeat_alloc;
 354 }
 355
 356 void create_bounce(unsigned long pfn, struct bio **bio_orig)
 357 {
 358         struct page *page;
 359         struct bio *bio = NULL;
 360         int i, rw = bio_data_dir(*bio_orig);
 361         struct bio_vec *to, *from;
 362
 363         BUG_ON((*bio_orig)->bi_idx);
 364
 365         bio_for_each_segment(from, *bio_orig, i) {
 366                 page = from->bv_page;
 367
 368                 /*
 369                  * is destination page below bounce pfn?
 370                  */
 371                 if ((page - page->zone->zone_mem_map) + (page->zone->zone_start_paddr >> PAGE_SHIFT) < pfn)
 372                         continue;
 373
 374                 /*
 375                  * irk, bounce it
 376                  */
 377                 if (!bio)
 378                         bio = bio_alloc(GFP_NOHIGHIO, (*bio_orig)->bi_vcnt);
 379
 380                 to = &bio->bi_io_vec[i];
 381
 382                 to->bv_page = alloc_bounce_page(GFP_NOHIGHIO);
 383                 to->bv_len = from->bv_len;
 384                 to->bv_offset = from->bv_offset;
 385
 386                 if (rw & WRITE) {
 387                         char *vto, *vfrom;
 388
 389                         vto = page_address(to->bv_page) + to->bv_offset;
 390                         vfrom = kmap(from->bv_page) + from->bv_offset;
 391                         memcpy(vto, vfrom, to->bv_len);
 392                         kunmap(from->bv_page);
 393                 }
 394         }
 395
 396         /*
 397          * no pages bounced
 398          */
 399         if (!bio)
 400                 return;
 401
 402         /*
 403          * at least one page was bounced, fill in possible non-highmem
 404          * pages
 405          */
 406         bio_for_each_segment(from, *bio_orig, i) {
 407                 to = &bio->bi_io_vec[i];
 408                 if (!to->bv_page) {
 409                         to->bv_page = from->bv_page;
 410                         to->bv_len = from->bv_len;
 411                         to->bv_offset = to->bv_offset;
 412                 }
 413         }
 414
 415         bio->bi_dev = (*bio_orig)->bi_dev;
 416         bio->bi_sector = (*bio_orig)->bi_sector;
 417         bio->bi_rw = (*bio_orig)->bi_rw;
 418
 419         bio->bi_vcnt = (*bio_orig)->bi_vcnt;
 420         bio->bi_idx = 0;
 421         bio->bi_size = (*bio_orig)->bi_size;
 422
 423         if (rw & WRITE)
 424                 bio->bi_end_io = bounce_end_io_write;
 425         else
 426                 bio->bi_end_io = bounce_end_io_read;
 427
 428         bio->bi_private = *bio_orig;
 429         *bio_orig = bio;
 430 }