arch/sparc64/mm/init.c

   1 /*  $Id: init.c,v 1.164 2001/03/03 10:34:45 davem Exp $
   2  *  arch/sparc64/mm/init.c
   3  *
   4  *  Copyright (C) 1996-1999 David S. Miller (davem@caip.rutgers.edu)
   5  *  Copyright (C) 1997-1999 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
   6  */
   7
   8 #include <linux/config.h>
   9 #include <linux/kernel.h>
  10 #include <linux/sched.h>
  11 #include <linux/string.h>
  12 #include <linux/init.h>
  13 #include <linux/bootmem.h>
  14 #include <linux/mm.h>
  15 #include <linux/slab.h>
  16 #include <linux/blk.h>
  17 #include <linux/swap.h>
  18 #include <linux/swapctl.h>
  19
  20 #include <asm/head.h>
  21 #include <asm/system.h>
  22 #include <asm/page.h>
  23 #include <asm/pgalloc.h>
  24 #include <asm/pgtable.h>
  25 #include <asm/oplib.h>
  26 #include <asm/iommu.h>
  27 #include <asm/io.h>
  28 #include <asm/uaccess.h>
  29 #include <asm/mmu_context.h>
  30 #include <asm/vaddrs.h>
  31 #include <asm/dma.h>
  32 #include <asm/starfire.h>
  33
  34 extern void device_scan(void);
  35
  36 struct sparc_phys_banks sp_banks[SPARC_PHYS_BANKS];
  37
  38 unsigned long *sparc64_valid_addr_bitmap;
  39
  40 /* Ugly, but necessary... -DaveM */
  41 unsigned long phys_base;
  42
  43 enum ultra_tlb_layout tlb_type = spitfire;
  44
  45 /* get_new_mmu_context() uses "cache + 1".  */
  46 spinlock_t ctx_alloc_lock = SPIN_LOCK_UNLOCKED;
  47 unsigned long tlb_context_cache = CTX_FIRST_VERSION - 1;
  48 #define CTX_BMAP_SLOTS (1UL << (CTX_VERSION_SHIFT - 6))
  49 unsigned long mmu_context_bmap[CTX_BMAP_SLOTS];
  50
  51 /* References to section boundaries */
  52 extern char __init_begin, __init_end, _start, _end, etext, edata;
  53
  54 /* Initial ramdisk setup */
  55 extern unsigned int sparc_ramdisk_image;
  56 extern unsigned int sparc_ramdisk_size;
  57
  58 int do_check_pgt_cache(int low, int high)
  59 {
  60         int freed = 0;
  61
  62         if (pgtable_cache_size > high) {
  63                 do {
  64 #ifdef CONFIG_SMP
  65                         if (pgd_quicklist)
  66                                 free_pgd_slow(get_pgd_fast()), freed++;
  67 #endif
  68                         if (pte_quicklist[0])
  69                                 free_pte_slow(get_pte_fast(0)), freed++;
  70                         if (pte_quicklist[1])
  71                                 free_pte_slow(get_pte_fast(1)), freed++;
  72                 } while (pgtable_cache_size > low);
  73         }
  74 #ifndef CONFIG_SMP
  75         if (pgd_cache_size > high / 4) {
  76                 struct page *page, *page2;
  77                 for (page2 = NULL, page = (struct page *)pgd_quicklist; page;) {
  78                         if ((unsigned long)page->pprev_hash == 3) {
  79                                 if (page2)
  80                                         page2->next_hash = page->next_hash;
  81                                 else
  82                                         (struct page *)pgd_quicklist = page->next_hash;
  83                                 page->next_hash = NULL;
  84                                 page->pprev_hash = NULL;
  85                                 pgd_cache_size -= 2;
  86                                 __free_page(page);
  87                                 freed++;
  88                                 if (page2)
  89                                         page = page2->next_hash;
  90                                 else
  91                                         page = (struct page *)pgd_quicklist;
  92                                 if (pgd_cache_size <= low / 4)
  93                                         break;
  94                                 continue;
  95                         }
  96                         page2 = page;
  97                         page = page->next_hash;
  98                 }
  99         }
 100 #endif
 101         return freed;
 102 }
 103
 104 extern void __update_mmu_cache(struct vm_area_struct *, unsigned long, pte_t);
 105
 106 void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t pte)
 107 {
 108         struct page *page = pte_page(pte);
 109
 110         if (VALID_PAGE(page) && page->mapping &&
 111             test_bit(PG_dcache_dirty, &page->flags)) {
 112                 __flush_dcache_page(page->virtual, 1);
 113                 clear_bit(PG_dcache_dirty, &page->flags);
 114         }
 115         __update_mmu_cache(vma, address, pte);
 116 }
 117
 118 /* In arch/sparc64/mm/ultra.S */
 119 extern void __flush_icache_page(unsigned long);
 120
 121 void flush_icache_range(unsigned long start, unsigned long end)
 122 {
 123         unsigned long kaddr;
 124
 125         for (kaddr = start; kaddr < end; kaddr += PAGE_SIZE)
 126                 __flush_icache_page(__get_phys(kaddr));
 127 }
 128
 129 /*
 130  * BAD_PAGE is the page that is used for page faults when linux
 131  * is out-of-memory. Older versions of linux just did a
 132  * do_exit(), but using this instead means there is less risk
 133  * for a process dying in kernel mode, possibly leaving an inode
 134  * unused etc..
 135  *
 136  * BAD_PAGETABLE is the accompanying page-table: it is initialized
 137  * to point to BAD_PAGE entries.
 138  *
 139  * ZERO_PAGE is a special page that is used for zero-initialized
 140  * data and COW.
 141  */
 142 pte_t __bad_page(void)
 143 {
 144         memset((void *) &empty_bad_page, 0, PAGE_SIZE);
 145         return pte_mkdirty(mk_pte_phys((((unsigned long) &empty_bad_page)
 146                                         - ((unsigned long)&empty_zero_page)
 147                                         + phys_base),
 148                                        PAGE_SHARED));
 149 }
 150
 151 void show_mem(void)
 152 {
 153         printk("Mem-info:\n");
 154         show_free_areas();
 155         printk("Free swap:       %6dkB\n",
 156                nr_swap_pages << (PAGE_SHIFT-10));
 157         printk("%ld pages of RAM\n", num_physpages);
 158         printk("%d free pages\n", nr_free_pages());
 159         printk("%d pages in page table cache\n",pgtable_cache_size);
 160 #ifndef CONFIG_SMP
 161         printk("%d entries in page dir cache\n",pgd_cache_size);
 162 #endif
 163         show_buffers();
 164 }
 165
 166 int mmu_info(char *buf)
 167 {
 168         if (tlb_type == cheetah)
 169                 return sprintf(buf, "MMU Type\t: Cheetah\n");
 170         else if (tlb_type == spitfire)
 171                 return sprintf(buf, "MMU Type\t: Spitfire\n");
 172         else
 173                 return sprintf(buf, "MMU Type\t: ???\n");
 174 }
 175
 176 struct linux_prom_translation {
 177         unsigned long virt;
 178         unsigned long size;
 179         unsigned long data;
 180 };
 181
 182 extern unsigned long prom_boot_page;
 183 extern void prom_remap(unsigned long physpage, unsigned long virtpage, int mmu_ihandle);
 184 extern int prom_get_mmu_ihandle(void);
 185 extern void register_prom_callbacks(void);
 186
 187 /* Exported for SMP bootup purposes. */
 188 unsigned long kern_locked_tte_data;
 189
 190 void __init early_pgtable_allocfail(char *type)
 191 {
 192         prom_printf("inherit_prom_mappings: Cannot alloc kernel %s.\n", type);
 193         prom_halt();
 194 }
 195
 196 static void inherit_prom_mappings(void)
 197 {
 198         struct linux_prom_translation *trans;
 199         unsigned long phys_page, tte_vaddr, tte_data;
 200         void (*remap_func)(unsigned long, unsigned long, int);
 201         pgd_t *pgdp;
 202         pmd_t *pmdp;
 203         pte_t *ptep;
 204         int node, n, i, tsz;
 205
 206         node = prom_finddevice("/virtual-memory");
 207         n = prom_getproplen(node, "translations");
 208         if (n == 0 || n == -1) {
 209                 prom_printf("Couldn't get translation property\n");
 210                 prom_halt();
 211         }
 212         n += 5 * sizeof(struct linux_prom_translation);
 213         for (tsz = 1; tsz < n; tsz <<= 1)
 214                 /* empty */;
 215         trans = __alloc_bootmem(tsz, SMP_CACHE_BYTES, 0UL);
 216         if (trans == NULL) {
 217                 prom_printf("inherit_prom_mappings: Cannot alloc translations.\n");
 218                 prom_halt();
 219         }
 220         memset(trans, 0, tsz);
 221
 222         if ((n = prom_getproperty(node, "translations", (char *)trans, tsz)) == -1) {
 223                 prom_printf("Couldn't get translation property\n");
 224                 prom_halt();
 225         }
 226         n = n / sizeof(*trans);
 227
 228         for (i = 0; i < n; i++) {
 229                 unsigned long vaddr;
 230
 231                 if (trans[i].virt >= 0xf0000000 && trans[i].virt < 0x100000000) {
 232                         for (vaddr = trans[i].virt;
 233                              vaddr < trans[i].virt + trans[i].size;
 234                              vaddr += PAGE_SIZE) {
 235                                 unsigned long val;
 236
 237                                 pgdp = pgd_offset(&init_mm, vaddr);
 238                                 if (pgd_none(*pgdp)) {
 239                                         pmdp = __alloc_bootmem(PMD_TABLE_SIZE,
 240                                                                PMD_TABLE_SIZE,
 241                                                                0UL);
 242                                         if (pmdp == NULL)
 243                                                 early_pgtable_allocfail("pmd");
 244                                         memset(pmdp, 0, PMD_TABLE_SIZE);
 245                                         pgd_set(pgdp, pmdp);
 246                                 }
 247                                 pmdp = pmd_offset(pgdp, vaddr);
 248                                 if (pmd_none(*pmdp)) {
 249                                         ptep = __alloc_bootmem(PTE_TABLE_SIZE,
 250                                                                PTE_TABLE_SIZE,
 251                                                                0UL);
 252                                         if (ptep == NULL)
 253                                                 early_pgtable_allocfail("pte");
 254                                         memset(ptep, 0, PTE_TABLE_SIZE);
 255                                         pmd_set(pmdp, ptep);
 256                                 }
 257                                 ptep = pte_offset(pmdp, vaddr);
 258
 259                                 val = trans[i].data;
 260
 261                                 /* Clear diag TTE bits. */
 262                                 if (tlb_type == spitfire)
 263                                         val &= ~0x0003fe0000000000UL;
 264
 265                                 set_pte (ptep, __pte(val | _PAGE_MODIFIED));
 266                                 trans[i].data += PAGE_SIZE;
 267                         }
 268                 }
 269         }
 270
 271         /* Now fixup OBP's idea about where we really are mapped. */
 272         prom_printf("Remapping the kernel... ");
 273
 274         /* Spitfire Errata #32 workaround */
 275         __asm__ __volatile__("stxa      %0, [%1] %2\n\t"
 276                              "flush     %%g6"
 277                              : /* No outputs */
 278                              : "r" (0),
 279                              "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
 280
 281         switch (tlb_type) {
 282         default:
 283         case spitfire:
 284                 phys_page = spitfire_get_dtlb_data(sparc64_highest_locked_tlbent());
 285                 break;
 286
 287         case cheetah:
 288                 phys_page = cheetah_get_ldtlb_data(sparc64_highest_locked_tlbent());
 289                 break;
 290         };
 291
 292         phys_page &= _PAGE_PADDR;
 293         phys_page += ((unsigned long)&prom_boot_page -
 294                       (unsigned long)&empty_zero_page);
 295
 296         if (tlb_type == spitfire) {
 297                 /* Lock this into i/d tlb entry 59 */
 298                 __asm__ __volatile__(
 299                         "stxa   %%g0, [%2] %3\n\t"
 300                         "stxa   %0, [%1] %4\n\t"
 301                         "membar #Sync\n\t"
 302                         "flush  %%g6\n\t"
 303                         "stxa   %%g0, [%2] %5\n\t"
 304                         "stxa   %0, [%1] %6\n\t"
 305                         "membar #Sync\n\t"
 306                         "flush  %%g6"
 307                         : : "r" (phys_page | _PAGE_VALID | _PAGE_SZ8K | _PAGE_CP |
 308                                  _PAGE_CV | _PAGE_P | _PAGE_L | _PAGE_W),
 309                         "r" (59 << 3), "r" (TLB_TAG_ACCESS),
 310                         "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS),
 311                         "i" (ASI_IMMU), "i" (ASI_ITLB_DATA_ACCESS)
 312                         : "memory");
 313         } else if (tlb_type == cheetah) {
 314                 /* Lock this into i/d tlb-0 entry 11 */
 315                 __asm__ __volatile__(
 316                         "stxa   %%g0, [%2] %3\n\t"
 317                         "stxa   %0, [%1] %4\n\t"
 318                         "membar #Sync\n\t"
 319                         "flush  %%g6\n\t"
 320                         "stxa   %%g0, [%2] %5\n\t"
 321                         "stxa   %0, [%1] %6\n\t"
 322                         "membar #Sync\n\t"
 323                         "flush  %%g6"
 324                         : : "r" (phys_page | _PAGE_VALID | _PAGE_SZ8K | _PAGE_CP |
 325                                  _PAGE_CV | _PAGE_P | _PAGE_L | _PAGE_W),
 326                         "r" ((0 << 16) | (11 << 3)), "r" (TLB_TAG_ACCESS),
 327                         "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS),
 328                         "i" (ASI_IMMU), "i" (ASI_ITLB_DATA_ACCESS)
 329                         : "memory");
 330         } else {
 331                 /* Implement me :-) */
 332                 BUG();
 333         }
 334
 335         tte_vaddr = (unsigned long) &empty_zero_page;
 336
 337         /* Spitfire Errata #32 workaround */
 338         __asm__ __volatile__("stxa      %0, [%1] %2\n\t"
 339                              "flush     %%g6"
 340                              : /* No outputs */
 341                              : "r" (0),
 342                              "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
 343
 344         if (tlb_type == spitfire)
 345                 tte_data = spitfire_get_dtlb_data(sparc64_highest_locked_tlbent());
 346         else
 347                 tte_data = cheetah_get_ldtlb_data(sparc64_highest_locked_tlbent());
 348
 349         kern_locked_tte_data = tte_data;
 350
 351         remap_func = (void *)  ((unsigned long) &prom_remap -
 352                                 (unsigned long) &prom_boot_page);
 353
 354
 355         /* Spitfire Errata #32 workaround */
 356         __asm__ __volatile__("stxa      %0, [%1] %2\n\t"
 357                              "flush     %%g6"
 358                              : /* No outputs */
 359                              : "r" (0),
 360                              "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
 361
 362         remap_func((tlb_type == spitfire ?
 363                     (spitfire_get_dtlb_data(sparc64_highest_locked_tlbent()) & _PAGE_PADDR) :
 364                     (cheetah_get_ldtlb_data(sparc64_highest_locked_tlbent()) & _PAGE_PADDR)),
 365                    (unsigned long) &empty_zero_page,
 366                    prom_get_mmu_ihandle());
 367
 368         /* Flush out that temporary mapping. */
 369         spitfire_flush_dtlb_nucleus_page(0x0);
 370         spitfire_flush_itlb_nucleus_page(0x0);
 371
 372         /* Now lock us back into the TLBs via OBP. */
 373         prom_dtlb_load(sparc64_highest_locked_tlbent(), tte_data, tte_vaddr);
 374         prom_itlb_load(sparc64_highest_locked_tlbent(), tte_data, tte_vaddr);
 375
 376         /* Re-read translations property. */
 377         if ((n = prom_getproperty(node, "translations", (char *)trans, tsz)) == -1) {
 378                 prom_printf("Couldn't get translation property\n");
 379                 prom_halt();
 380         }
 381         n = n / sizeof(*trans);
 382
 383         for (i = 0; i < n; i++) {
 384                 unsigned long vaddr = trans[i].virt;
 385                 unsigned long size = trans[i].size;
 386
 387                 if (vaddr < 0xf0000000UL) {
 388                         unsigned long avoid_start = (unsigned long) &empty_zero_page;
 389                         unsigned long avoid_end = avoid_start + (4 * 1024 * 1024);
 390
 391                         if (vaddr < avoid_start) {
 392                                 unsigned long top = vaddr + size;
 393
 394                                 if (top > avoid_start)
 395                                         top = avoid_start;
 396                                 prom_unmap(top - vaddr, vaddr);
 397                         }
 398                         if ((vaddr + size) > avoid_end) {
 399                                 unsigned long bottom = vaddr;
 400
 401                                 if (bottom < avoid_end)
 402                                         bottom = avoid_end;
 403                                 prom_unmap((vaddr + size) - bottom, bottom);
 404                         }
 405                 }
 406         }
 407
 408         prom_printf("done.\n");
 409
 410         register_prom_callbacks();
 411 }
 412
 413 /* The OBP specifications for sun4u mark 0xfffffffc00000000 and
 414  * upwards as reserved for use by the firmware (I wonder if this
 415  * will be the same on Cheetah...).  We use this virtual address
 416  * range for the VPTE table mappings of the nucleus so we need
 417  * to zap them when we enter the PROM.  -DaveM
 418  */
 419 static void __flush_nucleus_vptes(void)
 420 {
 421         unsigned long prom_reserved_base = 0xfffffffc00000000UL;
 422         int i;
 423
 424         /* Only DTLB must be checked for VPTE entries. */
 425         if (tlb_type == spitfire) {
 426                 for (i = 0; i < 63; i++) {
 427                         unsigned long tag;
 428
 429                         /* Spitfire Errata #32 workaround */
 430                         __asm__ __volatile__("stxa      %0, [%1] %2\n\t"
 431                                              "flush     %%g6"
 432                                              : /* No outputs */
 433                                              : "r" (0),
 434                                              "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
 435
 436                         tag = spitfire_get_dtlb_tag(i);
 437                         if (((tag & ~(PAGE_MASK)) == 0) &&
 438                             ((tag &  (PAGE_MASK)) >= prom_reserved_base)) {
 439                                 __asm__ __volatile__("stxa %%g0, [%0] %1"
 440                                                      : /* no outputs */
 441                                                      : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
 442                                 membar("#Sync");
 443                                 spitfire_put_dtlb_data(i, 0x0UL);
 444                                 membar("#Sync");
 445                         }
 446                 }
 447         } else if (tlb_type == cheetah) {
 448                 for (i = 0; i < 511; i++) {
 449                         unsigned long tag = cheetah_get_dtlb_tag(i);
 450
 451                         if ((tag & ~PAGE_MASK) == 0 &&
 452                             (tag & PAGE_MASK) >= prom_reserved_base) {
 453                                 __asm__ __volatile__("stxa %%g0, [%0] %1"
 454                                                      : /* no outputs */
 455                                                      : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
 456                                 membar("#Sync");
 457                                 cheetah_put_dtlb_data(i, 0x0UL);
 458                                 membar("#Sync");
 459                         }
 460                 }
 461         } else {
 462                 /* Implement me :-) */
 463                 BUG();
 464         }
 465 }
 466
 467 static int prom_ditlb_set = 0;
 468 struct prom_tlb_entry {
 469         int             tlb_ent;
 470         unsigned long   tlb_tag;
 471         unsigned long   tlb_data;
 472 };
 473 struct prom_tlb_entry prom_itlb[16], prom_dtlb[16];
 474
 475 void prom_world(int enter)
 476 {
 477         unsigned long pstate;
 478         int i;
 479
 480         if (!enter)
 481                 set_fs(current->thread.current_ds);
 482
 483         if (!prom_ditlb_set)
 484                 return;
 485
 486         /* Make sure the following runs atomically. */
 487         __asm__ __volatile__("flushw\n\t"
 488                              "rdpr      %%pstate, %0\n\t"
 489                              "wrpr      %0, %1, %%pstate"
 490                              : "=r" (pstate)
 491                              : "i" (PSTATE_IE));
 492
 493         if (enter) {
 494                 /* Kick out nucleus VPTEs. */
 495                 __flush_nucleus_vptes();
 496
 497                 /* Install PROM world. */
 498                 for (i = 0; i < 16; i++) {
 499                         if (prom_dtlb[i].tlb_ent != -1) {
 500                                 __asm__ __volatile__("stxa %0, [%1] %2"
 501                                         : : "r" (prom_dtlb[i].tlb_tag), "r" (TLB_TAG_ACCESS),
 502                                         "i" (ASI_DMMU));
 503                                 membar("#Sync");
 504                                 if (tlb_type == spitfire)
 505                                         spitfire_put_dtlb_data(prom_dtlb[i].tlb_ent,
 506                                                                prom_dtlb[i].tlb_data);
 507                                 else if (tlb_type == cheetah)
 508                                         cheetah_put_ldtlb_data(prom_dtlb[i].tlb_ent,
 509                                                                prom_dtlb[i].tlb_data);
 510                                 membar("#Sync");
 511                         }
 512                         if (prom_itlb[i].tlb_ent != -1) {
 513                                 __asm__ __volatile__("stxa %0, [%1] %2"
 514                                                      : : "r" (prom_itlb[i].tlb_tag),
 515                                                      "r" (TLB_TAG_ACCESS),
 516                                                      "i" (ASI_IMMU));
 517                                 membar("#Sync");
 518                                 if (tlb_type == spitfire)
 519                                         spitfire_put_itlb_data(prom_itlb[i].tlb_ent,
 520                                                                prom_itlb[i].tlb_data);
 521                                 else if (tlb_type == cheetah)
 522                                         cheetah_put_litlb_data(prom_itlb[i].tlb_ent,
 523                                                                prom_itlb[i].tlb_data);
 524                                 membar("#Sync");
 525                         }
 526                 }
 527         } else {
 528                 for (i = 0; i < 16; i++) {
 529                         if (prom_dtlb[i].tlb_ent != -1) {
 530                                 __asm__ __volatile__("stxa %%g0, [%0] %1"
 531                                         : : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
 532                                 membar("#Sync");
 533                                 if (tlb_type == spitfire)
 534                                         spitfire_put_dtlb_data(prom_dtlb[i].tlb_ent, 0x0UL);
 535                                 else
 536                                         cheetah_put_ldtlb_data(prom_dtlb[i].tlb_ent, 0x0UL);
 537                                 membar("#Sync");
 538                         }
 539                         if (prom_itlb[i].tlb_ent != -1) {
 540                                 __asm__ __volatile__("stxa %%g0, [%0] %1"
 541                                                      : : "r" (TLB_TAG_ACCESS),
 542                                                      "i" (ASI_IMMU));
 543                                 membar("#Sync");
 544                                 if (tlb_type == spitfire)
 545                                         spitfire_put_itlb_data(prom_itlb[i].tlb_ent, 0x0UL);
 546                                 else
 547                                         cheetah_put_litlb_data(prom_itlb[i].tlb_ent, 0x0UL);
 548                                 membar("#Sync");
 549                         }
 550                 }
 551         }
 552         __asm__ __volatile__("wrpr      %0, 0, %%pstate"
 553                              : : "r" (pstate));
 554 }
 555
 556 void inherit_locked_prom_mappings(int save_p)
 557 {
 558         int i;
 559         int dtlb_seen = 0;
 560         int itlb_seen = 0;
 561
 562         /* Fucking losing PROM has more mappings in the TLB, but
 563          * it (conveniently) fails to mention any of these in the
 564          * translations property.  The only ones that matter are
 565          * the locked PROM tlb entries, so we impose the following
 566          * irrecovable rule on the PROM, it is allowed 8 locked
 567          * entries in the ITLB and 8 in the DTLB.
 568          *
 569          * Supposedly the upper 16GB of the address space is
 570          * reserved for OBP, BUT I WISH THIS WAS DOCUMENTED
 571          * SOMEWHERE!!!!!!!!!!!!!!!!!  Furthermore the entire interface
 572          * used between the client program and the firmware on sun5
 573          * systems to coordinate mmu mappings is also COMPLETELY
 574          * UNDOCUMENTED!!!!!! Thanks S(t)un!
 575          */
 576         if (save_p) {
 577                 for (i = 0; i < 16; i++) {
 578                         prom_itlb[i].tlb_ent = -1;
 579                         prom_dtlb[i].tlb_ent = -1;
 580                 }
 581         }
 582         if (tlb_type == spitfire) {
 583                 for (i = 0; i < SPITFIRE_HIGHEST_LOCKED_TLBENT; i++) {
 584                         unsigned long data;
 585
 586                         /* Spitfire Errata #32 workaround */
 587                         __asm__ __volatile__("stxa      %0, [%1] %2\n\t"
 588                                              "flush     %%g6"
 589                                              : /* No outputs */
 590                                              : "r" (0),
 591                                              "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
 592
 593                         data = spitfire_get_dtlb_data(i);
 594                         if ((data & (_PAGE_L|_PAGE_VALID)) == (_PAGE_L|_PAGE_VALID)) {
 595                                 unsigned long tag;
 596
 597                                 /* Spitfire Errata #32 workaround */
 598                                 __asm__ __volatile__("stxa      %0, [%1] %2\n\t"
 599                                                      "flush     %%g6"
 600                                                      : /* No outputs */
 601                                                      : "r" (0),
 602                                                      "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
 603
 604                                 tag = spitfire_get_dtlb_tag(i);
 605                                 if (save_p) {
 606                                         prom_dtlb[dtlb_seen].tlb_ent = i;
 607                                         prom_dtlb[dtlb_seen].tlb_tag = tag;
 608                                         prom_dtlb[dtlb_seen].tlb_data = data;
 609                                 }
 610                                 __asm__ __volatile__("stxa %%g0, [%0] %1"
 611                                                      : : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
 612                                 membar("#Sync");
 613                                 spitfire_put_dtlb_data(i, 0x0UL);
 614                                 membar("#Sync");
 615
 616                                 dtlb_seen++;
 617                                 if (dtlb_seen > 15)
 618                                         break;
 619                         }
 620                 }
 621
 622                 for (i = 0; i < SPITFIRE_HIGHEST_LOCKED_TLBENT; i++) {
 623                         unsigned long data;
 624
 625                         /* Spitfire Errata #32 workaround */
 626                         __asm__ __volatile__("stxa      %0, [%1] %2\n\t"
 627                                              "flush     %%g6"
 628                                              : /* No outputs */
 629                                              : "r" (0),
 630                                              "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
 631
 632                         data = spitfire_get_itlb_data(i);
 633                         if ((data & (_PAGE_L|_PAGE_VALID)) == (_PAGE_L|_PAGE_VALID)) {
 634                                 unsigned long tag;
 635
 636                                 /* Spitfire Errata #32 workaround */
 637                                 __asm__ __volatile__("stxa      %0, [%1] %2\n\t"
 638                                                      "flush     %%g6"
 639                                                      : /* No outputs */
 640                                                      : "r" (0),
 641                                                      "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
 642
 643                                 tag = spitfire_get_itlb_tag(i);
 644                                 if (save_p) {
 645                                         prom_itlb[itlb_seen].tlb_ent = i;
 646                                         prom_itlb[itlb_seen].tlb_tag = tag;
 647                                         prom_itlb[itlb_seen].tlb_data = data;
 648                                 }
 649                                 __asm__ __volatile__("stxa %%g0, [%0] %1"
 650                                                      : : "r" (TLB_TAG_ACCESS), "i" (ASI_IMMU));
 651                                 membar("#Sync");
 652                                 spitfire_put_itlb_data(i, 0x0UL);
 653                                 membar("#Sync");
 654
 655                                 itlb_seen++;
 656                                 if (itlb_seen > 15)
 657                                         break;
 658                         }
 659                 }
 660         } else if (tlb_type == cheetah) {
 661                 for (i = 0; i < CHEETAH_HIGHEST_LOCKED_TLBENT; i++) {
 662                         unsigned long data;
 663
 664                         data = cheetah_get_ldtlb_data(i);
 665                         if ((data & (_PAGE_L|_PAGE_VALID)) == (_PAGE_L|_PAGE_VALID)) {
 666                                 unsigned long tag;
 667
 668                                 tag = cheetah_get_ldtlb_tag(i);
 669                                 if (save_p) {
 670                                         prom_dtlb[dtlb_seen].tlb_ent = i;
 671                                         prom_dtlb[dtlb_seen].tlb_tag = tag;
 672                                         prom_dtlb[dtlb_seen].tlb_data = data;
 673                                 }
 674                                 __asm__ __volatile__("stxa %%g0, [%0] %1"
 675                                                      : : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
 676                                 membar("#Sync");
 677                                 cheetah_put_ldtlb_data(i, 0x0UL);
 678                                 membar("#Sync");
 679
 680                                 dtlb_seen++;
 681                                 if (dtlb_seen > 15)
 682                                         break;
 683                         }
 684                 }
 685
 686                 for (i = 0; i < CHEETAH_HIGHEST_LOCKED_TLBENT; i++) {
 687                         unsigned long data;
 688
 689                         data = cheetah_get_litlb_data(i);
 690                         if ((data & (_PAGE_L|_PAGE_VALID)) == (_PAGE_L|_PAGE_VALID)) {
 691                                 unsigned long tag;
 692
 693                                 tag = cheetah_get_litlb_tag(i);
 694                                 if (save_p) {
 695                                         prom_itlb[itlb_seen].tlb_ent = i;
 696                                         prom_itlb[itlb_seen].tlb_tag = tag;
 697                                         prom_itlb[itlb_seen].tlb_data = data;
 698                                 }
 699                                 __asm__ __volatile__("stxa %%g0, [%0] %1"
 700                                                      : : "r" (TLB_TAG_ACCESS), "i" (ASI_IMMU));
 701                                 membar("#Sync");
 702                                 cheetah_put_litlb_data(i, 0x0UL);
 703                                 membar("#Sync");
 704
 705                                 itlb_seen++;
 706                                 if (itlb_seen > 15)
 707                                         break;
 708                         }
 709                 }
 710         } else {
 711                 /* Implement me :-) */
 712                 BUG();
 713         }
 714         if (save_p)
 715                 prom_ditlb_set = 1;
 716 }
 717
 718 /* Give PROM back his world, done during reboots... */
 719 void prom_reload_locked(void)
 720 {
 721         int i;
 722
 723         for (i = 0; i < 16; i++) {
 724                 if (prom_dtlb[i].tlb_ent != -1) {
 725                         __asm__ __volatile__("stxa %0, [%1] %2"
 726                                 : : "r" (prom_dtlb[i].tlb_tag), "r" (TLB_TAG_ACCESS),
 727                                 "i" (ASI_DMMU));
 728                         membar("#Sync");
 729                         if (tlb_type == spitfire)
 730                                 spitfire_put_dtlb_data(prom_dtlb[i].tlb_ent,
 731                                                        prom_dtlb[i].tlb_data);
 732                         else if (tlb_type == cheetah)
 733                                 cheetah_put_ldtlb_data(prom_dtlb[i].tlb_ent,
 734                                                       prom_dtlb[i].tlb_data);
 735                         membar("#Sync");
 736                 }
 737
 738                 if (prom_itlb[i].tlb_ent != -1) {
 739                         __asm__ __volatile__("stxa %0, [%1] %2"
 740                                              : : "r" (prom_itlb[i].tlb_tag),
 741                                              "r" (TLB_TAG_ACCESS),
 742                                              "i" (ASI_IMMU));
 743                         membar("#Sync");
 744                         if (tlb_type == spitfire)
 745                                 spitfire_put_itlb_data(prom_itlb[i].tlb_ent,
 746                                                        prom_itlb[i].tlb_data);
 747                         else
 748                                 cheetah_put_litlb_data(prom_itlb[i].tlb_ent,
 749                                                        prom_itlb[i].tlb_data);
 750                         membar("#Sync");
 751                 }
 752         }
 753 }
 754
 755 void __flush_dcache_range(unsigned long start, unsigned long end)
 756 {
 757         unsigned long va;
 758         int n = 0;
 759
 760         for (va = start; va < end; va += 32) {
 761                 spitfire_put_dcache_tag(va & 0x3fe0, 0x0);
 762                 if (++n >= 512)
 763                         break;
 764         }
 765 }
 766
 767 void __flush_cache_all(void)
 768 {
 769         unsigned long va;
 770
 771         flushw_all();
 772         for (va =  0; va < (PAGE_SIZE << 1); va += 32)
 773                 spitfire_put_icache_tag(va, 0x0);
 774 }
 775
 776 /* If not locked, zap it. */
 777 void __flush_tlb_all(void)
 778 {
 779         unsigned long pstate;
 780         int i;
 781
 782         __asm__ __volatile__("flushw\n\t"
 783                              "rdpr      %%pstate, %0\n\t"
 784                              "wrpr      %0, %1, %%pstate"
 785                              : "=r" (pstate)
 786                              : "i" (PSTATE_IE));
 787         if (tlb_type == spitfire) {
 788                 for (i = 0; i < 64; i++) {
 789                         /* Spitfire Errata #32 workaround */
 790                         __asm__ __volatile__("stxa      %0, [%1] %2\n\t"
 791                                              "flush     %%g6"
 792                                              : /* No outputs */
 793                                              : "r" (0),
 794                                              "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
 795
 796                         if (!(spitfire_get_dtlb_data(i) & _PAGE_L)) {
 797                                 __asm__ __volatile__("stxa %%g0, [%0] %1"
 798                                                      : /* no outputs */
 799                                                      : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
 800                                 membar("#Sync");
 801                                 spitfire_put_dtlb_data(i, 0x0UL);
 802                                 membar("#Sync");
 803                         }
 804
 805                         /* Spitfire Errata #32 workaround */
 806                         __asm__ __volatile__("stxa      %0, [%1] %2\n\t"
 807                                              "flush     %%g6"
 808                                              : /* No outputs */
 809                                              : "r" (0),
 810                                              "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
 811
 812                         if (!(spitfire_get_itlb_data(i) & _PAGE_L)) {
 813                                 __asm__ __volatile__("stxa %%g0, [%0] %1"
 814                                                      : /* no outputs */
 815                                                      : "r" (TLB_TAG_ACCESS), "i" (ASI_IMMU));
 816                                 membar("#Sync");
 817                                 spitfire_put_itlb_data(i, 0x0UL);
 818                                 membar("#Sync");
 819                         }
 820                 }
 821         } else if (tlb_type == cheetah) {
 822                 cheetah_flush_dtlb_all();
 823                 cheetah_flush_itlb_all();
 824         }
 825         __asm__ __volatile__("wrpr      %0, 0, %%pstate"
 826                              : : "r" (pstate));
 827 }
 828
 829 /* Caller does TLB context flushing on local CPU if necessary.
 830  *
 831  * We must be careful about boundary cases so that we never
 832  * let the user have CTX 0 (nucleus) or we ever use a CTX
 833  * version of zero (and thus NO_CONTEXT would not be caught
 834  * by version mis-match tests in mmu_context.h).
 835  */
 836 void get_new_mmu_context(struct mm_struct *mm)
 837 {
 838         unsigned long ctx, new_ctx;
 839
 840         spin_lock(&ctx_alloc_lock);
 841         ctx = CTX_HWBITS(tlb_context_cache + 1);
 842         if (ctx == 0)
 843                 ctx = 1;
 844         if (CTX_VALID(mm->context)) {
 845                 unsigned long nr = CTX_HWBITS(mm->context);
 846                 mmu_context_bmap[nr>>6] &= ~(1UL << (nr & 63));
 847         }
 848         new_ctx = find_next_zero_bit(mmu_context_bmap, 1UL << CTX_VERSION_SHIFT, ctx);
 849         if (new_ctx >= (1UL << CTX_VERSION_SHIFT)) {
 850                 new_ctx = find_next_zero_bit(mmu_context_bmap, ctx, 1);
 851                 if (new_ctx >= ctx) {
 852                         int i;
 853                         new_ctx = (tlb_context_cache & CTX_VERSION_MASK) +
 854                                 CTX_FIRST_VERSION;
 855                         if (new_ctx == 1)
 856                                 new_ctx = CTX_FIRST_VERSION;
 857
 858                         /* Don't call memset, for 16 entries that's just
 859                          * plain silly...
 860                          */
 861                         mmu_context_bmap[0] = 3;
 862                         mmu_context_bmap[1] = 0;
 863                         mmu_context_bmap[2] = 0;
 864                         mmu_context_bmap[3] = 0;
 865                         for (i = 4; i < CTX_BMAP_SLOTS; i += 4) {
 866                                 mmu_context_bmap[i + 0] = 0;
 867                                 mmu_context_bmap[i + 1] = 0;
 868                                 mmu_context_bmap[i + 2] = 0;
 869                                 mmu_context_bmap[i + 3] = 0;
 870                         }
 871                         goto out;
 872                 }
 873         }
 874         mmu_context_bmap[new_ctx>>6] |= (1UL << (new_ctx & 63));
 875         new_ctx |= (tlb_context_cache & CTX_VERSION_MASK);
 876 out:
 877         tlb_context_cache = new_ctx;
 878         spin_unlock(&ctx_alloc_lock);
 879
 880         mm->context = new_ctx;
 881 }
 882
 883 #ifndef CONFIG_SMP
 884 struct pgtable_cache_struct pgt_quicklists;
 885 #endif
 886
 887 /* For PMDs we don't care about the color, writes are
 888  * only done via Dcache which is write-thru, so non-Dcache
 889  * reads will always see correct data.
 890  */
 891 pmd_t *get_pmd_slow(pgd_t *pgd, unsigned long offset)
 892 {
 893         pmd_t *pmd;
 894
 895         pmd = (pmd_t *) __get_free_page(GFP_KERNEL);
 896         if (pmd) {
 897                 memset(pmd, 0, PAGE_SIZE);
 898                 pgd_set(pgd, pmd);
 899                 return pmd + offset;
 900         }
 901         return NULL;
 902 }
 903
 904 /* OK, we have to color these pages because during DTLB
 905  * protection faults we set the dirty bit via a non-Dcache
 906  * enabled mapping in the VPTE area.  The kernel can end
 907  * up missing the dirty bit resulting in processes crashing
 908  * _iff_ the VPTE mapping of the ptes have a virtual address
 909  * bit 13 which is different from bit 13 of the physical address.
 910  *
 911  * The sequence is:
 912  *      1) DTLB protection fault, write dirty bit into pte via VPTE
 913  *         mappings.
 914  *      2) Swapper checks pte, does not see dirty bit, frees page.
 915  *      3) Process faults back in the page, the old pre-dirtied copy
 916  *         is provided and here is the corruption.
 917  */
 918 pte_t *get_pte_slow(pmd_t *pmd, unsigned long offset, unsigned long color)
 919 {
 920         struct page *page = alloc_pages(GFP_KERNEL, 1);
 921
 922         if (page) {
 923                 unsigned long *to_free;
 924                 unsigned long paddr;
 925                 pte_t *pte;
 926
 927                 set_page_count((page + 1), 1);
 928                 paddr = (unsigned long) page_address(page);
 929                 memset((char *)paddr, 0, (PAGE_SIZE << 1));
 930
 931                 if (!color) {
 932                         pte = (pte_t *) paddr;
 933                         to_free = (unsigned long *) (paddr + PAGE_SIZE);
 934                 } else {
 935                         pte = (pte_t *) (paddr + PAGE_SIZE);
 936                         to_free = (unsigned long *) paddr;
 937                 }
 938
 939                 /* Now free the other one up, adjust cache size. */
 940                 *to_free = (unsigned long) pte_quicklist[color ^ 0x1];
 941                 pte_quicklist[color ^ 0x1] = to_free;
 942                 pgtable_cache_size++;
 943
 944                 pmd_set(pmd, pte);
 945                 return pte + offset;
 946         }
 947         return NULL;
 948 }
 949
 950 void sparc_ultra_dump_itlb(void)
 951 {
 952         int slot;
 953
 954         if (tlb_type == spitfire) {
 955                 printk ("Contents of itlb: ");
 956                 for (slot = 0; slot < 14; slot++) printk ("    ");
 957                 printk ("%2x:%016lx,%016lx\n",
 958                         0,
 959                         spitfire_get_itlb_tag(0), spitfire_get_itlb_data(0));
 960                 for (slot = 1; slot < 64; slot+=3) {
 961                         printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx %2x:%016lx,%016lx\n",
 962                                 slot,
 963                                 spitfire_get_itlb_tag(slot), spitfire_get_itlb_data(slot),
 964                                 slot+1,
 965                                 spitfire_get_itlb_tag(slot+1), spitfire_get_itlb_data(slot+1),
 966                                 slot+2,
 967                                 spitfire_get_itlb_tag(slot+2), spitfire_get_itlb_data(slot+2));
 968                 }
 969         } else if (tlb_type == cheetah) {
 970                 printk ("Contents of itlb0:\n");
 971                 for (slot = 0; slot < 16; slot+=2) {
 972                         printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
 973                                 slot,
 974                                 cheetah_get_litlb_tag(slot), cheetah_get_litlb_data(slot),
 975                                 slot+1,
 976                                 cheetah_get_litlb_tag(slot+1), cheetah_get_litlb_data(slot+1));
 977                 }
 978                 printk ("Contents of itlb2:\n");
 979                 for (slot = 0; slot < 128; slot+=2) {
 980                         printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
 981                                 slot,
 982                                 cheetah_get_itlb_tag(slot), cheetah_get_itlb_data(slot),
 983                                 slot+1,
 984                                 cheetah_get_itlb_tag(slot+1), cheetah_get_itlb_data(slot+1));
 985                 }
 986         }
 987 }
 988
 989 void sparc_ultra_dump_dtlb(void)
 990 {
 991         int slot;
 992
 993         if (tlb_type == spitfire) {
 994                 printk ("Contents of dtlb: ");
 995                 for (slot = 0; slot < 14; slot++) printk ("    ");
 996                 printk ("%2x:%016lx,%016lx\n", 0,
 997                         spitfire_get_dtlb_tag(0), spitfire_get_dtlb_data(0));
 998                 for (slot = 1; slot < 64; slot+=3) {
 999                         printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx %2x:%016lx,%016lx\n",
1000                                 slot,
1001                                 spitfire_get_dtlb_tag(slot), spitfire_get_dtlb_data(slot),
1002                                 slot+1,
1003                                 spitfire_get_dtlb_tag(slot+1), spitfire_get_dtlb_data(slot+1),
1004                                 slot+2,
1005                                 spitfire_get_dtlb_tag(slot+2), spitfire_get_dtlb_data(slot+2));
1006                 }
1007         } else if (tlb_type == cheetah) {
1008                 printk ("Contents of dtlb0:\n");
1009                 for (slot = 0; slot < 16; slot+=2) {
1010                         printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
1011                                 slot,
1012                                 cheetah_get_ldtlb_tag(slot), cheetah_get_ldtlb_data(slot),
1013                                 slot+1,
1014                                 cheetah_get_ldtlb_tag(slot+1), cheetah_get_ldtlb_data(slot+1));
1015                 }
1016                 printk ("Contents of dtlb2:\n");
1017                 for (slot = 0; slot < 512; slot+=2) {
1018                         printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
1019                                 slot,
1020                                 cheetah_get_dtlb_tag(slot), cheetah_get_dtlb_data(slot),
1021                                 slot+1,
1022                                 cheetah_get_dtlb_tag(slot+1), cheetah_get_dtlb_data(slot+1));
1023                 }
1024         }
1025 }
1026
1027 extern unsigned long cmdline_memory_size;
1028
1029 unsigned long __init bootmem_init(unsigned long *pages_avail)
1030 {
1031         unsigned long bootmap_size, start_pfn, end_pfn;
1032         unsigned long end_of_phys_memory = 0UL;
1033         unsigned long bootmap_pfn, bytes_avail, size;
1034         int i;
1035
1036         bytes_avail = 0UL;
1037         for (i = 0; sp_banks[i].num_bytes != 0; i++) {
1038                 end_of_phys_memory = sp_banks[i].base_addr +
1039                         sp_banks[i].num_bytes;
1040                 bytes_avail += sp_banks[i].num_bytes;
1041                 if (cmdline_memory_size) {
1042                         if (bytes_avail > cmdline_memory_size) {
1043                                 unsigned long slack = bytes_avail - cmdline_memory_size;
1044
1045                                 bytes_avail -= slack;
1046                                 end_of_phys_memory -= slack;
1047
1048                                 sp_banks[i].num_bytes -= slack;
1049                                 if (sp_banks[i].num_bytes == 0) {
1050                                         sp_banks[i].base_addr = 0xdeadbeef;
1051                                 } else {
1052                                         sp_banks[i+1].num_bytes = 0;
1053                                         sp_banks[i+1].base_addr = 0xdeadbeef;
1054                                 }
1055                                 break;
1056                         }
1057                 }
1058         }
1059
1060         *pages_avail = bytes_avail >> PAGE_SHIFT;
1061
1062         /* Start with page aligned address of last symbol in kernel
1063          * image.  The kernel is hard mapped below PAGE_OFFSET in a
1064          * 4MB locked TLB translation.
1065          */
1066         start_pfn  = PAGE_ALIGN((unsigned long) &_end) -
1067                 ((unsigned long) &empty_zero_page);
1068
1069         /* Adjust up to the physical address where the kernel begins. */
1070         start_pfn += phys_base;
1071
1072         /* Now shift down to get the real physical page frame number. */
1073         start_pfn >>= PAGE_SHIFT;
1074
1075         bootmap_pfn = start_pfn;
1076
1077         end_pfn = end_of_phys_memory >> PAGE_SHIFT;
1078
1079 #ifdef CONFIG_BLK_DEV_INITRD
1080         /* Now have to check initial ramdisk, so that bootmap does not overwrite it */
1081         if (sparc_ramdisk_image) {
1082                 if (sparc_ramdisk_image >= (unsigned long)&_end - 2 * PAGE_SIZE)
1083                         sparc_ramdisk_image -= KERNBASE;
1084                 initrd_start = sparc_ramdisk_image + phys_base;
1085                 initrd_end = initrd_start + sparc_ramdisk_size;
1086                 if (initrd_end > end_of_phys_memory) {
1087                         printk(KERN_CRIT "initrd extends beyond end of memory "
1088                                          "(0x%016lx > 0x%016lx)\ndisabling initrd\n",
1089                                initrd_end, end_of_phys_memory);
1090                         initrd_start = 0;
1091                 }
1092                 if (initrd_start) {
1093                         if (initrd_start >= (start_pfn << PAGE_SHIFT) &&
1094                             initrd_start < (start_pfn << PAGE_SHIFT) + 2 * PAGE_SIZE)
1095                                 bootmap_pfn = PAGE_ALIGN (initrd_end) >> PAGE_SHIFT;
1096                 }
1097         }
1098 #endif
1099         /* Initialize the boot-time allocator. */
1100         bootmap_size = init_bootmem_node(NODE_DATA(0), bootmap_pfn, phys_base>>PAGE_SHIFT, end_pfn);
1101
1102         /* Now register the available physical memory with the
1103          * allocator.
1104          */
1105         for (i = 0; sp_banks[i].num_bytes != 0; i++)
1106                 free_bootmem(sp_banks[i].base_addr,
1107                              sp_banks[i].num_bytes);
1108
1109 #ifdef CONFIG_BLK_DEV_INITRD
1110         if (initrd_start) {
1111                 size = initrd_end - initrd_start;
1112
1113                 /* Resert the initrd image area. */
1114                 reserve_bootmem(initrd_start, size);
1115                 *pages_avail -= PAGE_ALIGN(size) >> PAGE_SHIFT;
1116
1117                 initrd_start += PAGE_OFFSET;
1118                 initrd_end += PAGE_OFFSET;
1119         }
1120 #endif
1121         /* Reserve the kernel text/data/bss. */
1122         size = (start_pfn << PAGE_SHIFT) - phys_base;
1123         reserve_bootmem(phys_base, size);
1124         *pages_avail -= PAGE_ALIGN(size) >> PAGE_SHIFT;
1125
1126         /* Reserve the bootmem map.   We do not account for it
1127          * in pages_avail because we will release that memory
1128          * in free_all_bootmem.
1129          */
1130         size = bootmap_size;
1131         reserve_bootmem((bootmap_pfn << PAGE_SHIFT), size);
1132         *pages_avail -= PAGE_ALIGN(size) >> PAGE_SHIFT;
1133
1134         return end_pfn;
1135 }
1136
1137 /* paging_init() sets up the page tables */
1138
1139 extern void sun_serial_setup(void);
1140
1141 static unsigned long last_valid_pfn;
1142
1143 void __init paging_init(void)
1144 {
1145         extern pmd_t swapper_pmd_dir[1024];
1146         extern unsigned int sparc64_vpte_patchme1[1];
1147         extern unsigned int sparc64_vpte_patchme2[1];
1148         unsigned long alias_base = phys_base + PAGE_OFFSET;
1149         unsigned long second_alias_page = 0;
1150         unsigned long pt, flags, end_pfn, pages_avail;
1151         unsigned long shift = alias_base - ((unsigned long)&empty_zero_page);
1152
1153         set_bit(0, mmu_context_bmap);
1154         /* We assume physical memory starts at some 4mb multiple,
1155          * if this were not true we wouldn't boot up to this point
1156          * anyways.
1157          */
1158         pt  = phys_base | _PAGE_VALID | _PAGE_SZ4MB;
1159         pt |= _PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_L | _PAGE_W;
1160         __save_and_cli(flags);
1161         if (tlb_type == spitfire) {
1162                 __asm__ __volatile__("
1163                 stxa    %1, [%0] %3
1164                 stxa    %2, [%5] %4
1165                 membar  #Sync
1166                 flush   %%g6
1167                 nop
1168                 nop
1169                 nop"
1170                 : /* No outputs */
1171                 : "r" (TLB_TAG_ACCESS), "r" (alias_base), "r" (pt),
1172                   "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS), "r" (61 << 3)
1173                 : "memory");
1174                 if (((unsigned long)&_end) >= KERNBASE + 0x340000) {
1175                         second_alias_page = alias_base + 0x400000;
1176                         __asm__ __volatile__("
1177                         stxa    %1, [%0] %3
1178                         stxa    %2, [%5] %4
1179                         membar  #Sync
1180                         flush   %%g6
1181                         nop
1182                         nop
1183                         nop"
1184                         : /* No outputs */
1185                         : "r" (TLB_TAG_ACCESS), "r" (second_alias_page), "r" (pt + 0x400000),
1186                           "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS), "r" (60 << 3)
1187                         : "memory");
1188                 }
1189         } else if (tlb_type == cheetah) {
1190                 __asm__ __volatile__("
1191                 stxa    %1, [%0] %3
1192                 stxa    %2, [%5] %4
1193                 membar  #Sync
1194                 flush   %%g6
1195                 nop
1196                 nop
1197                 nop"
1198                 : /* No outputs */
1199                 : "r" (TLB_TAG_ACCESS), "r" (alias_base), "r" (pt),
1200                   "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS), "r" ((0<<16) | (13<<3))
1201                 : "memory");
1202                 if (((unsigned long)&_end) >= KERNBASE + 0x340000) {
1203                         second_alias_page = alias_base + 0x400000;
1204                         __asm__ __volatile__("
1205                         stxa    %1, [%0] %3
1206                         stxa    %2, [%5] %4
1207                         membar  #Sync
1208                         flush   %%g6
1209                         nop
1210                         nop
1211                         nop"
1212                         : /* No outputs */
1213                         : "r" (TLB_TAG_ACCESS), "r" (second_alias_page), "r" (pt + 0x400000),
1214                           "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS), "r" ((0<<16) | (12<<3))
1215                         : "memory");
1216                 }
1217         }
1218         __restore_flags(flags);
1219
1220         /* Now set kernel pgd to upper alias so physical page computations
1221          * work.
1222          */
1223         init_mm.pgd += ((shift) / (sizeof(pgd_t)));
1224
1225         memset(swapper_pmd_dir, 0, sizeof(swapper_pmd_dir));
1226
1227         /* Now can init the kernel/bad page tables. */
1228         pgd_set(&swapper_pg_dir[0], swapper_pmd_dir + (shift / sizeof(pgd_t)));
1229
1230         sparc64_vpte_patchme1[0] |= (pgd_val(init_mm.pgd[0]) >> 10);
1231         sparc64_vpte_patchme2[0] |= (pgd_val(init_mm.pgd[0]) & 0x3ff);
1232         flushi((long)&sparc64_vpte_patchme1[0]);
1233
1234         /* Setup bootmem... */
1235         pages_avail = 0;
1236         last_valid_pfn = end_pfn = bootmem_init(&pages_avail);
1237
1238 #ifdef CONFIG_SUN_SERIAL
1239         /* This does not logically belong here, but we need to
1240          * call it at the moment we are able to use the bootmem
1241          * allocator.
1242          */
1243         sun_serial_setup();
1244 #endif
1245
1246         /* Inherit non-locked OBP mappings. */
1247         inherit_prom_mappings();
1248
1249         /* Ok, we can use our TLB miss and window trap handlers safely.
1250          * We need to do a quick peek here to see if we are on StarFire
1251          * or not, so setup_tba can setup the IRQ globals correctly (it
1252          * needs to get the hard smp processor id correctly).
1253          */
1254         {
1255                 extern void setup_tba(int);
1256                 setup_tba(this_is_starfire);
1257         }
1258
1259         inherit_locked_prom_mappings(1);
1260
1261         /* We only created DTLB mapping of this stuff. */
1262         spitfire_flush_dtlb_nucleus_page(alias_base);
1263         if (second_alias_page)
1264                 spitfire_flush_dtlb_nucleus_page(second_alias_page);
1265
1266         flush_tlb_all();
1267
1268         {
1269                 unsigned long zones_size[MAX_NR_ZONES];
1270                 unsigned long zholes_size[MAX_NR_ZONES];
1271                 unsigned long npages;
1272                 int znum;
1273
1274                 for (znum = 0; znum < MAX_NR_ZONES; znum++)
1275                         zones_size[znum] = zholes_size[znum] = 0;
1276
1277                 npages = end_pfn - (phys_base >> PAGE_SHIFT);
1278                 zones_size[ZONE_DMA] = npages;
1279                 zholes_size[ZONE_DMA] = npages - pages_avail;
1280
1281                 free_area_init_node(0, NULL, NULL, zones_size,
1282                                     phys_base, zholes_size);
1283         }
1284
1285         device_scan();
1286 }
1287
1288 /* Ok, it seems that the prom can allocate some more memory chunks
1289  * as a side effect of some prom calls we perform during the
1290  * boot sequence.  My most likely theory is that it is from the
1291  * prom_set_traptable() call, and OBP is allocating a scratchpad
1292  * for saving client program register state etc.
1293  */
1294 void __init sort_memlist(struct linux_mlist_p1275 *thislist)
1295 {
1296         int swapi = 0;
1297         int i, mitr;
1298         unsigned long tmpaddr, tmpsize;
1299         unsigned long lowest;
1300
1301         for (i = 0; thislist[i].theres_more != 0; i++) {
1302                 lowest = thislist[i].start_adr;
1303                 for (mitr = i+1; thislist[mitr-1].theres_more != 0; mitr++)
1304                         if (thislist[mitr].start_adr < lowest) {
1305                                 lowest = thislist[mitr].start_adr;
1306                                 swapi = mitr;
1307                         }
1308                 if (lowest == thislist[i].start_adr)
1309                         continue;
1310                 tmpaddr = thislist[swapi].start_adr;
1311                 tmpsize = thislist[swapi].num_bytes;
1312                 for (mitr = swapi; mitr > i; mitr--) {
1313                         thislist[mitr].start_adr = thislist[mitr-1].start_adr;
1314                         thislist[mitr].num_bytes = thislist[mitr-1].num_bytes;
1315                 }
1316                 thislist[i].start_adr = tmpaddr;
1317                 thislist[i].num_bytes = tmpsize;
1318         }
1319 }
1320
1321 void __init rescan_sp_banks(void)
1322 {
1323         struct linux_prom64_registers memlist[64];
1324         struct linux_mlist_p1275 avail[64], *mlist;
1325         unsigned long bytes, base_paddr;
1326         int num_regs, node = prom_finddevice("/memory");
1327         int i;
1328
1329         num_regs = prom_getproperty(node, "available",
1330                                     (char *) memlist, sizeof(memlist));
1331         num_regs = (num_regs / sizeof(struct linux_prom64_registers));
1332         for (i = 0; i < num_regs; i++) {
1333                 avail[i].start_adr = memlist[i].phys_addr;
1334                 avail[i].num_bytes = memlist[i].reg_size;
1335                 avail[i].theres_more = &avail[i + 1];
1336         }
1337         avail[i - 1].theres_more = NULL;
1338         sort_memlist(avail);
1339
1340         mlist = &avail[0];
1341         i = 0;
1342         bytes = mlist->num_bytes;
1343         base_paddr = mlist->start_adr;
1344
1345         sp_banks[0].base_addr = base_paddr;
1346         sp_banks[0].num_bytes = bytes;
1347
1348         while (mlist->theres_more != NULL){
1349                 i++;
1350                 mlist = mlist->theres_more;
1351                 bytes = mlist->num_bytes;
1352                 if (i >= SPARC_PHYS_BANKS-1) {
1353                         printk ("The machine has more banks than "
1354                                 "this kernel can support\n"
1355                                 "Increase the SPARC_PHYS_BANKS "
1356                                 "setting (currently %d)\n",
1357                                 SPARC_PHYS_BANKS);
1358                         i = SPARC_PHYS_BANKS-1;
1359                         break;
1360                 }
1361
1362                 sp_banks[i].base_addr = mlist->start_adr;
1363                 sp_banks[i].num_bytes = mlist->num_bytes;
1364         }
1365
1366         i++;
1367         sp_banks[i].base_addr = 0xdeadbeefbeefdeadUL;
1368         sp_banks[i].num_bytes = 0;
1369
1370         for (i = 0; sp_banks[i].num_bytes != 0; i++)
1371                 sp_banks[i].num_bytes &= PAGE_MASK;
1372 }
1373
1374 static void __init taint_real_pages(void)
1375 {
1376         struct sparc_phys_banks saved_sp_banks[SPARC_PHYS_BANKS];
1377         int i;
1378
1379         for (i = 0; i < SPARC_PHYS_BANKS; i++) {
1380                 saved_sp_banks[i].base_addr =
1381                         sp_banks[i].base_addr;
1382                 saved_sp_banks[i].num_bytes =
1383                         sp_banks[i].num_bytes;
1384         }
1385
1386         rescan_sp_banks();
1387
1388         /* Find changes discovered in the sp_bank rescan and
1389          * reserve the lost portions in the bootmem maps.
1390          */
1391         for (i = 0; saved_sp_banks[i].num_bytes; i++) {
1392                 unsigned long old_start, old_end;
1393
1394                 old_start = saved_sp_banks[i].base_addr;
1395                 old_end = old_start +
1396                         saved_sp_banks[i].num_bytes;
1397                 while (old_start < old_end) {
1398                         int n;
1399
1400                         for (n = 0; sp_banks[n].num_bytes; n++) {
1401                                 unsigned long new_start, new_end;
1402
1403                                 new_start = sp_banks[n].base_addr;
1404                                 new_end = new_start + sp_banks[n].num_bytes;
1405
1406                                 if (new_start <= old_start &&
1407                                     new_end >= (old_start + PAGE_SIZE)) {
1408                                         set_bit (old_start >> 22,
1409                                                  sparc64_valid_addr_bitmap);
1410                                         goto do_next_page;
1411                                 }
1412                         }
1413                         reserve_bootmem(old_start, PAGE_SIZE);
1414
1415                 do_next_page:
1416                         old_start += PAGE_SIZE;
1417                 }
1418         }
1419 }
1420
1421 void __init mem_init(void)
1422 {
1423         unsigned long codepages, datapages, initpages;
1424         unsigned long addr, last;
1425         int i;
1426
1427         i = last_valid_pfn >> ((22 - PAGE_SHIFT) + 6);
1428         i += 1;
1429         sparc64_valid_addr_bitmap = (unsigned long *)
1430                 __alloc_bootmem(i << 3, SMP_CACHE_BYTES, 0UL);
1431         if (sparc64_valid_addr_bitmap == NULL) {
1432                 prom_printf("mem_init: Cannot alloc valid_addr_bitmap.\n");
1433                 prom_halt();
1434         }
1435         memset(sparc64_valid_addr_bitmap, 0, i << 3);
1436
1437         addr = PAGE_OFFSET + phys_base;
1438         last = PAGE_ALIGN((unsigned long)&_end) -
1439                 ((unsigned long) &empty_zero_page);
1440         last += PAGE_OFFSET + phys_base;
1441         while (addr < last) {
1442                 set_bit(__pa(addr) >> 22, sparc64_valid_addr_bitmap);
1443                 addr += PAGE_SIZE;
1444         }
1445
1446         taint_real_pages();
1447
1448         max_mapnr = last_valid_pfn - (phys_base >> PAGE_SHIFT);
1449         high_memory = __va(last_valid_pfn << PAGE_SHIFT);
1450
1451         num_physpages = free_all_bootmem();
1452         codepages = (((unsigned long) &etext) - ((unsigned long)&_start));
1453         codepages = PAGE_ALIGN(codepages) >> PAGE_SHIFT;
1454         datapages = (((unsigned long) &edata) - ((unsigned long)&etext));
1455         datapages = PAGE_ALIGN(datapages) >> PAGE_SHIFT;
1456         initpages = (((unsigned long) &__init_end) - ((unsigned long) &__init_begin));
1457         initpages = PAGE_ALIGN(initpages) >> PAGE_SHIFT;
1458
1459 #ifndef CONFIG_SMP
1460         {
1461                 /* Put empty_pg_dir on pgd_quicklist */
1462                 extern pgd_t empty_pg_dir[1024];
1463                 unsigned long addr = (unsigned long)empty_pg_dir;
1464                 unsigned long alias_base = phys_base + PAGE_OFFSET -
1465                         (long)(&empty_zero_page);
1466
1467                 memset(empty_pg_dir, 0, sizeof(empty_pg_dir));
1468                 addr += alias_base;
1469                 free_pgd_fast((pgd_t *)addr);
1470                 num_physpages++;
1471         }
1472 #endif
1473
1474         printk("Memory: %uk available (%ldk kernel code, %ldk data, %ldk init) [%016lx,%016lx]\n",
1475                nr_free_pages() << (PAGE_SHIFT-10),
1476                codepages << (PAGE_SHIFT-10),
1477                datapages << (PAGE_SHIFT-10),
1478                initpages << (PAGE_SHIFT-10),
1479                PAGE_OFFSET, (last_valid_pfn << PAGE_SHIFT));
1480 }
1481
1482 void free_initmem (void)
1483 {
1484         unsigned long addr;
1485
1486         addr = (unsigned long)(&__init_begin);
1487         for (; addr < (unsigned long)(&__init_end); addr += PAGE_SIZE) {
1488                 unsigned long page;
1489                 struct page *p;
1490
1491                 page = (addr +
1492                         ((unsigned long) __va(phys_base)) -
1493                         ((unsigned long) &empty_zero_page));
1494                 p = virt_to_page(page);
1495
1496                 ClearPageReserved(p);
1497                 set_page_count(p, 1);
1498                 __free_page(p);
1499                 num_physpages++;
1500         }
1501 }
1502
1503 #ifdef CONFIG_BLK_DEV_INITRD
1504 void free_initrd_mem(unsigned long start, unsigned long end)
1505 {
1506         if (start < end)
1507                 printk ("Freeing initrd memory: %ldk freed\n", (end - start) >> 10);
1508         for (; start < end; start += PAGE_SIZE) {
1509                 struct page *p = virt_to_page(start);
1510
1511                 ClearPageReserved(p);
1512                 set_page_count(p, 1);
1513                 __free_page(p);
1514                 num_physpages++;
1515         }
1516 }
1517 #endif
1518
1519 void si_meminfo(struct sysinfo *val)
1520 {
1521         val->totalram = num_physpages;
1522         val->sharedram = 0;
1523         val->freeram = nr_free_pages();
1524         val->bufferram = atomic_read(&buffermem_pages);
1525
1526         /* These are always zero on Sparc64. */
1527         val->totalhigh = 0;
1528         val->freehigh = 0;
1529
1530         val->mem_unit = PAGE_SIZE;
1531 }