arch/sparc/mm/init_64.c

   1 /*
   2  *  arch/sparc64/mm/init.c
   3  *
   4  *  Copyright (C) 1996-1999 David S. Miller (davem@caip.rutgers.edu)
   5  *  Copyright (C) 1997-1999 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kernel.h>
  10 #include <linux/sched.h>
  11 #include <linux/string.h>
  12 #include <linux/init.h>
  13 #include <linux/bootmem.h>
  14 #include <linux/mm.h>
  15 #include <linux/hugetlb.h>
  16 #include <linux/initrd.h>
  17 #include <linux/swap.h>
  18 #include <linux/pagemap.h>
  19 #include <linux/poison.h>
  20 #include <linux/fs.h>
  21 #include <linux/seq_file.h>
  22 #include <linux/kprobes.h>
  23 #include <linux/cache.h>
  24 #include <linux/sort.h>
  25 #include <linux/percpu.h>
  26 #include <linux/memblock.h>
  27 #include <linux/mmzone.h>
  28 #include <linux/gfp.h>
  29
  30 #include <asm/head.h>
  31 #include <asm/page.h>
  32 #include <asm/pgalloc.h>
  33 #include <asm/pgtable.h>
  34 #include <asm/oplib.h>
  35 #include <asm/iommu.h>
  36 #include <asm/io.h>
  37 #include <asm/uaccess.h>
  38 #include <asm/mmu_context.h>
  39 #include <asm/tlbflush.h>
  40 #include <asm/dma.h>
  41 #include <asm/starfire.h>
  42 #include <asm/tlb.h>
  43 #include <asm/spitfire.h>
  44 #include <asm/sections.h>
  45 #include <asm/tsb.h>
  46 #include <asm/hypervisor.h>
  47 #include <asm/prom.h>
  48 #include <asm/mdesc.h>
  49 #include <asm/cpudata.h>
  50 #include <asm/irq.h>
  51
  52 #include "init_64.h"
  53
  54 unsigned long kern_linear_pte_xor[4] __read_mostly;
  55
  56 /* A bitmap, two bits for every 256MB of physical memory.  These two
  57  * bits determine what page size we use for kernel linear
  58  * translations.  They form an index into kern_linear_pte_xor[].  The
  59  * value in the indexed slot is XOR'd with the TLB miss virtual
  60  * address to form the resulting TTE.  The mapping is:
  61  *
  62  *      0       ==>     4MB
  63  *      1       ==>     256MB
  64  *      2       ==>     2GB
  65  *      3       ==>     16GB
  66  *
  67  * All sun4v chips support 256MB pages.  Only SPARC-T4 and later
  68  * support 2GB pages, and hopefully future cpus will support the 16GB
  69  * pages as well.  For slots 2 and 3, we encode a 256MB TTE xor there
  70  * if these larger page sizes are not supported by the cpu.
  71  *
  72  * It would be nice to determine this from the machine description
  73  * 'cpu' properties, but we need to have this table setup before the
  74  * MDESC is initialized.
  75  */
  76 unsigned long kpte_linear_bitmap[KPTE_BITMAP_BYTES / sizeof(unsigned long)];
  77
  78 #ifndef CONFIG_DEBUG_PAGEALLOC
  79 /* A special kernel TSB for 4MB, 256MB, 2GB and 16GB linear mappings.
  80  * Space is allocated for this right after the trap table in
  81  * arch/sparc64/kernel/head.S
  82  */
  83 extern struct tsb swapper_4m_tsb[KERNEL_TSB4M_NENTRIES];
  84 #endif
  85
  86 static unsigned long cpu_pgsz_mask;
  87
  88 #define MAX_BANKS       32
  89
  90 static struct linux_prom64_registers pavail[MAX_BANKS] __devinitdata;
  91 static int pavail_ents __devinitdata;
  92
  93 static int cmp_p64(const void *a, const void *b)
  94 {
  95         const struct linux_prom64_registers *x = a, *y = b;
  96
  97         if (x->phys_addr > y->phys_addr)
  98                 return 1;
  99         if (x->phys_addr < y->phys_addr)
 100                 return -1;
 101         return 0;
 102 }
 103
 104 static void __init read_obp_memory(const char *property,
 105                                    struct linux_prom64_registers *regs,
 106                                    int *num_ents)
 107 {
 108         phandle node = prom_finddevice("/memory");
 109         int prop_size = prom_getproplen(node, property);
 110         int ents, ret, i;
 111
 112         ents = prop_size / sizeof(struct linux_prom64_registers);
 113         if (ents > MAX_BANKS) {
 114                 prom_printf("The machine has more %s property entries than "
 115                             "this kernel can support (%d).\n",
 116                             property, MAX_BANKS);
 117                 prom_halt();
 118         }
 119
 120         ret = prom_getproperty(node, property, (char *) regs, prop_size);
 121         if (ret == -1) {
 122                 prom_printf("Couldn't get %s property from /memory.\n");
 123                 prom_halt();
 124         }
 125
 126         /* Sanitize what we got from the firmware, by page aligning
 127          * everything.
 128          */
 129         for (i = 0; i < ents; i++) {
 130                 unsigned long base, size;
 131
 132                 base = regs[i].phys_addr;
 133                 size = regs[i].reg_size;
 134
 135                 size &= PAGE_MASK;
 136                 if (base & ~PAGE_MASK) {
 137                         unsigned long new_base = PAGE_ALIGN(base);
 138
 139                         size -= new_base - base;
 140                         if ((long) size < 0L)
 141                                 size = 0UL;
 142                         base = new_base;
 143                 }
 144                 if (size == 0UL) {
 145                         /* If it is empty, simply get rid of it.
 146                          * This simplifies the logic of the other
 147                          * functions that process these arrays.
 148                          */
 149                         memmove(&regs[i], &regs[i + 1],
 150                                 (ents - i - 1) * sizeof(regs[0]));
 151                         i--;
 152                         ents--;
 153                         continue;
 154                 }
 155                 regs[i].phys_addr = base;
 156                 regs[i].reg_size = size;
 157         }
 158
 159         *num_ents = ents;
 160
 161         sort(regs, ents, sizeof(struct linux_prom64_registers),
 162              cmp_p64, NULL);
 163 }
 164
 165 unsigned long sparc64_valid_addr_bitmap[VALID_ADDR_BITMAP_BYTES /
 166                                         sizeof(unsigned long)];
 167 EXPORT_SYMBOL(sparc64_valid_addr_bitmap);
 168
 169 /* Kernel physical address base and size in bytes.  */
 170 unsigned long kern_base __read_mostly;
 171 unsigned long kern_size __read_mostly;
 172
 173 /* Initial ramdisk setup */
 174 extern unsigned long sparc_ramdisk_image64;
 175 extern unsigned int sparc_ramdisk_image;
 176 extern unsigned int sparc_ramdisk_size;
 177
 178 struct page *mem_map_zero __read_mostly;
 179 EXPORT_SYMBOL(mem_map_zero);
 180
 181 unsigned int sparc64_highest_unlocked_tlb_ent __read_mostly;
 182
 183 unsigned long sparc64_kern_pri_context __read_mostly;
 184 unsigned long sparc64_kern_pri_nuc_bits __read_mostly;
 185 unsigned long sparc64_kern_sec_context __read_mostly;
 186
 187 int num_kernel_image_mappings;
 188
 189 #ifdef CONFIG_DEBUG_DCFLUSH
 190 atomic_t dcpage_flushes = ATOMIC_INIT(0);
 191 #ifdef CONFIG_SMP
 192 atomic_t dcpage_flushes_xcall = ATOMIC_INIT(0);
 193 #endif
 194 #endif
 195
 196 inline void flush_dcache_page_impl(struct page *page)
 197 {
 198         BUG_ON(tlb_type == hypervisor);
 199 #ifdef CONFIG_DEBUG_DCFLUSH
 200         atomic_inc(&dcpage_flushes);
 201 #endif
 202
 203 #ifdef DCACHE_ALIASING_POSSIBLE
 204         __flush_dcache_page(page_address(page),
 205                             ((tlb_type == spitfire) &&
 206                              page_mapping(page) != NULL));
 207 #else
 208         if (page_mapping(page) != NULL &&
 209             tlb_type == spitfire)
 210                 __flush_icache_page(__pa(page_address(page)));
 211 #endif
 212 }
 213
 214 #define PG_dcache_dirty         PG_arch_1
 215 #define PG_dcache_cpu_shift     32UL
 216 #define PG_dcache_cpu_mask      \
 217         ((1UL<<ilog2(roundup_pow_of_two(NR_CPUS)))-1UL)
 218
 219 #define dcache_dirty_cpu(page) \
 220         (((page)->flags >> PG_dcache_cpu_shift) & PG_dcache_cpu_mask)
 221
 222 static inline void set_dcache_dirty(struct page *page, int this_cpu)
 223 {
 224         unsigned long mask = this_cpu;
 225         unsigned long non_cpu_bits;
 226
 227         non_cpu_bits = ~(PG_dcache_cpu_mask << PG_dcache_cpu_shift);
 228         mask = (mask << PG_dcache_cpu_shift) | (1UL << PG_dcache_dirty);
 229
 230         __asm__ __volatile__("1:\n\t"
 231                              "ldx       [%2], %%g7\n\t"
 232                              "and       %%g7, %1, %%g1\n\t"
 233                              "or        %%g1, %0, %%g1\n\t"
 234                              "casx      [%2], %%g7, %%g1\n\t"
 235                              "cmp       %%g7, %%g1\n\t"
 236                              "bne,pn    %%xcc, 1b\n\t"
 237                              " nop"
 238                              : /* no outputs */
 239                              : "r" (mask), "r" (non_cpu_bits), "r" (&page->flags)
 240                              : "g1", "g7");
 241 }
 242
 243 static inline void clear_dcache_dirty_cpu(struct page *page, unsigned long cpu)
 244 {
 245         unsigned long mask = (1UL << PG_dcache_dirty);
 246
 247         __asm__ __volatile__("! test_and_clear_dcache_dirty\n"
 248                              "1:\n\t"
 249                              "ldx       [%2], %%g7\n\t"
 250                              "srlx      %%g7, %4, %%g1\n\t"
 251                              "and       %%g1, %3, %%g1\n\t"
 252                              "cmp       %%g1, %0\n\t"
 253                              "bne,pn    %%icc, 2f\n\t"
 254                              " andn     %%g7, %1, %%g1\n\t"
 255                              "casx      [%2], %%g7, %%g1\n\t"
 256                              "cmp       %%g7, %%g1\n\t"
 257                              "bne,pn    %%xcc, 1b\n\t"
 258                              " nop\n"
 259                              "2:"
 260                              : /* no outputs */
 261                              : "r" (cpu), "r" (mask), "r" (&page->flags),
 262                                "i" (PG_dcache_cpu_mask),
 263                                "i" (PG_dcache_cpu_shift)
 264                              : "g1", "g7");
 265 }
 266
 267 static inline void tsb_insert(struct tsb *ent, unsigned long tag, unsigned long pte)
 268 {
 269         unsigned long tsb_addr = (unsigned long) ent;
 270
 271         if (tlb_type == cheetah_plus || tlb_type == hypervisor)
 272                 tsb_addr = __pa(tsb_addr);
 273
 274         __tsb_insert(tsb_addr, tag, pte);
 275 }
 276
 277 unsigned long _PAGE_ALL_SZ_BITS __read_mostly;
 278 unsigned long _PAGE_SZBITS __read_mostly;
 279
 280 static void flush_dcache(unsigned long pfn)
 281 {
 282         struct page *page;
 283
 284         page = pfn_to_page(pfn);
 285         if (page) {
 286                 unsigned long pg_flags;
 287
 288                 pg_flags = page->flags;
 289                 if (pg_flags & (1UL << PG_dcache_dirty)) {
 290                         int cpu = ((pg_flags >> PG_dcache_cpu_shift) &
 291                                    PG_dcache_cpu_mask);
 292                         int this_cpu = get_cpu();
 293
 294                         /* This is just to optimize away some function calls
 295                          * in the SMP case.
 296                          */
 297                         if (cpu == this_cpu)
 298                                 flush_dcache_page_impl(page);
 299                         else
 300                                 smp_flush_dcache_page_impl(page, cpu);
 301
 302                         clear_dcache_dirty_cpu(page, cpu);
 303
 304                         put_cpu();
 305                 }
 306         }
 307 }
 308
 309 void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep)
 310 {
 311         struct mm_struct *mm;
 312         struct tsb *tsb;
 313         unsigned long tag, flags;
 314         unsigned long tsb_index, tsb_hash_shift;
 315         pte_t pte = *ptep;
 316
 317         if (tlb_type != hypervisor) {
 318                 unsigned long pfn = pte_pfn(pte);
 319
 320                 if (pfn_valid(pfn))
 321                         flush_dcache(pfn);
 322         }
 323
 324         mm = vma->vm_mm;
 325
 326         tsb_index = MM_TSB_BASE;
 327         tsb_hash_shift = PAGE_SHIFT;
 328
 329         spin_lock_irqsave(&mm->context.lock, flags);
 330
 331 #ifdef CONFIG_HUGETLB_PAGE
 332         if (mm->context.tsb_block[MM_TSB_HUGE].tsb != NULL) {
 333                 if ((tlb_type == hypervisor &&
 334                      (pte_val(pte) & _PAGE_SZALL_4V) == _PAGE_SZHUGE_4V) ||
 335                     (tlb_type != hypervisor &&
 336                      (pte_val(pte) & _PAGE_SZALL_4U) == _PAGE_SZHUGE_4U)) {
 337                         tsb_index = MM_TSB_HUGE;
 338                         tsb_hash_shift = HPAGE_SHIFT;
 339                 }
 340         }
 341 #endif
 342
 343         tsb = mm->context.tsb_block[tsb_index].tsb;
 344         tsb += ((address >> tsb_hash_shift) &
 345                 (mm->context.tsb_block[tsb_index].tsb_nentries - 1UL));
 346         tag = (address >> 22UL);
 347         tsb_insert(tsb, tag, pte_val(pte));
 348
 349         spin_unlock_irqrestore(&mm->context.lock, flags);
 350 }
 351
 352 void flush_dcache_page(struct page *page)
 353 {
 354         struct address_space *mapping;
 355         int this_cpu;
 356
 357         if (tlb_type == hypervisor)
 358                 return;
 359
 360         /* Do not bother with the expensive D-cache flush if it
 361          * is merely the zero page.  The 'bigcore' testcase in GDB
 362          * causes this case to run millions of times.
 363          */
 364         if (page == ZERO_PAGE(0))
 365                 return;
 366
 367         this_cpu = get_cpu();
 368
 369         mapping = page_mapping(page);
 370         if (mapping && !mapping_mapped(mapping)) {
 371                 int dirty = test_bit(PG_dcache_dirty, &page->flags);
 372                 if (dirty) {
 373                         int dirty_cpu = dcache_dirty_cpu(page);
 374
 375                         if (dirty_cpu == this_cpu)
 376                                 goto out;
 377                         smp_flush_dcache_page_impl(page, dirty_cpu);
 378                 }
 379                 set_dcache_dirty(page, this_cpu);
 380         } else {
 381                 /* We could delay the flush for the !page_mapping
 382                  * case too.  But that case is for exec env/arg
 383                  * pages and those are %99 certainly going to get
 384                  * faulted into the tlb (and thus flushed) anyways.
 385                  */
 386                 flush_dcache_page_impl(page);
 387         }
 388
 389 out:
 390         put_cpu();
 391 }
 392 EXPORT_SYMBOL(flush_dcache_page);
 393
 394 void __kprobes flush_icache_range(unsigned long start, unsigned long end)
 395 {
 396         /* Cheetah and Hypervisor platform cpus have coherent I-cache. */
 397         if (tlb_type == spitfire) {
 398                 unsigned long kaddr;
 399
 400                 /* This code only runs on Spitfire cpus so this is
 401                  * why we can assume _PAGE_PADDR_4U.
 402                  */
 403                 for (kaddr = start; kaddr < end; kaddr += PAGE_SIZE) {
 404                         unsigned long paddr, mask = _PAGE_PADDR_4U;
 405
 406                         if (kaddr >= PAGE_OFFSET)
 407                                 paddr = kaddr & mask;
 408                         else {
 409                                 pgd_t *pgdp = pgd_offset_k(kaddr);
 410                                 pud_t *pudp = pud_offset(pgdp, kaddr);
 411                                 pmd_t *pmdp = pmd_offset(pudp, kaddr);
 412                                 pte_t *ptep = pte_offset_kernel(pmdp, kaddr);
 413
 414                                 paddr = pte_val(*ptep) & mask;
 415                         }
 416                         __flush_icache_page(paddr);
 417                 }
 418         }
 419 }
 420 EXPORT_SYMBOL(flush_icache_range);
 421
 422 void mmu_info(struct seq_file *m)
 423 {
 424         static const char *pgsz_strings[] = {
 425                 "8K", "64K", "512K", "4MB", "32MB",
 426                 "256MB", "2GB", "16GB",
 427         };
 428         int i, printed;
 429
 430         if (tlb_type == cheetah)
 431                 seq_printf(m, "MMU Type\t: Cheetah\n");
 432         else if (tlb_type == cheetah_plus)
 433                 seq_printf(m, "MMU Type\t: Cheetah+\n");
 434         else if (tlb_type == spitfire)
 435                 seq_printf(m, "MMU Type\t: Spitfire\n");
 436         else if (tlb_type == hypervisor)
 437                 seq_printf(m, "MMU Type\t: Hypervisor (sun4v)\n");
 438         else
 439                 seq_printf(m, "MMU Type\t: ???\n");
 440
 441         seq_printf(m, "MMU PGSZs\t: ");
 442         printed = 0;
 443         for (i = 0; i < ARRAY_SIZE(pgsz_strings); i++) {
 444                 if (cpu_pgsz_mask & (1UL << i)) {
 445                         seq_printf(m, "%s%s",
 446                                    printed ? "," : "", pgsz_strings[i]);
 447                         printed++;
 448                 }
 449         }
 450         seq_putc(m, '\n');
 451
 452 #ifdef CONFIG_DEBUG_DCFLUSH
 453         seq_printf(m, "DCPageFlushes\t: %d\n",
 454                    atomic_read(&dcpage_flushes));
 455 #ifdef CONFIG_SMP
 456         seq_printf(m, "DCPageFlushesXC\t: %d\n",
 457                    atomic_read(&dcpage_flushes_xcall));
 458 #endif /* CONFIG_SMP */
 459 #endif /* CONFIG_DEBUG_DCFLUSH */
 460 }
 461
 462 struct linux_prom_translation prom_trans[512] __read_mostly;
 463 unsigned int prom_trans_ents __read_mostly;
 464
 465 unsigned long kern_locked_tte_data;
 466
 467 /* The obp translations are saved based on 8k pagesize, since obp can
 468  * use a mixture of pagesizes. Misses to the LOW_OBP_ADDRESS ->
 469  * HI_OBP_ADDRESS range are handled in ktlb.S.
 470  */
 471 static inline int in_obp_range(unsigned long vaddr)
 472 {
 473         return (vaddr >= LOW_OBP_ADDRESS &&
 474                 vaddr < HI_OBP_ADDRESS);
 475 }
 476
 477 static int cmp_ptrans(const void *a, const void *b)
 478 {
 479         const struct linux_prom_translation *x = a, *y = b;
 480
 481         if (x->virt > y->virt)
 482                 return 1;
 483         if (x->virt < y->virt)
 484                 return -1;
 485         return 0;
 486 }
 487
 488 /* Read OBP translations property into 'prom_trans[]'.  */
 489 static void __init read_obp_translations(void)
 490 {
 491         int n, node, ents, first, last, i;
 492
 493         node = prom_finddevice("/virtual-memory");
 494         n = prom_getproplen(node, "translations");
 495         if (unlikely(n == 0 || n == -1)) {
 496                 prom_printf("prom_mappings: Couldn't get size.\n");
 497                 prom_halt();
 498         }
 499         if (unlikely(n > sizeof(prom_trans))) {
 500                 prom_printf("prom_mappings: Size %Zd is too big.\n", n);
 501                 prom_halt();
 502         }
 503
 504         if ((n = prom_getproperty(node, "translations",
 505                                   (char *)&prom_trans[0],
 506                                   sizeof(prom_trans))) == -1) {
 507                 prom_printf("prom_mappings: Couldn't get property.\n");
 508                 prom_halt();
 509         }
 510
 511         n = n / sizeof(struct linux_prom_translation);
 512
 513         ents = n;
 514
 515         sort(prom_trans, ents, sizeof(struct linux_prom_translation),
 516              cmp_ptrans, NULL);
 517
 518         /* Now kick out all the non-OBP entries.  */
 519         for (i = 0; i < ents; i++) {
 520                 if (in_obp_range(prom_trans[i].virt))
 521                         break;
 522         }
 523         first = i;
 524         for (; i < ents; i++) {
 525                 if (!in_obp_range(prom_trans[i].virt))
 526                         break;
 527         }
 528         last = i;
 529
 530         for (i = 0; i < (last - first); i++) {
 531                 struct linux_prom_translation *src = &prom_trans[i + first];
 532                 struct linux_prom_translation *dest = &prom_trans[i];
 533
 534                 *dest = *src;
 535         }
 536         for (; i < ents; i++) {
 537                 struct linux_prom_translation *dest = &prom_trans[i];
 538                 dest->virt = dest->size = dest->data = 0x0UL;
 539         }
 540
 541         prom_trans_ents = last - first;
 542
 543         if (tlb_type == spitfire) {
 544                 /* Clear diag TTE bits. */
 545                 for (i = 0; i < prom_trans_ents; i++)
 546                         prom_trans[i].data &= ~0x0003fe0000000000UL;
 547         }
 548
 549         /* Force execute bit on.  */
 550         for (i = 0; i < prom_trans_ents; i++)
 551                 prom_trans[i].data |= (tlb_type == hypervisor ?
 552                                        _PAGE_EXEC_4V : _PAGE_EXEC_4U);
 553 }
 554
 555 static void __init hypervisor_tlb_lock(unsigned long vaddr,
 556                                        unsigned long pte,
 557                                        unsigned long mmu)
 558 {
 559         unsigned long ret = sun4v_mmu_map_perm_addr(vaddr, 0, pte, mmu);
 560
 561         if (ret != 0) {
 562                 prom_printf("hypervisor_tlb_lock[%lx:%lx:%lx:%lx]: "
 563                             "errors with %lx\n", vaddr, 0, pte, mmu, ret);
 564                 prom_halt();
 565         }
 566 }
 567
 568 static unsigned long kern_large_tte(unsigned long paddr);
 569
 570 static void __init remap_kernel(void)
 571 {
 572         unsigned long phys_page, tte_vaddr, tte_data;
 573         int i, tlb_ent = sparc64_highest_locked_tlbent();
 574
 575         tte_vaddr = (unsigned long) KERNBASE;
 576         phys_page = (prom_boot_mapping_phys_low >> 22UL) << 22UL;
 577         tte_data = kern_large_tte(phys_page);
 578
 579         kern_locked_tte_data = tte_data;
 580
 581         /* Now lock us into the TLBs via Hypervisor or OBP. */
 582         if (tlb_type == hypervisor) {
 583                 for (i = 0; i < num_kernel_image_mappings; i++) {
 584                         hypervisor_tlb_lock(tte_vaddr, tte_data, HV_MMU_DMMU);
 585                         hypervisor_tlb_lock(tte_vaddr, tte_data, HV_MMU_IMMU);
 586                         tte_vaddr += 0x400000;
 587                         tte_data += 0x400000;
 588                 }
 589         } else {
 590                 for (i = 0; i < num_kernel_image_mappings; i++) {
 591                         prom_dtlb_load(tlb_ent - i, tte_data, tte_vaddr);
 592                         prom_itlb_load(tlb_ent - i, tte_data, tte_vaddr);
 593                         tte_vaddr += 0x400000;
 594                         tte_data += 0x400000;
 595                 }
 596                 sparc64_highest_unlocked_tlb_ent = tlb_ent - i;
 597         }
 598         if (tlb_type == cheetah_plus) {
 599                 sparc64_kern_pri_context = (CTX_CHEETAH_PLUS_CTX0 |
 600                                             CTX_CHEETAH_PLUS_NUC);
 601                 sparc64_kern_pri_nuc_bits = CTX_CHEETAH_PLUS_NUC;
 602                 sparc64_kern_sec_context = CTX_CHEETAH_PLUS_CTX0;
 603         }
 604 }
 605
 606
 607 static void __init inherit_prom_mappings(void)
 608 {
 609         /* Now fixup OBP's idea about where we really are mapped. */
 610         printk("Remapping the kernel... ");
 611         remap_kernel();
 612         printk("done.\n");
 613 }
 614
 615 void prom_world(int enter)
 616 {
 617         if (!enter)
 618                 set_fs((mm_segment_t) { get_thread_current_ds() });
 619
 620         __asm__ __volatile__("flushw");
 621 }
 622
 623 void __flush_dcache_range(unsigned long start, unsigned long end)
 624 {
 625         unsigned long va;
 626
 627         if (tlb_type == spitfire) {
 628                 int n = 0;
 629
 630                 for (va = start; va < end; va += 32) {
 631                         spitfire_put_dcache_tag(va & 0x3fe0, 0x0);
 632                         if (++n >= 512)
 633                                 break;
 634                 }
 635         } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
 636                 start = __pa(start);
 637                 end = __pa(end);
 638                 for (va = start; va < end; va += 32)
 639                         __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
 640                                              "membar #Sync"
 641                                              : /* no outputs */
 642                                              : "r" (va),
 643                                                "i" (ASI_DCACHE_INVALIDATE));
 644         }
 645 }
 646 EXPORT_SYMBOL(__flush_dcache_range);
 647
 648 /* get_new_mmu_context() uses "cache + 1".  */
 649 DEFINE_SPINLOCK(ctx_alloc_lock);
 650 unsigned long tlb_context_cache = CTX_FIRST_VERSION - 1;
 651 #define MAX_CTX_NR      (1UL << CTX_NR_BITS)
 652 #define CTX_BMAP_SLOTS  BITS_TO_LONGS(MAX_CTX_NR)
 653 DECLARE_BITMAP(mmu_context_bmap, MAX_CTX_NR);
 654
 655 /* Caller does TLB context flushing on local CPU if necessary.
 656  * The caller also ensures that CTX_VALID(mm->context) is false.
 657  *
 658  * We must be careful about boundary cases so that we never
 659  * let the user have CTX 0 (nucleus) or we ever use a CTX
 660  * version of zero (and thus NO_CONTEXT would not be caught
 661  * by version mis-match tests in mmu_context.h).
 662  *
 663  * Always invoked with interrupts disabled.
 664  */
 665 void get_new_mmu_context(struct mm_struct *mm)
 666 {
 667         unsigned long ctx, new_ctx;
 668         unsigned long orig_pgsz_bits;
 669         unsigned long flags;
 670         int new_version;
 671
 672         spin_lock_irqsave(&ctx_alloc_lock, flags);
 673         orig_pgsz_bits = (mm->context.sparc64_ctx_val & CTX_PGSZ_MASK);
 674         ctx = (tlb_context_cache + 1) & CTX_NR_MASK;
 675         new_ctx = find_next_zero_bit(mmu_context_bmap, 1 << CTX_NR_BITS, ctx);
 676         new_version = 0;
 677         if (new_ctx >= (1 << CTX_NR_BITS)) {
 678                 new_ctx = find_next_zero_bit(mmu_context_bmap, ctx, 1);
 679                 if (new_ctx >= ctx) {
 680                         int i;
 681                         new_ctx = (tlb_context_cache & CTX_VERSION_MASK) +
 682                                 CTX_FIRST_VERSION;
 683                         if (new_ctx == 1)
 684                                 new_ctx = CTX_FIRST_VERSION;
 685
 686                         /* Don't call memset, for 16 entries that's just
 687                          * plain silly...
 688                          */
 689                         mmu_context_bmap[0] = 3;
 690                         mmu_context_bmap[1] = 0;
 691                         mmu_context_bmap[2] = 0;
 692                         mmu_context_bmap[3] = 0;
 693                         for (i = 4; i < CTX_BMAP_SLOTS; i += 4) {
 694                                 mmu_context_bmap[i + 0] = 0;
 695                                 mmu_context_bmap[i + 1] = 0;
 696                                 mmu_context_bmap[i + 2] = 0;
 697                                 mmu_context_bmap[i + 3] = 0;
 698                         }
 699                         new_version = 1;
 700                         goto out;
 701                 }
 702         }
 703         mmu_context_bmap[new_ctx>>6] |= (1UL << (new_ctx & 63));
 704         new_ctx |= (tlb_context_cache & CTX_VERSION_MASK);
 705 out:
 706         tlb_context_cache = new_ctx;
 707         mm->context.sparc64_ctx_val = new_ctx | orig_pgsz_bits;
 708         spin_unlock_irqrestore(&ctx_alloc_lock, flags);
 709
 710         if (unlikely(new_version))
 711                 smp_new_mmu_context_version();
 712 }
 713
 714 static int numa_enabled = 1;
 715 static int numa_debug;
 716
 717 static int __init early_numa(char *p)
 718 {
 719         if (!p)
 720                 return 0;
 721
 722         if (strstr(p, "off"))
 723                 numa_enabled = 0;
 724
 725         if (strstr(p, "debug"))
 726                 numa_debug = 1;
 727
 728         return 0;
 729 }
 730 early_param("numa", early_numa);
 731
 732 #define numadbg(f, a...) \
 733 do {    if (numa_debug) \
 734                 printk(KERN_INFO f, ## a); \
 735 } while (0)
 736
 737 static void __init find_ramdisk(unsigned long phys_base)
 738 {
 739 #ifdef CONFIG_BLK_DEV_INITRD
 740         if (sparc_ramdisk_image || sparc_ramdisk_image64) {
 741                 unsigned long ramdisk_image;
 742
 743                 /* Older versions of the bootloader only supported a
 744                  * 32-bit physical address for the ramdisk image
 745                  * location, stored at sparc_ramdisk_image.  Newer
 746                  * SILO versions set sparc_ramdisk_image to zero and
 747                  * provide a full 64-bit physical address at
 748                  * sparc_ramdisk_image64.
 749                  */
 750                 ramdisk_image = sparc_ramdisk_image;
 751                 if (!ramdisk_image)
 752                         ramdisk_image = sparc_ramdisk_image64;
 753
 754                 /* Another bootloader quirk.  The bootloader normalizes
 755                  * the physical address to KERNBASE, so we have to
 756                  * factor that back out and add in the lowest valid
 757                  * physical page address to get the true physical address.
 758                  */
 759                 ramdisk_image -= KERNBASE;
 760                 ramdisk_image += phys_base;
 761
 762                 numadbg("Found ramdisk at physical address 0x%lx, size %u\n",
 763                         ramdisk_image, sparc_ramdisk_size);
 764
 765                 initrd_start = ramdisk_image;
 766                 initrd_end = ramdisk_image + sparc_ramdisk_size;
 767
 768                 memblock_reserve(initrd_start, sparc_ramdisk_size);
 769
 770                 initrd_start += PAGE_OFFSET;
 771                 initrd_end += PAGE_OFFSET;
 772         }
 773 #endif
 774 }
 775
 776 struct node_mem_mask {
 777         unsigned long mask;
 778         unsigned long val;
 779 };
 780 static struct node_mem_mask node_masks[MAX_NUMNODES];
 781 static int num_node_masks;
 782
 783 int numa_cpu_lookup_table[NR_CPUS];
 784 cpumask_t numa_cpumask_lookup_table[MAX_NUMNODES];
 785
 786 #ifdef CONFIG_NEED_MULTIPLE_NODES
 787
 788 struct mdesc_mblock {
 789         u64     base;
 790         u64     size;
 791         u64     offset; /* RA-to-PA */
 792 };
 793 static struct mdesc_mblock *mblocks;
 794 static int num_mblocks;
 795
 796 static unsigned long ra_to_pa(unsigned long addr)
 797 {
 798         int i;
 799
 800         for (i = 0; i < num_mblocks; i++) {
 801                 struct mdesc_mblock *m = &mblocks[i];
 802
 803                 if (addr >= m->base &&
 804                     addr < (m->base + m->size)) {
 805                         addr += m->offset;
 806                         break;
 807                 }
 808         }
 809         return addr;
 810 }
 811
 812 static int find_node(unsigned long addr)
 813 {
 814         int i;
 815
 816         addr = ra_to_pa(addr);
 817         for (i = 0; i < num_node_masks; i++) {
 818                 struct node_mem_mask *p = &node_masks[i];
 819
 820                 if ((addr & p->mask) == p->val)
 821                         return i;
 822         }
 823         return -1;
 824 }
 825
 826 static u64 memblock_nid_range(u64 start, u64 end, int *nid)
 827 {
 828         *nid = find_node(start);
 829         start += PAGE_SIZE;
 830         while (start < end) {
 831                 int n = find_node(start);
 832
 833                 if (n != *nid)
 834                         break;
 835                 start += PAGE_SIZE;
 836         }
 837
 838         if (start > end)
 839                 start = end;
 840
 841         return start;
 842 }
 843 #endif
 844
 845 /* This must be invoked after performing all of the necessary
 846  * memblock_set_node() calls for 'nid'.  We need to be able to get
 847  * correct data from get_pfn_range_for_nid().
 848  */
 849 static void __init allocate_node_data(int nid)
 850 {
 851         struct pglist_data *p;
 852         unsigned long start_pfn, end_pfn;
 853 #ifdef CONFIG_NEED_MULTIPLE_NODES
 854         unsigned long paddr;
 855
 856         paddr = memblock_alloc_try_nid(sizeof(struct pglist_data), SMP_CACHE_BYTES, nid);
 857         if (!paddr) {
 858                 prom_printf("Cannot allocate pglist_data for nid[%d]\n", nid);
 859                 prom_halt();
 860         }
 861         NODE_DATA(nid) = __va(paddr);
 862         memset(NODE_DATA(nid), 0, sizeof(struct pglist_data));
 863
 864         NODE_DATA(nid)->node_id = nid;
 865 #endif
 866
 867         p = NODE_DATA(nid);
 868
 869         get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
 870         p->node_start_pfn = start_pfn;
 871         p->node_spanned_pages = end_pfn - start_pfn;
 872 }
 873
 874 static void init_node_masks_nonnuma(void)
 875 {
 876         int i;
 877
 878         numadbg("Initializing tables for non-numa.\n");
 879
 880         node_masks[0].mask = node_masks[0].val = 0;
 881         num_node_masks = 1;
 882
 883         for (i = 0; i < NR_CPUS; i++)
 884                 numa_cpu_lookup_table[i] = 0;
 885
 886         cpumask_setall(&numa_cpumask_lookup_table[0]);
 887 }
 888
 889 #ifdef CONFIG_NEED_MULTIPLE_NODES
 890 struct pglist_data *node_data[MAX_NUMNODES];
 891
 892 EXPORT_SYMBOL(numa_cpu_lookup_table);
 893 EXPORT_SYMBOL(numa_cpumask_lookup_table);
 894 EXPORT_SYMBOL(node_data);
 895
 896 struct mdesc_mlgroup {
 897         u64     node;
 898         u64     latency;
 899         u64     match;
 900         u64     mask;
 901 };
 902 static struct mdesc_mlgroup *mlgroups;
 903 static int num_mlgroups;
 904
 905 static int scan_pio_for_cfg_handle(struct mdesc_handle *md, u64 pio,
 906                                    u32 cfg_handle)
 907 {
 908         u64 arc;
 909
 910         mdesc_for_each_arc(arc, md, pio, MDESC_ARC_TYPE_FWD) {
 911                 u64 target = mdesc_arc_target(md, arc);
 912                 const u64 *val;
 913
 914                 val = mdesc_get_property(md, target,
 915                                          "cfg-handle", NULL);
 916                 if (val && *val == cfg_handle)
 917                         return 0;
 918         }
 919         return -ENODEV;
 920 }
 921
 922 static int scan_arcs_for_cfg_handle(struct mdesc_handle *md, u64 grp,
 923                                     u32 cfg_handle)
 924 {
 925         u64 arc, candidate, best_latency = ~(u64)0;
 926
 927         candidate = MDESC_NODE_NULL;
 928         mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_FWD) {
 929                 u64 target = mdesc_arc_target(md, arc);
 930                 const char *name = mdesc_node_name(md, target);
 931                 const u64 *val;
 932
 933                 if (strcmp(name, "pio-latency-group"))
 934                         continue;
 935
 936                 val = mdesc_get_property(md, target, "latency", NULL);
 937                 if (!val)
 938                         continue;
 939
 940                 if (*val < best_latency) {
 941                         candidate = target;
 942                         best_latency = *val;
 943                 }
 944         }
 945
 946         if (candidate == MDESC_NODE_NULL)
 947                 return -ENODEV;
 948
 949         return scan_pio_for_cfg_handle(md, candidate, cfg_handle);
 950 }
 951
 952 int of_node_to_nid(struct device_node *dp)
 953 {
 954         const struct linux_prom64_registers *regs;
 955         struct mdesc_handle *md;
 956         u32 cfg_handle;
 957         int count, nid;
 958         u64 grp;
 959
 960         /* This is the right thing to do on currently supported
 961          * SUN4U NUMA platforms as well, as the PCI controller does
 962          * not sit behind any particular memory controller.
 963          */
 964         if (!mlgroups)
 965                 return -1;
 966
 967         regs = of_get_property(dp, "reg", NULL);
 968         if (!regs)
 969                 return -1;
 970
 971         cfg_handle = (regs->phys_addr >> 32UL) & 0x0fffffff;
 972
 973         md = mdesc_grab();
 974
 975         count = 0;
 976         nid = -1;
 977         mdesc_for_each_node_by_name(md, grp, "group") {
 978                 if (!scan_arcs_for_cfg_handle(md, grp, cfg_handle)) {
 979                         nid = count;
 980                         break;
 981                 }
 982                 count++;
 983         }
 984
 985         mdesc_release(md);
 986
 987         return nid;
 988 }
 989
 990 static void __init add_node_ranges(void)
 991 {
 992         struct memblock_region *reg;
 993
 994         for_each_memblock(memory, reg) {
 995                 unsigned long size = reg->size;
 996                 unsigned long start, end;
 997
 998                 start = reg->base;
 999                 end = start + size;
1000                 while (start < end) {
1001                         unsigned long this_end;
1002                         int nid;
1003
1004                         this_end = memblock_nid_range(start, end, &nid);
1005
1006                         numadbg("Setting memblock NUMA node nid[%d] "
1007                                 "start[%lx] end[%lx]\n",
1008                                 nid, start, this_end);
1009
1010                         memblock_set_node(start, this_end - start, nid);
1011                         start = this_end;
1012                 }
1013         }
1014 }
1015
1016 static int __init grab_mlgroups(struct mdesc_handle *md)
1017 {
1018         unsigned long paddr;
1019         int count = 0;
1020         u64 node;
1021
1022         mdesc_for_each_node_by_name(md, node, "memory-latency-group")
1023                 count++;
1024         if (!count)
1025                 return -ENOENT;
1026
1027         paddr = memblock_alloc(count * sizeof(struct mdesc_mlgroup),
1028                           SMP_CACHE_BYTES);
1029         if (!paddr)
1030                 return -ENOMEM;
1031
1032         mlgroups = __va(paddr);
1033         num_mlgroups = count;
1034
1035         count = 0;
1036         mdesc_for_each_node_by_name(md, node, "memory-latency-group") {
1037                 struct mdesc_mlgroup *m = &mlgroups[count++];
1038                 const u64 *val;
1039
1040                 m->node = node;
1041
1042                 val = mdesc_get_property(md, node, "latency", NULL);
1043                 m->latency = *val;
1044                 val = mdesc_get_property(md, node, "address-match", NULL);
1045                 m->match = *val;
1046                 val = mdesc_get_property(md, node, "address-mask", NULL);
1047                 m->mask = *val;
1048
1049                 numadbg("MLGROUP[%d]: node[%llx] latency[%llx] "
1050                         "match[%llx] mask[%llx]\n",
1051                         count - 1, m->node, m->latency, m->match, m->mask);
1052         }
1053
1054         return 0;
1055 }
1056
1057 static int __init grab_mblocks(struct mdesc_handle *md)
1058 {
1059         unsigned long paddr;
1060         int count = 0;
1061         u64 node;
1062
1063         mdesc_for_each_node_by_name(md, node, "mblock")
1064                 count++;
1065         if (!count)
1066                 return -ENOENT;
1067
1068         paddr = memblock_alloc(count * sizeof(struct mdesc_mblock),
1069                           SMP_CACHE_BYTES);
1070         if (!paddr)
1071                 return -ENOMEM;
1072
1073         mblocks = __va(paddr);
1074         num_mblocks = count;
1075
1076         count = 0;
1077         mdesc_for_each_node_by_name(md, node, "mblock") {
1078                 struct mdesc_mblock *m = &mblocks[count++];
1079                 const u64 *val;
1080
1081                 val = mdesc_get_property(md, node, "base", NULL);
1082                 m->base = *val;
1083                 val = mdesc_get_property(md, node, "size", NULL);
1084                 m->size = *val;
1085                 val = mdesc_get_property(md, node,
1086                                          "address-congruence-offset", NULL);
1087                 m->offset = *val;
1088
1089                 numadbg("MBLOCK[%d]: base[%llx] size[%llx] offset[%llx]\n",
1090                         count - 1, m->base, m->size, m->offset);
1091         }
1092
1093         return 0;
1094 }
1095
1096 static void __init numa_parse_mdesc_group_cpus(struct mdesc_handle *md,
1097                                                u64 grp, cpumask_t *mask)
1098 {
1099         u64 arc;
1100
1101         cpumask_clear(mask);
1102
1103         mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_BACK) {
1104                 u64 target = mdesc_arc_target(md, arc);
1105                 const char *name = mdesc_node_name(md, target);
1106                 const u64 *id;
1107
1108                 if (strcmp(name, "cpu"))
1109                         continue;
1110                 id = mdesc_get_property(md, target, "id", NULL);
1111                 if (*id < nr_cpu_ids)
1112                         cpumask_set_cpu(*id, mask);
1113         }
1114 }
1115
1116 static struct mdesc_mlgroup * __init find_mlgroup(u64 node)
1117 {
1118         int i;
1119
1120         for (i = 0; i < num_mlgroups; i++) {
1121                 struct mdesc_mlgroup *m = &mlgroups[i];
1122                 if (m->node == node)
1123                         return m;
1124         }
1125         return NULL;
1126 }
1127
1128 static int __init numa_attach_mlgroup(struct mdesc_handle *md, u64 grp,
1129                                       int index)
1130 {
1131         struct mdesc_mlgroup *candidate = NULL;
1132         u64 arc, best_latency = ~(u64)0;
1133         struct node_mem_mask *n;
1134
1135         mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_FWD) {
1136                 u64 target = mdesc_arc_target(md, arc);
1137                 struct mdesc_mlgroup *m = find_mlgroup(target);
1138                 if (!m)
1139                         continue;
1140                 if (m->latency < best_latency) {
1141                         candidate = m;
1142                         best_latency = m->latency;
1143                 }
1144         }
1145         if (!candidate)
1146                 return -ENOENT;
1147
1148         if (num_node_masks != index) {
1149                 printk(KERN_ERR "Inconsistent NUMA state, "
1150                        "index[%d] != num_node_masks[%d]\n",
1151                        index, num_node_masks);
1152                 return -EINVAL;
1153         }
1154
1155         n = &node_masks[num_node_masks++];
1156
1157         n->mask = candidate->mask;
1158         n->val = candidate->match;
1159
1160         numadbg("NUMA NODE[%d]: mask[%lx] val[%lx] (latency[%llx])\n",
1161                 index, n->mask, n->val, candidate->latency);
1162
1163         return 0;
1164 }
1165
1166 static int __init numa_parse_mdesc_group(struct mdesc_handle *md, u64 grp,
1167                                          int index)
1168 {
1169         cpumask_t mask;
1170         int cpu;
1171
1172         numa_parse_mdesc_group_cpus(md, grp, &mask);
1173
1174         for_each_cpu(cpu, &mask)
1175                 numa_cpu_lookup_table[cpu] = index;
1176         cpumask_copy(&numa_cpumask_lookup_table[index], &mask);
1177
1178         if (numa_debug) {
1179                 printk(KERN_INFO "NUMA GROUP[%d]: cpus [ ", index);
1180                 for_each_cpu(cpu, &mask)
1181                         printk("%d ", cpu);
1182                 printk("]\n");
1183         }
1184
1185         return numa_attach_mlgroup(md, grp, index);
1186 }
1187
1188 static int __init numa_parse_mdesc(void)
1189 {
1190         struct mdesc_handle *md = mdesc_grab();
1191         int i, err, count;
1192         u64 node;
1193
1194         node = mdesc_node_by_name(md, MDESC_NODE_NULL, "latency-groups");
1195         if (node == MDESC_NODE_NULL) {
1196                 mdesc_release(md);
1197                 return -ENOENT;
1198         }
1199
1200         err = grab_mblocks(md);
1201         if (err < 0)
1202                 goto out;
1203
1204         err = grab_mlgroups(md);
1205         if (err < 0)
1206                 goto out;
1207
1208         count = 0;
1209         mdesc_for_each_node_by_name(md, node, "group") {
1210                 err = numa_parse_mdesc_group(md, node, count);
1211                 if (err < 0)
1212                         break;
1213                 count++;
1214         }
1215
1216         add_node_ranges();
1217
1218         for (i = 0; i < num_node_masks; i++) {
1219                 allocate_node_data(i);
1220                 node_set_online(i);
1221         }
1222
1223         err = 0;
1224 out:
1225         mdesc_release(md);
1226         return err;
1227 }
1228
1229 static int __init numa_parse_jbus(void)
1230 {
1231         unsigned long cpu, index;
1232
1233         /* NUMA node id is encoded in bits 36 and higher, and there is
1234          * a 1-to-1 mapping from CPU ID to NUMA node ID.
1235          */
1236         index = 0;
1237         for_each_present_cpu(cpu) {
1238                 numa_cpu_lookup_table[cpu] = index;
1239                 cpumask_copy(&numa_cpumask_lookup_table[index], cpumask_of(cpu));
1240                 node_masks[index].mask = ~((1UL << 36UL) - 1UL);
1241                 node_masks[index].val = cpu << 36UL;
1242
1243                 index++;
1244         }
1245         num_node_masks = index;
1246
1247         add_node_ranges();
1248
1249         for (index = 0; index < num_node_masks; index++) {
1250                 allocate_node_data(index);
1251                 node_set_online(index);
1252         }
1253
1254         return 0;
1255 }
1256
1257 static int __init numa_parse_sun4u(void)
1258 {
1259         if (tlb_type == cheetah || tlb_type == cheetah_plus) {
1260                 unsigned long ver;
1261
1262                 __asm__ ("rdpr %%ver, %0" : "=r" (ver));
1263                 if ((ver >> 32UL) == __JALAPENO_ID ||
1264                     (ver >> 32UL) == __SERRANO_ID)
1265                         return numa_parse_jbus();
1266         }
1267         return -1;
1268 }
1269
1270 static int __init bootmem_init_numa(void)
1271 {
1272         int err = -1;
1273
1274         numadbg("bootmem_init_numa()\n");
1275
1276         if (numa_enabled) {
1277                 if (tlb_type == hypervisor)
1278                         err = numa_parse_mdesc();
1279                 else
1280                         err = numa_parse_sun4u();
1281         }
1282         return err;
1283 }
1284
1285 #else
1286
1287 static int bootmem_init_numa(void)
1288 {
1289         return -1;
1290 }
1291
1292 #endif
1293
1294 static void __init bootmem_init_nonnuma(void)
1295 {
1296         unsigned long top_of_ram = memblock_end_of_DRAM();
1297         unsigned long total_ram = memblock_phys_mem_size();
1298
1299         numadbg("bootmem_init_nonnuma()\n");
1300
1301         printk(KERN_INFO "Top of RAM: 0x%lx, Total RAM: 0x%lx\n",
1302                top_of_ram, total_ram);
1303         printk(KERN_INFO "Memory hole size: %ldMB\n",
1304                (top_of_ram - total_ram) >> 20);
1305
1306         init_node_masks_nonnuma();
1307         memblock_set_node(0, (phys_addr_t)ULLONG_MAX, 0);
1308         allocate_node_data(0);
1309         node_set_online(0);
1310 }
1311
1312 static unsigned long __init bootmem_init(unsigned long phys_base)
1313 {
1314         unsigned long end_pfn;
1315
1316         end_pfn = memblock_end_of_DRAM() >> PAGE_SHIFT;
1317         max_pfn = max_low_pfn = end_pfn;
1318         min_low_pfn = (phys_base >> PAGE_SHIFT);
1319
1320         if (bootmem_init_numa() < 0)
1321                 bootmem_init_nonnuma();
1322
1323         /* Dump memblock with node info. */
1324         memblock_dump_all();
1325
1326         /* XXX cpu notifier XXX */
1327
1328         sparse_memory_present_with_active_regions(MAX_NUMNODES);
1329         sparse_init();
1330
1331         return end_pfn;
1332 }
1333
1334 static struct linux_prom64_registers pall[MAX_BANKS] __initdata;
1335 static int pall_ents __initdata;
1336
1337 #ifdef CONFIG_DEBUG_PAGEALLOC
1338 static unsigned long __ref kernel_map_range(unsigned long pstart,
1339                                             unsigned long pend, pgprot_t prot)
1340 {
1341         unsigned long vstart = PAGE_OFFSET + pstart;
1342         unsigned long vend = PAGE_OFFSET + pend;
1343         unsigned long alloc_bytes = 0UL;
1344
1345         if ((vstart & ~PAGE_MASK) || (vend & ~PAGE_MASK)) {
1346                 prom_printf("kernel_map: Unaligned physmem[%lx:%lx]\n",
1347                             vstart, vend);
1348                 prom_halt();
1349         }
1350
1351         while (vstart < vend) {
1352                 unsigned long this_end, paddr = __pa(vstart);
1353                 pgd_t *pgd = pgd_offset_k(vstart);
1354                 pud_t *pud;
1355                 pmd_t *pmd;
1356                 pte_t *pte;
1357
1358                 pud = pud_offset(pgd, vstart);
1359                 if (pud_none(*pud)) {
1360                         pmd_t *new;
1361
1362                         new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
1363                         alloc_bytes += PAGE_SIZE;
1364                         pud_populate(&init_mm, pud, new);
1365                 }
1366
1367                 pmd = pmd_offset(pud, vstart);
1368                 if (!pmd_present(*pmd)) {
1369                         pte_t *new;
1370
1371                         new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
1372                         alloc_bytes += PAGE_SIZE;
1373                         pmd_populate_kernel(&init_mm, pmd, new);
1374                 }
1375
1376                 pte = pte_offset_kernel(pmd, vstart);
1377                 this_end = (vstart + PMD_SIZE) & PMD_MASK;
1378                 if (this_end > vend)
1379                         this_end = vend;
1380
1381                 while (vstart < this_end) {
1382                         pte_val(*pte) = (paddr | pgprot_val(prot));
1383
1384                         vstart += PAGE_SIZE;
1385                         paddr += PAGE_SIZE;
1386                         pte++;
1387                 }
1388         }
1389
1390         return alloc_bytes;
1391 }
1392
1393 extern unsigned int kvmap_linear_patch[1];
1394 #endif /* CONFIG_DEBUG_PAGEALLOC */
1395
1396 static void __init kpte_set_val(unsigned long index, unsigned long val)
1397 {
1398         unsigned long *ptr = kpte_linear_bitmap;
1399
1400         val <<= ((index % (BITS_PER_LONG / 2)) * 2);
1401         ptr += (index / (BITS_PER_LONG / 2));
1402
1403         *ptr |= val;
1404 }
1405
1406 static const unsigned long kpte_shift_min = 28; /* 256MB */
1407 static const unsigned long kpte_shift_max = 34; /* 16GB */
1408 static const unsigned long kpte_shift_incr = 3;
1409
1410 static unsigned long kpte_mark_using_shift(unsigned long start, unsigned long end,
1411                                            unsigned long shift)
1412 {
1413         unsigned long size = (1UL << shift);
1414         unsigned long mask = (size - 1UL);
1415         unsigned long remains = end - start;
1416         unsigned long val;
1417
1418         if (remains < size || (start & mask))
1419                 return start;
1420
1421         /* VAL maps:
1422          *
1423          *      shift 28 --> kern_linear_pte_xor index 1
1424          *      shift 31 --> kern_linear_pte_xor index 2
1425          *      shift 34 --> kern_linear_pte_xor index 3
1426          */
1427         val = ((shift - kpte_shift_min) / kpte_shift_incr) + 1;
1428
1429         remains &= ~mask;
1430         if (shift != kpte_shift_max)
1431                 remains = size;
1432
1433         while (remains) {
1434                 unsigned long index = start >> kpte_shift_min;
1435
1436                 kpte_set_val(index, val);
1437
1438                 start += 1UL << kpte_shift_min;
1439                 remains -= 1UL << kpte_shift_min;
1440         }
1441
1442         return start;
1443 }
1444
1445 static void __init mark_kpte_bitmap(unsigned long start, unsigned long end)
1446 {
1447         unsigned long smallest_size, smallest_mask;
1448         unsigned long s;
1449
1450         smallest_size = (1UL << kpte_shift_min);
1451         smallest_mask = (smallest_size - 1UL);
1452
1453         while (start < end) {
1454                 unsigned long orig_start = start;
1455
1456                 for (s = kpte_shift_max; s >= kpte_shift_min; s -= kpte_shift_incr) {
1457                         start = kpte_mark_using_shift(start, end, s);
1458
1459                         if (start != orig_start)
1460                                 break;
1461                 }
1462
1463                 if (start == orig_start)
1464                         start = (start + smallest_size) & ~smallest_mask;
1465         }
1466 }
1467
1468 static void __init init_kpte_bitmap(void)
1469 {
1470         unsigned long i;
1471
1472         for (i = 0; i < pall_ents; i++) {
1473                 unsigned long phys_start, phys_end;
1474
1475                 phys_start = pall[i].phys_addr;
1476                 phys_end = phys_start + pall[i].reg_size;
1477
1478                 mark_kpte_bitmap(phys_start, phys_end);
1479         }
1480 }
1481
1482 static void __init kernel_physical_mapping_init(void)
1483 {
1484 #ifdef CONFIG_DEBUG_PAGEALLOC
1485         unsigned long i, mem_alloced = 0UL;
1486
1487         for (i = 0; i < pall_ents; i++) {
1488                 unsigned long phys_start, phys_end;
1489
1490                 phys_start = pall[i].phys_addr;
1491                 phys_end = phys_start + pall[i].reg_size;
1492
1493                 mem_alloced += kernel_map_range(phys_start, phys_end,
1494                                                 PAGE_KERNEL);
1495         }
1496
1497         printk("Allocated %ld bytes for kernel page tables.\n",
1498                mem_alloced);
1499
1500         kvmap_linear_patch[0] = 0x01000000; /* nop */
1501         flushi(&kvmap_linear_patch[0]);
1502
1503         __flush_tlb_all();
1504 #endif
1505 }
1506
1507 #ifdef CONFIG_DEBUG_PAGEALLOC
1508 void kernel_map_pages(struct page *page, int numpages, int enable)
1509 {
1510         unsigned long phys_start = page_to_pfn(page) << PAGE_SHIFT;
1511         unsigned long phys_end = phys_start + (numpages * PAGE_SIZE);
1512
1513         kernel_map_range(phys_start, phys_end,
1514                          (enable ? PAGE_KERNEL : __pgprot(0)));
1515
1516         flush_tsb_kernel_range(PAGE_OFFSET + phys_start,
1517                                PAGE_OFFSET + phys_end);
1518
1519         /* we should perform an IPI and flush all tlbs,
1520          * but that can deadlock->flush only current cpu.
1521          */
1522         __flush_tlb_kernel_range(PAGE_OFFSET + phys_start,
1523                                  PAGE_OFFSET + phys_end);
1524 }
1525 #endif
1526
1527 unsigned long __init find_ecache_flush_span(unsigned long size)
1528 {
1529         int i;
1530
1531         for (i = 0; i < pavail_ents; i++) {
1532                 if (pavail[i].reg_size >= size)
1533                         return pavail[i].phys_addr;
1534         }
1535
1536         return ~0UL;
1537 }
1538
1539 static void __init tsb_phys_patch(void)
1540 {
1541         struct tsb_ldquad_phys_patch_entry *pquad;
1542         struct tsb_phys_patch_entry *p;
1543
1544         pquad = &__tsb_ldquad_phys_patch;
1545         while (pquad < &__tsb_ldquad_phys_patch_end) {
1546                 unsigned long addr = pquad->addr;
1547
1548                 if (tlb_type == hypervisor)
1549                         *(unsigned int *) addr = pquad->sun4v_insn;
1550                 else
1551                         *(unsigned int *) addr = pquad->sun4u_insn;
1552                 wmb();
1553                 __asm__ __volatile__("flush     %0"
1554                                      : /* no outputs */
1555                                      : "r" (addr));
1556
1557                 pquad++;
1558         }
1559
1560         p = &__tsb_phys_patch;
1561         while (p < &__tsb_phys_patch_end) {
1562                 unsigned long addr = p->addr;
1563
1564                 *(unsigned int *) addr = p->insn;
1565                 wmb();
1566                 __asm__ __volatile__("flush     %0"
1567                                      : /* no outputs */
1568                                      : "r" (addr));
1569
1570                 p++;
1571         }
1572 }
1573
1574 /* Don't mark as init, we give this to the Hypervisor.  */
1575 #ifndef CONFIG_DEBUG_PAGEALLOC
1576 #define NUM_KTSB_DESCR  2
1577 #else
1578 #define NUM_KTSB_DESCR  1
1579 #endif
1580 static struct hv_tsb_descr ktsb_descr[NUM_KTSB_DESCR];
1581 extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES];
1582
1583 static void patch_one_ktsb_phys(unsigned int *start, unsigned int *end, unsigned long pa)
1584 {
1585         pa >>= KTSB_PHYS_SHIFT;
1586
1587         while (start < end) {
1588                 unsigned int *ia = (unsigned int *)(unsigned long)*start;
1589
1590                 ia[0] = (ia[0] & ~0x3fffff) | (pa >> 10);
1591                 __asm__ __volatile__("flush     %0" : : "r" (ia));
1592
1593                 ia[1] = (ia[1] & ~0x3ff) | (pa & 0x3ff);
1594                 __asm__ __volatile__("flush     %0" : : "r" (ia + 1));
1595
1596                 start++;
1597         }
1598 }
1599
1600 static void ktsb_phys_patch(void)
1601 {
1602         extern unsigned int __swapper_tsb_phys_patch;
1603         extern unsigned int __swapper_tsb_phys_patch_end;
1604         unsigned long ktsb_pa;
1605
1606         ktsb_pa = kern_base + ((unsigned long)&swapper_tsb[0] - KERNBASE);
1607         patch_one_ktsb_phys(&__swapper_tsb_phys_patch,
1608                             &__swapper_tsb_phys_patch_end, ktsb_pa);
1609 #ifndef CONFIG_DEBUG_PAGEALLOC
1610         {
1611         extern unsigned int __swapper_4m_tsb_phys_patch;
1612         extern unsigned int __swapper_4m_tsb_phys_patch_end;
1613         ktsb_pa = (kern_base +
1614                    ((unsigned long)&swapper_4m_tsb[0] - KERNBASE));
1615         patch_one_ktsb_phys(&__swapper_4m_tsb_phys_patch,
1616                             &__swapper_4m_tsb_phys_patch_end, ktsb_pa);
1617         }
1618 #endif
1619 }
1620
1621 static void __init sun4v_ktsb_init(void)
1622 {
1623         unsigned long ktsb_pa;
1624
1625         /* First KTSB for PAGE_SIZE mappings.  */
1626         ktsb_pa = kern_base + ((unsigned long)&swapper_tsb[0] - KERNBASE);
1627
1628         switch (PAGE_SIZE) {
1629         case 8 * 1024:
1630         default:
1631                 ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_8K;
1632                 ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_8K;
1633                 break;
1634
1635         case 64 * 1024:
1636                 ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_64K;
1637                 ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_64K;
1638                 break;
1639
1640         case 512 * 1024:
1641                 ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_512K;
1642                 ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_512K;
1643                 break;
1644
1645         case 4 * 1024 * 1024:
1646                 ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_4MB;
1647                 ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_4MB;
1648                 break;
1649         }
1650
1651         ktsb_descr[0].assoc = 1;
1652         ktsb_descr[0].num_ttes = KERNEL_TSB_NENTRIES;
1653         ktsb_descr[0].ctx_idx = 0;
1654         ktsb_descr[0].tsb_base = ktsb_pa;
1655         ktsb_descr[0].resv = 0;
1656
1657 #ifndef CONFIG_DEBUG_PAGEALLOC
1658         /* Second KTSB for 4MB/256MB/2GB/16GB mappings.  */
1659         ktsb_pa = (kern_base +
1660                    ((unsigned long)&swapper_4m_tsb[0] - KERNBASE));
1661
1662         ktsb_descr[1].pgsz_idx = HV_PGSZ_IDX_4MB;
1663         ktsb_descr[1].pgsz_mask = (HV_PGSZ_MASK_4MB |
1664                                    HV_PGSZ_MASK_256MB);
1665         if (sun4v_chip_type == SUN4V_CHIP_NIAGARA4)
1666                 ktsb_descr[1].pgsz_mask |= HV_PGSZ_MASK_2GB;
1667         ktsb_descr[1].assoc = 1;
1668         ktsb_descr[1].num_ttes = KERNEL_TSB4M_NENTRIES;
1669         ktsb_descr[1].ctx_idx = 0;
1670         ktsb_descr[1].tsb_base = ktsb_pa;
1671         ktsb_descr[1].resv = 0;
1672 #endif
1673 }
1674
1675 void __cpuinit sun4v_ktsb_register(void)
1676 {
1677         unsigned long pa, ret;
1678
1679         pa = kern_base + ((unsigned long)&ktsb_descr[0] - KERNBASE);
1680
1681         ret = sun4v_mmu_tsb_ctx0(NUM_KTSB_DESCR, pa);
1682         if (ret != 0) {
1683                 prom_printf("hypervisor_mmu_tsb_ctx0[%lx]: "
1684                             "errors with %lx\n", pa, ret);
1685                 prom_halt();
1686         }
1687 }
1688
1689 /* paging_init() sets up the page tables */
1690
1691 static unsigned long last_valid_pfn;
1692 pgd_t swapper_pg_dir[2048];
1693
1694 static void sun4u_pgprot_init(void);
1695 static void sun4v_pgprot_init(void);
1696
1697 void __init paging_init(void)
1698 {
1699         unsigned long end_pfn, shift, phys_base;
1700         unsigned long real_end, i;
1701         int node;
1702
1703         /* These build time checkes make sure that the dcache_dirty_cpu()
1704          * page->flags usage will work.
1705          *
1706          * When a page gets marked as dcache-dirty, we store the
1707          * cpu number starting at bit 32 in the page->flags.  Also,
1708          * functions like clear_dcache_dirty_cpu use the cpu mask
1709          * in 13-bit signed-immediate instruction fields.
1710          */
1711
1712         /*
1713          * Page flags must not reach into upper 32 bits that are used
1714          * for the cpu number
1715          */
1716         BUILD_BUG_ON(NR_PAGEFLAGS > 32);
1717
1718         /*
1719          * The bit fields placed in the high range must not reach below
1720          * the 32 bit boundary. Otherwise we cannot place the cpu field
1721          * at the 32 bit boundary.
1722          */
1723         BUILD_BUG_ON(SECTIONS_WIDTH + NODES_WIDTH + ZONES_WIDTH +
1724                 ilog2(roundup_pow_of_two(NR_CPUS)) > 32);
1725
1726         BUILD_BUG_ON(NR_CPUS > 4096);
1727
1728         kern_base = (prom_boot_mapping_phys_low >> 22UL) << 22UL;
1729         kern_size = (unsigned long)&_end - (unsigned long)KERNBASE;
1730
1731         /* Invalidate both kernel TSBs.  */
1732         memset(swapper_tsb, 0x40, sizeof(swapper_tsb));
1733 #ifndef CONFIG_DEBUG_PAGEALLOC
1734         memset(swapper_4m_tsb, 0x40, sizeof(swapper_4m_tsb));
1735 #endif
1736
1737         if (tlb_type == hypervisor)
1738                 sun4v_pgprot_init();
1739         else
1740                 sun4u_pgprot_init();
1741
1742         if (tlb_type == cheetah_plus ||
1743             tlb_type == hypervisor) {
1744                 tsb_phys_patch();
1745                 ktsb_phys_patch();
1746         }
1747
1748         if (tlb_type == hypervisor) {
1749                 sun4v_patch_tlb_handlers();
1750                 sun4v_ktsb_init();
1751         }
1752
1753         /* Find available physical memory...
1754          *
1755          * Read it twice in order to work around a bug in openfirmware.
1756          * The call to grab this table itself can cause openfirmware to
1757          * allocate memory, which in turn can take away some space from
1758          * the list of available memory.  Reading it twice makes sure
1759          * we really do get the final value.
1760          */
1761         read_obp_translations();
1762         read_obp_memory("reg", &pall[0], &pall_ents);
1763         read_obp_memory("available", &pavail[0], &pavail_ents);
1764         read_obp_memory("available", &pavail[0], &pavail_ents);
1765
1766         phys_base = 0xffffffffffffffffUL;
1767         for (i = 0; i < pavail_ents; i++) {
1768                 phys_base = min(phys_base, pavail[i].phys_addr);
1769                 memblock_add(pavail[i].phys_addr, pavail[i].reg_size);
1770         }
1771
1772         memblock_reserve(kern_base, kern_size);
1773
1774         find_ramdisk(phys_base);
1775
1776         memblock_enforce_memory_limit(cmdline_memory_size);
1777
1778         memblock_allow_resize();
1779         memblock_dump_all();
1780
1781         set_bit(0, mmu_context_bmap);
1782
1783         shift = kern_base + PAGE_OFFSET - ((unsigned long)KERNBASE);
1784
1785         real_end = (unsigned long)_end;
1786         num_kernel_image_mappings = DIV_ROUND_UP(real_end - KERNBASE, 1 << 22);
1787         printk("Kernel: Using %d locked TLB entries for main kernel image.\n",
1788                num_kernel_image_mappings);
1789
1790         /* Set kernel pgd to upper alias so physical page computations
1791          * work.
1792          */
1793         init_mm.pgd += ((shift) / (sizeof(pgd_t)));
1794
1795         memset(swapper_low_pmd_dir, 0, sizeof(swapper_low_pmd_dir));
1796
1797         /* Now can init the kernel/bad page tables. */
1798         pud_set(pud_offset(&swapper_pg_dir[0], 0),
1799                 swapper_low_pmd_dir + (shift / sizeof(pgd_t)));
1800
1801         inherit_prom_mappings();
1802
1803         init_kpte_bitmap();
1804
1805         /* Ok, we can use our TLB miss and window trap handlers safely.  */
1806         setup_tba();
1807
1808         __flush_tlb_all();
1809
1810         if (tlb_type == hypervisor)
1811                 sun4v_ktsb_register();
1812
1813         prom_build_devicetree();
1814         of_populate_present_mask();
1815 #ifndef CONFIG_SMP
1816         of_fill_in_cpu_data();
1817 #endif
1818
1819         if (tlb_type == hypervisor) {
1820                 sun4v_mdesc_init();
1821                 mdesc_populate_present_mask(cpu_all_mask);
1822 #ifndef CONFIG_SMP
1823                 mdesc_fill_in_cpu_data(cpu_all_mask);
1824 #endif
1825                 mdesc_get_page_sizes(cpu_all_mask, &cpu_pgsz_mask);
1826         } else {
1827                 unsigned long impl, ver;
1828
1829                 cpu_pgsz_mask = (HV_PGSZ_MASK_8K | HV_PGSZ_MASK_64K |
1830                                  HV_PGSZ_MASK_512K | HV_PGSZ_MASK_4MB);
1831
1832                 __asm__ __volatile__("rdpr %%ver, %0" : "=r" (ver));
1833                 impl = ((ver >> 32) & 0xffff);
1834                 if (impl == PANTHER_IMPL)
1835                         cpu_pgsz_mask |= (HV_PGSZ_MASK_32MB |
1836                                           HV_PGSZ_MASK_256MB);
1837         }
1838
1839         /* Setup bootmem... */
1840         last_valid_pfn = end_pfn = bootmem_init(phys_base);
1841
1842         /* Once the OF device tree and MDESC have been setup, we know
1843          * the list of possible cpus.  Therefore we can allocate the
1844          * IRQ stacks.
1845          */
1846         for_each_possible_cpu(i) {
1847                 node = cpu_to_node(i);
1848
1849                 softirq_stack[i] = __alloc_bootmem_node(NODE_DATA(node),
1850                                                         THREAD_SIZE,
1851                                                         THREAD_SIZE, 0);
1852                 hardirq_stack[i] = __alloc_bootmem_node(NODE_DATA(node),
1853                                                         THREAD_SIZE,
1854                                                         THREAD_SIZE, 0);
1855         }
1856
1857         kernel_physical_mapping_init();
1858
1859         {
1860                 unsigned long max_zone_pfns[MAX_NR_ZONES];
1861
1862                 memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
1863
1864                 max_zone_pfns[ZONE_NORMAL] = end_pfn;
1865
1866                 free_area_init_nodes(max_zone_pfns);
1867         }
1868
1869         printk("Booting Linux...\n");
1870 }
1871
1872 int __devinit page_in_phys_avail(unsigned long paddr)
1873 {
1874         int i;
1875
1876         paddr &= PAGE_MASK;
1877
1878         for (i = 0; i < pavail_ents; i++) {
1879                 unsigned long start, end;
1880
1881                 start = pavail[i].phys_addr;
1882                 end = start + pavail[i].reg_size;
1883
1884                 if (paddr >= start && paddr < end)
1885                         return 1;
1886         }
1887         if (paddr >= kern_base && paddr < (kern_base + kern_size))
1888                 return 1;
1889 #ifdef CONFIG_BLK_DEV_INITRD
1890         if (paddr >= __pa(initrd_start) &&
1891             paddr < __pa(PAGE_ALIGN(initrd_end)))
1892                 return 1;
1893 #endif
1894
1895         return 0;
1896 }
1897
1898 static struct linux_prom64_registers pavail_rescan[MAX_BANKS] __initdata;
1899 static int pavail_rescan_ents __initdata;
1900
1901 /* Certain OBP calls, such as fetching "available" properties, can
1902  * claim physical memory.  So, along with initializing the valid
1903  * address bitmap, what we do here is refetch the physical available
1904  * memory list again, and make sure it provides at least as much
1905  * memory as 'pavail' does.
1906  */
1907 static void __init setup_valid_addr_bitmap_from_pavail(unsigned long *bitmap)
1908 {
1909         int i;
1910
1911         read_obp_memory("available", &pavail_rescan[0], &pavail_rescan_ents);
1912
1913         for (i = 0; i < pavail_ents; i++) {
1914                 unsigned long old_start, old_end;
1915
1916                 old_start = pavail[i].phys_addr;
1917                 old_end = old_start + pavail[i].reg_size;
1918                 while (old_start < old_end) {
1919                         int n;
1920
1921                         for (n = 0; n < pavail_rescan_ents; n++) {
1922                                 unsigned long new_start, new_end;
1923
1924                                 new_start = pavail_rescan[n].phys_addr;
1925                                 new_end = new_start +
1926                                         pavail_rescan[n].reg_size;
1927
1928                                 if (new_start <= old_start &&
1929                                     new_end >= (old_start + PAGE_SIZE)) {
1930                                         set_bit(old_start >> 22, bitmap);
1931                                         goto do_next_page;
1932                                 }
1933                         }
1934
1935                         prom_printf("mem_init: Lost memory in pavail\n");
1936                         prom_printf("mem_init: OLD start[%lx] size[%lx]\n",
1937                                     pavail[i].phys_addr,
1938                                     pavail[i].reg_size);
1939                         prom_printf("mem_init: NEW start[%lx] size[%lx]\n",
1940                                     pavail_rescan[i].phys_addr,
1941                                     pavail_rescan[i].reg_size);
1942                         prom_printf("mem_init: Cannot continue, aborting.\n");
1943                         prom_halt();
1944
1945                 do_next_page:
1946                         old_start += PAGE_SIZE;
1947                 }
1948         }
1949 }
1950
1951 static void __init patch_tlb_miss_handler_bitmap(void)
1952 {
1953         extern unsigned int valid_addr_bitmap_insn[];
1954         extern unsigned int valid_addr_bitmap_patch[];
1955
1956         valid_addr_bitmap_insn[1] = valid_addr_bitmap_patch[1];
1957         mb();
1958         valid_addr_bitmap_insn[0] = valid_addr_bitmap_patch[0];
1959         flushi(&valid_addr_bitmap_insn[0]);
1960 }
1961
1962 void __init mem_init(void)
1963 {
1964         unsigned long codepages, datapages, initpages;
1965         unsigned long addr, last;
1966
1967         addr = PAGE_OFFSET + kern_base;
1968         last = PAGE_ALIGN(kern_size) + addr;
1969         while (addr < last) {
1970                 set_bit(__pa(addr) >> 22, sparc64_valid_addr_bitmap);
1971                 addr += PAGE_SIZE;
1972         }
1973
1974         setup_valid_addr_bitmap_from_pavail(sparc64_valid_addr_bitmap);
1975         patch_tlb_miss_handler_bitmap();
1976
1977         high_memory = __va(last_valid_pfn << PAGE_SHIFT);
1978
1979 #ifdef CONFIG_NEED_MULTIPLE_NODES
1980         {
1981                 int i;
1982                 for_each_online_node(i) {
1983                         if (NODE_DATA(i)->node_spanned_pages != 0) {
1984                                 totalram_pages +=
1985                                         free_all_bootmem_node(NODE_DATA(i));
1986                         }
1987                 }
1988                 totalram_pages += free_low_memory_core_early(MAX_NUMNODES);
1989         }
1990 #else
1991         totalram_pages = free_all_bootmem();
1992 #endif
1993
1994         /* We subtract one to account for the mem_map_zero page
1995          * allocated below.
1996          */
1997         totalram_pages -= 1;
1998         num_physpages = totalram_pages;
1999
2000         /*
2001          * Set up the zero page, mark it reserved, so that page count
2002          * is not manipulated when freeing the page from user ptes.
2003          */
2004         mem_map_zero = alloc_pages(GFP_KERNEL|__GFP_ZERO, 0);
2005         if (mem_map_zero == NULL) {
2006                 prom_printf("paging_init: Cannot alloc zero page.\n");
2007                 prom_halt();
2008         }
2009         SetPageReserved(mem_map_zero);
2010
2011         codepages = (((unsigned long) _etext) - ((unsigned long) _start));
2012         codepages = PAGE_ALIGN(codepages) >> PAGE_SHIFT;
2013         datapages = (((unsigned long) _edata) - ((unsigned long) _etext));
2014         datapages = PAGE_ALIGN(datapages) >> PAGE_SHIFT;
2015         initpages = (((unsigned long) __init_end) - ((unsigned long) __init_begin));
2016         initpages = PAGE_ALIGN(initpages) >> PAGE_SHIFT;
2017
2018         printk("Memory: %luk available (%ldk kernel code, %ldk data, %ldk init) [%016lx,%016lx]\n",
2019                nr_free_pages() << (PAGE_SHIFT-10),
2020                codepages << (PAGE_SHIFT-10),
2021                datapages << (PAGE_SHIFT-10),
2022                initpages << (PAGE_SHIFT-10),
2023                PAGE_OFFSET, (last_valid_pfn << PAGE_SHIFT));
2024
2025         if (tlb_type == cheetah || tlb_type == cheetah_plus)
2026                 cheetah_ecache_flush_init();
2027 }
2028
2029 void free_initmem(void)
2030 {
2031         unsigned long addr, initend;
2032         int do_free = 1;
2033
2034         /* If the physical memory maps were trimmed by kernel command
2035          * line options, don't even try freeing this initmem stuff up.
2036          * The kernel image could have been in the trimmed out region
2037          * and if so the freeing below will free invalid page structs.
2038          */
2039         if (cmdline_memory_size)
2040                 do_free = 0;
2041
2042         /*
2043          * The init section is aligned to 8k in vmlinux.lds. Page align for >8k pagesizes.
2044          */
2045         addr = PAGE_ALIGN((unsigned long)(__init_begin));
2046         initend = (unsigned long)(__init_end) & PAGE_MASK;
2047         for (; addr < initend; addr += PAGE_SIZE) {
2048                 unsigned long page;
2049                 struct page *p;
2050
2051                 page = (addr +
2052                         ((unsigned long) __va(kern_base)) -
2053                         ((unsigned long) KERNBASE));
2054                 memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE);
2055
2056                 if (do_free) {
2057                         p = virt_to_page(page);
2058
2059                         ClearPageReserved(p);
2060                         init_page_count(p);
2061                         __free_page(p);
2062                         num_physpages++;
2063                         totalram_pages++;
2064                 }
2065         }
2066 }
2067
2068 #ifdef CONFIG_BLK_DEV_INITRD
2069 void free_initrd_mem(unsigned long start, unsigned long end)
2070 {
2071         if (start < end)
2072                 printk ("Freeing initrd memory: %ldk freed\n", (end - start) >> 10);
2073         for (; start < end; start += PAGE_SIZE) {
2074                 struct page *p = virt_to_page(start);
2075
2076                 ClearPageReserved(p);
2077                 init_page_count(p);
2078                 __free_page(p);
2079                 num_physpages++;
2080                 totalram_pages++;
2081         }
2082 }
2083 #endif
2084
2085 #define _PAGE_CACHE_4U  (_PAGE_CP_4U | _PAGE_CV_4U)
2086 #define _PAGE_CACHE_4V  (_PAGE_CP_4V | _PAGE_CV_4V)
2087 #define __DIRTY_BITS_4U  (_PAGE_MODIFIED_4U | _PAGE_WRITE_4U | _PAGE_W_4U)
2088 #define __DIRTY_BITS_4V  (_PAGE_MODIFIED_4V | _PAGE_WRITE_4V | _PAGE_W_4V)
2089 #define __ACCESS_BITS_4U (_PAGE_ACCESSED_4U | _PAGE_READ_4U | _PAGE_R)
2090 #define __ACCESS_BITS_4V (_PAGE_ACCESSED_4V | _PAGE_READ_4V | _PAGE_R)
2091
2092 pgprot_t PAGE_KERNEL __read_mostly;
2093 EXPORT_SYMBOL(PAGE_KERNEL);
2094
2095 pgprot_t PAGE_KERNEL_LOCKED __read_mostly;
2096 pgprot_t PAGE_COPY __read_mostly;
2097
2098 pgprot_t PAGE_SHARED __read_mostly;
2099 EXPORT_SYMBOL(PAGE_SHARED);
2100
2101 unsigned long pg_iobits __read_mostly;
2102
2103 unsigned long _PAGE_IE __read_mostly;
2104 EXPORT_SYMBOL(_PAGE_IE);
2105
2106 unsigned long _PAGE_E __read_mostly;
2107 EXPORT_SYMBOL(_PAGE_E);
2108
2109 unsigned long _PAGE_CACHE __read_mostly;
2110 EXPORT_SYMBOL(_PAGE_CACHE);
2111
2112 #ifdef CONFIG_SPARSEMEM_VMEMMAP
2113 unsigned long vmemmap_table[VMEMMAP_SIZE];
2114
2115 static long __meminitdata addr_start, addr_end;
2116 static int __meminitdata node_start;
2117
2118 int __meminit vmemmap_populate(struct page *start, unsigned long nr, int node)
2119 {
2120         unsigned long vstart = (unsigned long) start;
2121         unsigned long vend = (unsigned long) (start + nr);
2122         unsigned long phys_start = (vstart - VMEMMAP_BASE);
2123         unsigned long phys_end = (vend - VMEMMAP_BASE);
2124         unsigned long addr = phys_start & VMEMMAP_CHUNK_MASK;
2125         unsigned long end = VMEMMAP_ALIGN(phys_end);
2126         unsigned long pte_base;
2127
2128         pte_base = (_PAGE_VALID | _PAGE_SZ4MB_4U |
2129                     _PAGE_CP_4U | _PAGE_CV_4U |
2130                     _PAGE_P_4U | _PAGE_W_4U);
2131         if (tlb_type == hypervisor)
2132                 pte_base = (_PAGE_VALID | _PAGE_SZ4MB_4V |
2133                             _PAGE_CP_4V | _PAGE_CV_4V |
2134                             _PAGE_P_4V | _PAGE_W_4V);
2135
2136         for (; addr < end; addr += VMEMMAP_CHUNK) {
2137                 unsigned long *vmem_pp =
2138                         vmemmap_table + (addr >> VMEMMAP_CHUNK_SHIFT);
2139                 void *block;
2140
2141                 if (!(*vmem_pp & _PAGE_VALID)) {
2142                         block = vmemmap_alloc_block(1UL << 22, node);
2143                         if (!block)
2144                                 return -ENOMEM;
2145
2146                         *vmem_pp = pte_base | __pa(block);
2147
2148                         /* check to see if we have contiguous blocks */
2149                         if (addr_end != addr || node_start != node) {
2150                                 if (addr_start)
2151                                         printk(KERN_DEBUG " [%lx-%lx] on node %d\n",
2152                                                addr_start, addr_end-1, node_start);
2153                                 addr_start = addr;
2154                                 node_start = node;
2155                         }
2156                         addr_end = addr + VMEMMAP_CHUNK;
2157                 }
2158         }
2159         return 0;
2160 }
2161
2162 void __meminit vmemmap_populate_print_last(void)
2163 {
2164         if (addr_start) {
2165                 printk(KERN_DEBUG " [%lx-%lx] on node %d\n",
2166                        addr_start, addr_end-1, node_start);
2167                 addr_start = 0;
2168                 addr_end = 0;
2169                 node_start = 0;
2170         }
2171 }
2172 #endif /* CONFIG_SPARSEMEM_VMEMMAP */
2173
2174 static void prot_init_common(unsigned long page_none,
2175                              unsigned long page_shared,
2176                              unsigned long page_copy,
2177                              unsigned long page_readonly,
2178                              unsigned long page_exec_bit)
2179 {
2180         PAGE_COPY = __pgprot(page_copy);
2181         PAGE_SHARED = __pgprot(page_shared);
2182
2183         protection_map[0x0] = __pgprot(page_none);
2184         protection_map[0x1] = __pgprot(page_readonly & ~page_exec_bit);
2185         protection_map[0x2] = __pgprot(page_copy & ~page_exec_bit);
2186         protection_map[0x3] = __pgprot(page_copy & ~page_exec_bit);
2187         protection_map[0x4] = __pgprot(page_readonly);
2188         protection_map[0x5] = __pgprot(page_readonly);
2189         protection_map[0x6] = __pgprot(page_copy);
2190         protection_map[0x7] = __pgprot(page_copy);
2191         protection_map[0x8] = __pgprot(page_none);
2192         protection_map[0x9] = __pgprot(page_readonly & ~page_exec_bit);
2193         protection_map[0xa] = __pgprot(page_shared & ~page_exec_bit);
2194         protection_map[0xb] = __pgprot(page_shared & ~page_exec_bit);
2195         protection_map[0xc] = __pgprot(page_readonly);
2196         protection_map[0xd] = __pgprot(page_readonly);
2197         protection_map[0xe] = __pgprot(page_shared);
2198         protection_map[0xf] = __pgprot(page_shared);
2199 }
2200
2201 static void __init sun4u_pgprot_init(void)
2202 {
2203         unsigned long page_none, page_shared, page_copy, page_readonly;
2204         unsigned long page_exec_bit;
2205         int i;
2206
2207         PAGE_KERNEL = __pgprot (_PAGE_PRESENT_4U | _PAGE_VALID |
2208                                 _PAGE_CACHE_4U | _PAGE_P_4U |
2209                                 __ACCESS_BITS_4U | __DIRTY_BITS_4U |
2210                                 _PAGE_EXEC_4U);
2211         PAGE_KERNEL_LOCKED = __pgprot (_PAGE_PRESENT_4U | _PAGE_VALID |
2212                                        _PAGE_CACHE_4U | _PAGE_P_4U |
2213                                        __ACCESS_BITS_4U | __DIRTY_BITS_4U |
2214                                        _PAGE_EXEC_4U | _PAGE_L_4U);
2215
2216         _PAGE_IE = _PAGE_IE_4U;
2217         _PAGE_E = _PAGE_E_4U;
2218         _PAGE_CACHE = _PAGE_CACHE_4U;
2219
2220         pg_iobits = (_PAGE_VALID | _PAGE_PRESENT_4U | __DIRTY_BITS_4U |
2221                      __ACCESS_BITS_4U | _PAGE_E_4U);
2222
2223 #ifdef CONFIG_DEBUG_PAGEALLOC
2224         kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZBITS_4U) ^
2225                 0xfffff80000000000UL;
2226 #else
2227         kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4U) ^
2228                 0xfffff80000000000UL;
2229 #endif
2230         kern_linear_pte_xor[0] |= (_PAGE_CP_4U | _PAGE_CV_4U |
2231                                    _PAGE_P_4U | _PAGE_W_4U);
2232
2233         /* XXX Should use 256MB on Panther. XXX */
2234         for (i = 1; i < 4; i++)
2235                 kern_linear_pte_xor[i] = kern_linear_pte_xor[0];
2236
2237         _PAGE_SZBITS = _PAGE_SZBITS_4U;
2238         _PAGE_ALL_SZ_BITS =  (_PAGE_SZ4MB_4U | _PAGE_SZ512K_4U |
2239                               _PAGE_SZ64K_4U | _PAGE_SZ8K_4U |
2240                               _PAGE_SZ32MB_4U | _PAGE_SZ256MB_4U);
2241
2242
2243         page_none = _PAGE_PRESENT_4U | _PAGE_ACCESSED_4U | _PAGE_CACHE_4U;
2244         page_shared = (_PAGE_VALID | _PAGE_PRESENT_4U | _PAGE_CACHE_4U |
2245                        __ACCESS_BITS_4U | _PAGE_WRITE_4U | _PAGE_EXEC_4U);
2246         page_copy   = (_PAGE_VALID | _PAGE_PRESENT_4U | _PAGE_CACHE_4U |
2247                        __ACCESS_BITS_4U | _PAGE_EXEC_4U);
2248         page_readonly   = (_PAGE_VALID | _PAGE_PRESENT_4U | _PAGE_CACHE_4U |
2249                            __ACCESS_BITS_4U | _PAGE_EXEC_4U);
2250
2251         page_exec_bit = _PAGE_EXEC_4U;
2252
2253         prot_init_common(page_none, page_shared, page_copy, page_readonly,
2254                          page_exec_bit);
2255 }
2256
2257 static void __init sun4v_pgprot_init(void)
2258 {
2259         unsigned long page_none, page_shared, page_copy, page_readonly;
2260         unsigned long page_exec_bit;
2261         int i;
2262
2263         PAGE_KERNEL = __pgprot (_PAGE_PRESENT_4V | _PAGE_VALID |
2264                                 _PAGE_CACHE_4V | _PAGE_P_4V |
2265                                 __ACCESS_BITS_4V | __DIRTY_BITS_4V |
2266                                 _PAGE_EXEC_4V);
2267         PAGE_KERNEL_LOCKED = PAGE_KERNEL;
2268
2269         _PAGE_IE = _PAGE_IE_4V;
2270         _PAGE_E = _PAGE_E_4V;
2271         _PAGE_CACHE = _PAGE_CACHE_4V;
2272
2273 #ifdef CONFIG_DEBUG_PAGEALLOC
2274         kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZBITS_4V) ^
2275                 0xfffff80000000000UL;
2276 #else
2277         kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4V) ^
2278                 0xfffff80000000000UL;
2279 #endif
2280         kern_linear_pte_xor[0] |= (_PAGE_CP_4V | _PAGE_CV_4V |
2281                                    _PAGE_P_4V | _PAGE_W_4V);
2282
2283 #ifdef CONFIG_DEBUG_PAGEALLOC
2284         kern_linear_pte_xor[1] = (_PAGE_VALID | _PAGE_SZBITS_4V) ^
2285                 0xfffff80000000000UL;
2286 #else
2287         kern_linear_pte_xor[1] = (_PAGE_VALID | _PAGE_SZ256MB_4V) ^
2288                 0xfffff80000000000UL;
2289 #endif
2290         kern_linear_pte_xor[1] |= (_PAGE_CP_4V | _PAGE_CV_4V |
2291                                    _PAGE_P_4V | _PAGE_W_4V);
2292
2293         i = 2;
2294
2295         if (sun4v_chip_type == SUN4V_CHIP_NIAGARA4) {
2296 #ifdef CONFIG_DEBUG_PAGEALLOC
2297                 kern_linear_pte_xor[2] = (_PAGE_VALID | _PAGE_SZBITS_4V) ^
2298                         0xfffff80000000000UL;
2299 #else
2300                 kern_linear_pte_xor[2] = (_PAGE_VALID | _PAGE_SZ2GB_4V) ^
2301                         0xfffff80000000000UL;
2302 #endif
2303                 kern_linear_pte_xor[2] |= (_PAGE_CP_4V | _PAGE_CV_4V |
2304                                            _PAGE_P_4V | _PAGE_W_4V);
2305
2306                 i = 3;
2307         }
2308
2309         for (; i < 4; i++)
2310                 kern_linear_pte_xor[i] = kern_linear_pte_xor[i - 1];
2311
2312         pg_iobits = (_PAGE_VALID | _PAGE_PRESENT_4V | __DIRTY_BITS_4V |
2313                      __ACCESS_BITS_4V | _PAGE_E_4V);
2314
2315         _PAGE_SZBITS = _PAGE_SZBITS_4V;
2316         _PAGE_ALL_SZ_BITS = (_PAGE_SZ16GB_4V | _PAGE_SZ2GB_4V |
2317                              _PAGE_SZ256MB_4V | _PAGE_SZ32MB_4V |
2318                              _PAGE_SZ4MB_4V | _PAGE_SZ512K_4V |
2319                              _PAGE_SZ64K_4V | _PAGE_SZ8K_4V);
2320
2321         page_none = _PAGE_PRESENT_4V | _PAGE_ACCESSED_4V | _PAGE_CACHE_4V;
2322         page_shared = (_PAGE_VALID | _PAGE_PRESENT_4V | _PAGE_CACHE_4V |
2323                        __ACCESS_BITS_4V | _PAGE_WRITE_4V | _PAGE_EXEC_4V);
2324         page_copy   = (_PAGE_VALID | _PAGE_PRESENT_4V | _PAGE_CACHE_4V |
2325                        __ACCESS_BITS_4V | _PAGE_EXEC_4V);
2326         page_readonly = (_PAGE_VALID | _PAGE_PRESENT_4V | _PAGE_CACHE_4V |
2327                          __ACCESS_BITS_4V | _PAGE_EXEC_4V);
2328
2329         page_exec_bit = _PAGE_EXEC_4V;
2330
2331         prot_init_common(page_none, page_shared, page_copy, page_readonly,
2332                          page_exec_bit);
2333 }
2334
2335 unsigned long pte_sz_bits(unsigned long sz)
2336 {
2337         if (tlb_type == hypervisor) {
2338                 switch (sz) {
2339                 case 8 * 1024:
2340                 default:
2341                         return _PAGE_SZ8K_4V;
2342                 case 64 * 1024:
2343                         return _PAGE_SZ64K_4V;
2344                 case 512 * 1024:
2345                         return _PAGE_SZ512K_4V;
2346                 case 4 * 1024 * 1024:
2347                         return _PAGE_SZ4MB_4V;
2348                 }
2349         } else {
2350                 switch (sz) {
2351                 case 8 * 1024:
2352                 default:
2353                         return _PAGE_SZ8K_4U;
2354                 case 64 * 1024:
2355                         return _PAGE_SZ64K_4U;
2356                 case 512 * 1024:
2357                         return _PAGE_SZ512K_4U;
2358                 case 4 * 1024 * 1024:
2359                         return _PAGE_SZ4MB_4U;
2360                 }
2361         }
2362 }
2363
2364 pte_t mk_pte_io(unsigned long page, pgprot_t prot, int space, unsigned long page_size)
2365 {
2366         pte_t pte;
2367
2368         pte_val(pte)  = page | pgprot_val(pgprot_noncached(prot));
2369         pte_val(pte) |= (((unsigned long)space) << 32);
2370         pte_val(pte) |= pte_sz_bits(page_size);
2371
2372         return pte;
2373 }
2374
2375 static unsigned long kern_large_tte(unsigned long paddr)
2376 {
2377         unsigned long val;
2378
2379         val = (_PAGE_VALID | _PAGE_SZ4MB_4U |
2380                _PAGE_CP_4U | _PAGE_CV_4U | _PAGE_P_4U |
2381                _PAGE_EXEC_4U | _PAGE_L_4U | _PAGE_W_4U);
2382         if (tlb_type == hypervisor)
2383                 val = (_PAGE_VALID | _PAGE_SZ4MB_4V |
2384                        _PAGE_CP_4V | _PAGE_CV_4V | _PAGE_P_4V |
2385                        _PAGE_EXEC_4V | _PAGE_W_4V);
2386
2387         return val | paddr;
2388 }
2389
2390 /* If not locked, zap it. */
2391 void __flush_tlb_all(void)
2392 {
2393         unsigned long pstate;
2394         int i;
2395
2396         __asm__ __volatile__("flushw\n\t"
2397                              "rdpr      %%pstate, %0\n\t"
2398                              "wrpr      %0, %1, %%pstate"
2399                              : "=r" (pstate)
2400                              : "i" (PSTATE_IE));
2401         if (tlb_type == hypervisor) {
2402                 sun4v_mmu_demap_all();
2403         } else if (tlb_type == spitfire) {
2404                 for (i = 0; i < 64; i++) {
2405                         /* Spitfire Errata #32 workaround */
2406                         /* NOTE: Always runs on spitfire, so no
2407                          *       cheetah+ page size encodings.
2408                          */
2409                         __asm__ __volatile__("stxa      %0, [%1] %2\n\t"
2410                                              "flush     %%g6"
2411                                              : /* No outputs */
2412                                              : "r" (0),
2413                                              "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
2414
2415                         if (!(spitfire_get_dtlb_data(i) & _PAGE_L_4U)) {
2416                                 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
2417                                                      "membar #Sync"
2418                                                      : /* no outputs */
2419                                                      : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
2420                                 spitfire_put_dtlb_data(i, 0x0UL);
2421                         }
2422
2423                         /* Spitfire Errata #32 workaround */
2424                         /* NOTE: Always runs on spitfire, so no
2425                          *       cheetah+ page size encodings.
2426                          */
2427                         __asm__ __volatile__("stxa      %0, [%1] %2\n\t"
2428                                              "flush     %%g6"
2429                                              : /* No outputs */
2430                                              : "r" (0),
2431                                              "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
2432
2433                         if (!(spitfire_get_itlb_data(i) & _PAGE_L_4U)) {
2434                                 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
2435                                                      "membar #Sync"
2436                                                      : /* no outputs */
2437                                                      : "r" (TLB_TAG_ACCESS), "i" (ASI_IMMU));
2438                                 spitfire_put_itlb_data(i, 0x0UL);
2439                         }
2440                 }
2441         } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
2442                 cheetah_flush_dtlb_all();
2443                 cheetah_flush_itlb_all();
2444         }
2445         __asm__ __volatile__("wrpr      %0, 0, %%pstate"
2446                              : : "r" (pstate));
2447 }