arch/mips/mm/r4xx0.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * r4xx0.c: R4000 processor variant specific MMU/Cache routines.
   7  *
   8  * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com)
   9  * Copyright (C) 1997, 1998, 1999, 2000 Ralf Baechle ralf@gnu.org
  10  *
  11  * To do:
  12  *
  13  *  - this code is a overbloated pig
  14  *  - many of the bug workarounds are not efficient at all, but at
  15  *    least they are functional ...
  16  */
  17 #include <linux/init.h>
  18 #include <linux/kernel.h>
  19 #include <linux/sched.h>
  20 #include <linux/mm.h>
  21
  22 #include <asm/bcache.h>
  23 #include <asm/io.h>
  24 #include <asm/page.h>
  25 #include <asm/pgtable.h>
  26 #include <asm/system.h>
  27 #include <asm/bootinfo.h>
  28 #include <asm/mmu_context.h>
  29
  30 /* CP0 hazard avoidance. */
  31 #define BARRIER __asm__ __volatile__(".set noreorder\n\t" \
  32                                      "nop; nop; nop; nop; nop; nop;\n\t" \
  33                                      ".set reorder\n\t")
  34
  35 /* Primary cache parameters. */
  36 static int icache_size, dcache_size; /* Size in bytes */
  37 static int ic_lsize, dc_lsize;       /* LineSize in bytes */
  38
  39 /* Secondary cache (if present) parameters. */
  40 static unsigned int scache_size, sc_lsize;      /* Again, in bytes */
  41
  42 #include <asm/cacheops.h>
  43 #include <asm/r4kcache.h>
  44
  45 #undef DEBUG_CACHE
  46
  47 /*
  48  * Dummy cache handling routines for machines without boardcaches
  49  */
  50 static void no_sc_noop(void) {}
  51
  52 static struct bcache_ops no_sc_ops = {
  53         (void *)no_sc_noop, (void *)no_sc_noop,
  54         (void *)no_sc_noop, (void *)no_sc_noop
  55 };
  56
  57 struct bcache_ops *bcops = &no_sc_ops;
  58
  59 /*
  60  * On processors with QED R4600 style two set assosicative cache
  61  * this is the bit which selects the way in the cache for the
  62  * indexed cachops.
  63  */
  64 #define icache_waybit (icache_size >> 1)
  65 #define dcache_waybit (dcache_size >> 1)
  66
  67 /*
  68  * Zero an entire page.  Basically a simple unrolled loop should do the
  69  * job but we want more performance by saving memory bus bandwidth.  We
  70  * have five flavours of the routine available for:
  71  *
  72  * - 16byte cachelines and no second level cache
  73  * - 32byte cachelines second level cache
  74  * - a version which handles the buggy R4600 v1.x
  75  * - a version which handles the buggy R4600 v2.0
  76  * - Finally a last version without fancy cache games for the SC and MC
  77  *   versions of R4000 and R4400.
  78  */
  79
  80 static void r4k_clear_page_d16(void * page)
  81 {
  82         __asm__ __volatile__(
  83                 ".set\tnoreorder\n\t"
  84                 ".set\tnoat\n\t"
  85                 ".set\tmips3\n\t"
  86                 "daddiu\t$1,%0,%2\n"
  87                 "1:\tcache\t%3,(%0)\n\t"
  88                 "sd\t$0,(%0)\n\t"
  89                 "sd\t$0,8(%0)\n\t"
  90                 "cache\t%3,16(%0)\n\t"
  91                 "sd\t$0,16(%0)\n\t"
  92                 "sd\t$0,24(%0)\n\t"
  93                 "daddiu\t%0,64\n\t"
  94                 "cache\t%3,-32(%0)\n\t"
  95                 "sd\t$0,-32(%0)\n\t"
  96                 "sd\t$0,-24(%0)\n\t"
  97                 "cache\t%3,-16(%0)\n\t"
  98                 "sd\t$0,-16(%0)\n\t"
  99                 "bne\t$1,%0,1b\n\t"
 100                 "sd\t$0,-8(%0)\n\t"
 101                 ".set\tmips0\n\t"
 102                 ".set\tat\n\t"
 103                 ".set\treorder"
 104                 :"=r" (page)
 105                 :"0" (page),
 106                  "I" (PAGE_SIZE),
 107                  "i" (Create_Dirty_Excl_D)
 108                 :"$1","memory");
 109 }
 110
 111 static void r4k_clear_page_d32(void * page)
 112 {
 113         __asm__ __volatile__(
 114                 ".set\tnoreorder\n\t"
 115                 ".set\tnoat\n\t"
 116                 ".set\tmips3\n\t"
 117                 "daddiu\t$1,%0,%2\n"
 118                 "1:\tcache\t%3,(%0)\n\t"
 119                 "sd\t$0,(%0)\n\t"
 120                 "sd\t$0,8(%0)\n\t"
 121                 "sd\t$0,16(%0)\n\t"
 122                 "sd\t$0,24(%0)\n\t"
 123                 "daddiu\t%0,64\n\t"
 124                 "cache\t%3,-32(%0)\n\t"
 125                 "sd\t$0,-32(%0)\n\t"
 126                 "sd\t$0,-24(%0)\n\t"
 127                 "sd\t$0,-16(%0)\n\t"
 128                 "bne\t$1,%0,1b\n\t"
 129                 "sd\t$0,-8(%0)\n\t"
 130                 ".set\tmips0\n\t"
 131                 ".set\tat\n\t"
 132                 ".set\treorder"
 133                 :"=r" (page)
 134                 :"0" (page),
 135                  "I" (PAGE_SIZE),
 136                  "i" (Create_Dirty_Excl_D)
 137                 :"$1","memory");
 138 }
 139
 140
 141 /*
 142  * This flavour of r4k_clear_page is for the R4600 V1.x.  Cite from the
 143  * IDT R4600 V1.7 errata:
 144  *
 145  *  18. The CACHE instructions Hit_Writeback_Invalidate_D, Hit_Writeback_D,
 146  *      Hit_Invalidate_D and Create_Dirty_Excl_D should only be
 147  *      executed if there is no other dcache activity. If the dcache is
 148  *      accessed for another instruction immeidately preceding when these
 149  *      cache instructions are executing, it is possible that the dcache
 150  *      tag match outputs used by these cache instructions will be
 151  *      incorrect. These cache instructions should be preceded by at least
 152  *      four instructions that are not any kind of load or store
 153  *      instruction.
 154  *
 155  *      This is not allowed:    lw
 156  *                              nop
 157  *                              nop
 158  *                              nop
 159  *                              cache       Hit_Writeback_Invalidate_D
 160  *
 161  *      This is allowed:        lw
 162  *                              nop
 163  *                              nop
 164  *                              nop
 165  *                              nop
 166  *                              cache       Hit_Writeback_Invalidate_D
 167  */
 168 static void r4k_clear_page_r4600_v1(void * page)
 169 {
 170         __asm__ __volatile__(
 171                 ".set\tnoreorder\n\t"
 172                 ".set\tnoat\n\t"
 173                 ".set\tmips3\n\t"
 174                 "daddiu\t$1,%0,%2\n"
 175                 "1:\tnop\n\t"
 176                 "nop\n\t"
 177                 "nop\n\t"
 178                 "nop\n\t"
 179                 "cache\t%3,(%0)\n\t"
 180                 "sd\t$0,(%0)\n\t"
 181                 "sd\t$0,8(%0)\n\t"
 182                 "sd\t$0,16(%0)\n\t"
 183                 "sd\t$0,24(%0)\n\t"
 184                 "daddiu\t%0,64\n\t"
 185                 "nop\n\t"
 186                 "nop\n\t"
 187                 "nop\n\t"
 188                 "cache\t%3,-32(%0)\n\t"
 189                 "sd\t$0,-32(%0)\n\t"
 190                 "sd\t$0,-24(%0)\n\t"
 191                 "sd\t$0,-16(%0)\n\t"
 192                 "bne\t$1,%0,1b\n\t"
 193                 "sd\t$0,-8(%0)\n\t"
 194                 ".set\tmips0\n\t"
 195                 ".set\tat\n\t"
 196                 ".set\treorder"
 197                 :"=r" (page)
 198                 :"0" (page),
 199                  "I" (PAGE_SIZE),
 200                  "i" (Create_Dirty_Excl_D)
 201                 :"$1","memory");
 202 }
 203
 204 /*
 205  * And this one is for the R4600 V2.0
 206  */
 207 static void r4k_clear_page_r4600_v2(void * page)
 208 {
 209         unsigned int flags;
 210
 211         save_and_cli(flags);
 212         *(volatile unsigned int *)KSEG1;
 213         __asm__ __volatile__(
 214                 ".set\tnoreorder\n\t"
 215                 ".set\tnoat\n\t"
 216                 ".set\tmips3\n\t"
 217                 "daddiu\t$1,%0,%2\n"
 218                 "1:\tcache\t%3,(%0)\n\t"
 219                 "sd\t$0,(%0)\n\t"
 220                 "sd\t$0,8(%0)\n\t"
 221                 "sd\t$0,16(%0)\n\t"
 222                 "sd\t$0,24(%0)\n\t"
 223                 "daddiu\t%0,64\n\t"
 224                 "cache\t%3,-32(%0)\n\t"
 225                 "sd\t$0,-32(%0)\n\t"
 226                 "sd\t$0,-24(%0)\n\t"
 227                 "sd\t$0,-16(%0)\n\t"
 228                 "bne\t$1,%0,1b\n\t"
 229                 "sd\t$0,-8(%0)\n\t"
 230                 ".set\tmips0\n\t"
 231                 ".set\tat\n\t"
 232                 ".set\treorder"
 233                 :"=r" (page)
 234                 :"0" (page),
 235                  "I" (PAGE_SIZE),
 236                  "i" (Create_Dirty_Excl_D)
 237                 :"$1","memory");
 238         restore_flags(flags);
 239 }
 240
 241 /*
 242  * The next 4 versions are optimized for all possible scache configurations
 243  * of the SC / MC versions of R4000 and R4400 ...
 244  *
 245  * Todo: For even better performance we should have a routine optimized for
 246  * every legal combination of dcache / scache linesize.  When I (Ralf) tried
 247  * this the kernel crashed shortly after mounting the root filesystem.  CPU
 248  * bug?  Weirdo cache instruction semantics?
 249  */
 250 static void r4k_clear_page_s16(void * page)
 251 {
 252         __asm__ __volatile__(
 253                 ".set\tnoreorder\n\t"
 254                 ".set\tnoat\n\t"
 255                 ".set\tmips3\n\t"
 256                 "daddiu\t$1,%0,%2\n"
 257                 "1:\tcache\t%3,(%0)\n\t"
 258                 "sd\t$0,(%0)\n\t"
 259                 "sd\t$0,8(%0)\n\t"
 260                 "cache\t%3,16(%0)\n\t"
 261                 "sd\t$0,16(%0)\n\t"
 262                 "sd\t$0,24(%0)\n\t"
 263                 "daddiu\t%0,64\n\t"
 264                 "cache\t%3,-32(%0)\n\t"
 265                 "sd\t$0,-32(%0)\n\t"
 266                 "sd\t$0,-24(%0)\n\t"
 267                 "cache\t%3,-16(%0)\n\t"
 268                 "sd\t$0,-16(%0)\n\t"
 269                 "bne\t$1,%0,1b\n\t"
 270                 "sd\t$0,-8(%0)\n\t"
 271                 ".set\tmips0\n\t"
 272                 ".set\tat\n\t"
 273                 ".set\treorder"
 274                 :"=r" (page)
 275                 :"0" (page),
 276                  "I" (PAGE_SIZE),
 277                  "i" (Create_Dirty_Excl_SD)
 278                 :"$1","memory");
 279 }
 280
 281 static void r4k_clear_page_s32(void * page)
 282 {
 283         __asm__ __volatile__(
 284                 ".set\tnoreorder\n\t"
 285                 ".set\tnoat\n\t"
 286                 ".set\tmips3\n\t"
 287                 "daddiu\t$1,%0,%2\n"
 288                 "1:\tcache\t%3,(%0)\n\t"
 289                 "sd\t$0,(%0)\n\t"
 290                 "sd\t$0,8(%0)\n\t"
 291                 "sd\t$0,16(%0)\n\t"
 292                 "sd\t$0,24(%0)\n\t"
 293                 "daddiu\t%0,64\n\t"
 294                 "cache\t%3,-32(%0)\n\t"
 295                 "sd\t$0,-32(%0)\n\t"
 296                 "sd\t$0,-24(%0)\n\t"
 297                 "sd\t$0,-16(%0)\n\t"
 298                 "bne\t$1,%0,1b\n\t"
 299                 "sd\t$0,-8(%0)\n\t"
 300                 ".set\tmips0\n\t"
 301                 ".set\tat\n\t"
 302                 ".set\treorder"
 303                 :"=r" (page)
 304                 :"0" (page),
 305                  "I" (PAGE_SIZE),
 306                  "i" (Create_Dirty_Excl_SD)
 307                 :"$1","memory");
 308 }
 309
 310 static void r4k_clear_page_s64(void * page)
 311 {
 312         __asm__ __volatile__(
 313                 ".set\tnoreorder\n\t"
 314                 ".set\tnoat\n\t"
 315                 ".set\tmips3\n\t"
 316                 "daddiu\t$1,%0,%2\n"
 317                 "1:\tcache\t%3,(%0)\n\t"
 318                 "sd\t$0,(%0)\n\t"
 319                 "sd\t$0,8(%0)\n\t"
 320                 "sd\t$0,16(%0)\n\t"
 321                 "sd\t$0,24(%0)\n\t"
 322                 "daddiu\t%0,64\n\t"
 323                 "sd\t$0,-32(%0)\n\t"
 324                 "sd\t$0,-24(%0)\n\t"
 325                 "sd\t$0,-16(%0)\n\t"
 326                 "bne\t$1,%0,1b\n\t"
 327                 "sd\t$0,-8(%0)\n\t"
 328                 ".set\tmips0\n\t"
 329                 ".set\tat\n\t"
 330                 ".set\treorder"
 331                 :"=r" (page)
 332                 :"0" (page),
 333                  "I" (PAGE_SIZE),
 334                  "i" (Create_Dirty_Excl_SD)
 335                 :"$1","memory");
 336 }
 337
 338 static void r4k_clear_page_s128(void * page)
 339 {
 340         __asm__ __volatile__(
 341                 ".set\tnoreorder\n\t"
 342                 ".set\tnoat\n\t"
 343                 ".set\tmips3\n\t"
 344                 "daddiu\t$1,%0,%2\n"
 345                 "1:\tcache\t%3,(%0)\n\t"
 346                 "sd\t$0,(%0)\n\t"
 347                 "sd\t$0,8(%0)\n\t"
 348                 "sd\t$0,16(%0)\n\t"
 349                 "sd\t$0,24(%0)\n\t"
 350                 "sd\t$0,32(%0)\n\t"
 351                 "sd\t$0,40(%0)\n\t"
 352                 "sd\t$0,48(%0)\n\t"
 353                 "sd\t$0,56(%0)\n\t"
 354                 "daddiu\t%0,128\n\t"
 355                 "sd\t$0,-64(%0)\n\t"
 356                 "sd\t$0,-56(%0)\n\t"
 357                 "sd\t$0,-48(%0)\n\t"
 358                 "sd\t$0,-40(%0)\n\t"
 359                 "sd\t$0,-32(%0)\n\t"
 360                 "sd\t$0,-24(%0)\n\t"
 361                 "sd\t$0,-16(%0)\n\t"
 362                 "bne\t$1,%0,1b\n\t"
 363                 "sd\t$0,-8(%0)\n\t"
 364                 ".set\tmips0\n\t"
 365                 ".set\tat\n\t"
 366                 ".set\treorder"
 367                 :"=r" (page)
 368                 :"0" (page),
 369                  "I" (PAGE_SIZE),
 370                  "i" (Create_Dirty_Excl_SD)
 371                 :"$1","memory");
 372 }
 373
 374
 375 /*
 376  * This is still inefficient.  We only can do better if we know the
 377  * virtual address where the copy will be accessed.
 378  */
 379
 380 static void r4k_copy_page_d16(void * to, void * from)
 381 {
 382         unsigned long dummy1, dummy2;
 383         unsigned long reg1, reg2, reg3, reg4;
 384
 385         __asm__ __volatile__(
 386                 ".set\tnoreorder\n\t"
 387                 ".set\tnoat\n\t"
 388                 ".set\tmips3\n\t"
 389                 "daddiu\t$1,%0,%8\n"
 390                 "1:\tcache\t%9,(%0)\n\t"
 391                 "lw\t%2,(%1)\n\t"
 392                 "lw\t%3,4(%1)\n\t"
 393                 "lw\t%4,8(%1)\n\t"
 394                 "lw\t%5,12(%1)\n\t"
 395                 "sw\t%2,(%0)\n\t"
 396                 "sw\t%3,4(%0)\n\t"
 397                 "sw\t%4,8(%0)\n\t"
 398                 "sw\t%5,12(%0)\n\t"
 399                 "cache\t%9,16(%0)\n\t"
 400                 "lw\t%2,16(%1)\n\t"
 401                 "lw\t%3,20(%1)\n\t"
 402                 "lw\t%4,24(%1)\n\t"
 403                 "lw\t%5,28(%1)\n\t"
 404                 "sw\t%2,16(%0)\n\t"
 405                 "sw\t%3,20(%0)\n\t"
 406                 "sw\t%4,24(%0)\n\t"
 407                 "sw\t%5,28(%0)\n\t"
 408                 "cache\t%9,32(%0)\n\t"
 409                 "daddiu\t%0,64\n\t"
 410                 "daddiu\t%1,64\n\t"
 411                 "lw\t%2,-32(%1)\n\t"
 412                 "lw\t%3,-28(%1)\n\t"
 413                 "lw\t%4,-24(%1)\n\t"
 414                 "lw\t%5,-20(%1)\n\t"
 415                 "sw\t%2,-32(%0)\n\t"
 416                 "sw\t%3,-28(%0)\n\t"
 417                 "sw\t%4,-24(%0)\n\t"
 418                 "sw\t%5,-20(%0)\n\t"
 419                 "cache\t%9,-16(%0)\n\t"
 420                 "lw\t%2,-16(%1)\n\t"
 421                 "lw\t%3,-12(%1)\n\t"
 422                 "lw\t%4,-8(%1)\n\t"
 423                 "lw\t%5,-4(%1)\n\t"
 424                 "sw\t%2,-16(%0)\n\t"
 425                 "sw\t%3,-12(%0)\n\t"
 426                 "sw\t%4,-8(%0)\n\t"
 427                 "bne\t$1,%0,1b\n\t"
 428                 "sw\t%5,-4(%0)\n\t"
 429                 ".set\tmips0\n\t"
 430                 ".set\tat\n\t"
 431                 ".set\treorder"
 432                 :"=r" (dummy1), "=r" (dummy2),
 433                  "=&r" (reg1), "=&r" (reg2), "=&r" (reg3), "=&r" (reg4)
 434                 :"0" (to), "1" (from),
 435                  "I" (PAGE_SIZE),
 436                  "i" (Create_Dirty_Excl_D));
 437 }
 438
 439 static void r4k_copy_page_d32(void * to, void * from)
 440 {
 441         unsigned long dummy1, dummy2;
 442         unsigned long reg1, reg2, reg3, reg4;
 443
 444         __asm__ __volatile__(
 445                 ".set\tnoreorder\n\t"
 446                 ".set\tnoat\n\t"
 447                 ".set\tmips3\n\t"
 448                 "daddiu\t$1,%0,%8\n"
 449                 "1:\tcache\t%9,(%0)\n\t"
 450                 "lw\t%2,(%1)\n\t"
 451                 "lw\t%3,4(%1)\n\t"
 452                 "lw\t%4,8(%1)\n\t"
 453                 "lw\t%5,12(%1)\n\t"
 454                 "sw\t%2,(%0)\n\t"
 455                 "sw\t%3,4(%0)\n\t"
 456                 "sw\t%4,8(%0)\n\t"
 457                 "sw\t%5,12(%0)\n\t"
 458                 "lw\t%2,16(%1)\n\t"
 459                 "lw\t%3,20(%1)\n\t"
 460                 "lw\t%4,24(%1)\n\t"
 461                 "lw\t%5,28(%1)\n\t"
 462                 "sw\t%2,16(%0)\n\t"
 463                 "sw\t%3,20(%0)\n\t"
 464                 "sw\t%4,24(%0)\n\t"
 465                 "sw\t%5,28(%0)\n\t"
 466                 "cache\t%9,32(%0)\n\t"
 467                 "daddiu\t%0,64\n\t"
 468                 "daddiu\t%1,64\n\t"
 469                 "lw\t%2,-32(%1)\n\t"
 470                 "lw\t%3,-28(%1)\n\t"
 471                 "lw\t%4,-24(%1)\n\t"
 472                 "lw\t%5,-20(%1)\n\t"
 473                 "sw\t%2,-32(%0)\n\t"
 474                 "sw\t%3,-28(%0)\n\t"
 475                 "sw\t%4,-24(%0)\n\t"
 476                 "sw\t%5,-20(%0)\n\t"
 477                 "lw\t%2,-16(%1)\n\t"
 478                 "lw\t%3,-12(%1)\n\t"
 479                 "lw\t%4,-8(%1)\n\t"
 480                 "lw\t%5,-4(%1)\n\t"
 481                 "sw\t%2,-16(%0)\n\t"
 482                 "sw\t%3,-12(%0)\n\t"
 483                 "sw\t%4,-8(%0)\n\t"
 484                 "bne\t$1,%0,1b\n\t"
 485                 "sw\t%5,-4(%0)\n\t"
 486                 ".set\tmips0\n\t"
 487                 ".set\tat\n\t"
 488                 ".set\treorder"
 489                 :"=r" (dummy1), "=r" (dummy2),
 490                  "=&r" (reg1), "=&r" (reg2), "=&r" (reg3), "=&r" (reg4)
 491                 :"0" (to), "1" (from),
 492                  "I" (PAGE_SIZE),
 493                  "i" (Create_Dirty_Excl_D));
 494 }
 495
 496 /*
 497  * Again a special version for the R4600 V1.x
 498  */
 499 static void r4k_copy_page_r4600_v1(void * to, void * from)
 500 {
 501         unsigned long dummy1, dummy2;
 502         unsigned long reg1, reg2, reg3, reg4;
 503
 504         __asm__ __volatile__(
 505                 ".set\tnoreorder\n\t"
 506                 ".set\tnoat\n\t"
 507                 ".set\tmips3\n\t"
 508                 "daddiu\t$1,%0,%8\n"
 509                 "1:\tnop\n\t"
 510                 "nop\n\t"
 511                 "nop\n\t"
 512                 "nop\n\t"
 513                 "\tcache\t%9,(%0)\n\t"
 514                 "lw\t%2,(%1)\n\t"
 515                 "lw\t%3,4(%1)\n\t"
 516                 "lw\t%4,8(%1)\n\t"
 517                 "lw\t%5,12(%1)\n\t"
 518                 "sw\t%2,(%0)\n\t"
 519                 "sw\t%3,4(%0)\n\t"
 520                 "sw\t%4,8(%0)\n\t"
 521                 "sw\t%5,12(%0)\n\t"
 522                 "lw\t%2,16(%1)\n\t"
 523                 "lw\t%3,20(%1)\n\t"
 524                 "lw\t%4,24(%1)\n\t"
 525                 "lw\t%5,28(%1)\n\t"
 526                 "sw\t%2,16(%0)\n\t"
 527                 "sw\t%3,20(%0)\n\t"
 528                 "sw\t%4,24(%0)\n\t"
 529                 "sw\t%5,28(%0)\n\t"
 530                 "nop\n\t"
 531                 "nop\n\t"
 532                 "nop\n\t"
 533                 "nop\n\t"
 534                 "cache\t%9,32(%0)\n\t"
 535                 "daddiu\t%0,64\n\t"
 536                 "daddiu\t%1,64\n\t"
 537                 "lw\t%2,-32(%1)\n\t"
 538                 "lw\t%3,-28(%1)\n\t"
 539                 "lw\t%4,-24(%1)\n\t"
 540                 "lw\t%5,-20(%1)\n\t"
 541                 "sw\t%2,-32(%0)\n\t"
 542                 "sw\t%3,-28(%0)\n\t"
 543                 "sw\t%4,-24(%0)\n\t"
 544                 "sw\t%5,-20(%0)\n\t"
 545                 "lw\t%2,-16(%1)\n\t"
 546                 "lw\t%3,-12(%1)\n\t"
 547                 "lw\t%4,-8(%1)\n\t"
 548                 "lw\t%5,-4(%1)\n\t"
 549                 "sw\t%2,-16(%0)\n\t"
 550                 "sw\t%3,-12(%0)\n\t"
 551                 "sw\t%4,-8(%0)\n\t"
 552                 "bne\t$1,%0,1b\n\t"
 553                 "sw\t%5,-4(%0)\n\t"
 554                 ".set\tmips0\n\t"
 555                 ".set\tat\n\t"
 556                 ".set\treorder"
 557                 :"=r" (dummy1), "=r" (dummy2),
 558                  "=&r" (reg1), "=&r" (reg2), "=&r" (reg3), "=&r" (reg4)
 559                 :"0" (to), "1" (from),
 560                  "I" (PAGE_SIZE),
 561                  "i" (Create_Dirty_Excl_D));
 562 }
 563
 564 static void r4k_copy_page_r4600_v2(void * to, void * from)
 565 {
 566         unsigned long dummy1, dummy2;
 567         unsigned long reg1, reg2, reg3, reg4;
 568         unsigned int flags;
 569
 570         __save_and_cli(flags);
 571         __asm__ __volatile__(
 572                 ".set\tnoreorder\n\t"
 573                 ".set\tnoat\n\t"
 574                 ".set\tmips3\n\t"
 575                 "daddiu\t$1,%0,%8\n"
 576                 "1:\tnop\n\t"
 577                 "nop\n\t"
 578                 "nop\n\t"
 579                 "nop\n\t"
 580                 "\tcache\t%9,(%0)\n\t"
 581                 "lw\t%2,(%1)\n\t"
 582                 "lw\t%3,4(%1)\n\t"
 583                 "lw\t%4,8(%1)\n\t"
 584                 "lw\t%5,12(%1)\n\t"
 585                 "sw\t%2,(%0)\n\t"
 586                 "sw\t%3,4(%0)\n\t"
 587                 "sw\t%4,8(%0)\n\t"
 588                 "sw\t%5,12(%0)\n\t"
 589                 "lw\t%2,16(%1)\n\t"
 590                 "lw\t%3,20(%1)\n\t"
 591                 "lw\t%4,24(%1)\n\t"
 592                 "lw\t%5,28(%1)\n\t"
 593                 "sw\t%2,16(%0)\n\t"
 594                 "sw\t%3,20(%0)\n\t"
 595                 "sw\t%4,24(%0)\n\t"
 596                 "sw\t%5,28(%0)\n\t"
 597                 "nop\n\t"
 598                 "nop\n\t"
 599                 "nop\n\t"
 600                 "nop\n\t"
 601                 "cache\t%9,32(%0)\n\t"
 602                 "daddiu\t%0,64\n\t"
 603                 "daddiu\t%1,64\n\t"
 604                 "lw\t%2,-32(%1)\n\t"
 605                 "lw\t%3,-28(%1)\n\t"
 606                 "lw\t%4,-24(%1)\n\t"
 607                 "lw\t%5,-20(%1)\n\t"
 608                 "sw\t%2,-32(%0)\n\t"
 609                 "sw\t%3,-28(%0)\n\t"
 610                 "sw\t%4,-24(%0)\n\t"
 611                 "sw\t%5,-20(%0)\n\t"
 612                 "lw\t%2,-16(%1)\n\t"
 613                 "lw\t%3,-12(%1)\n\t"
 614                 "lw\t%4,-8(%1)\n\t"
 615                 "lw\t%5,-4(%1)\n\t"
 616                 "sw\t%2,-16(%0)\n\t"
 617                 "sw\t%3,-12(%0)\n\t"
 618                 "sw\t%4,-8(%0)\n\t"
 619                 "bne\t$1,%0,1b\n\t"
 620                 "sw\t%5,-4(%0)\n\t"
 621                 ".set\tmips0\n\t"
 622                 ".set\tat\n\t"
 623                 ".set\treorder"
 624                 :"=r" (dummy1), "=r" (dummy2),
 625                  "=&r" (reg1), "=&r" (reg2), "=&r" (reg3), "=&r" (reg4)
 626                 :"0" (to), "1" (from),
 627                  "I" (PAGE_SIZE),
 628                  "i" (Create_Dirty_Excl_D));
 629         restore_flags(flags);
 630 }
 631
 632 /*
 633  * These are for R4000SC / R4400MC
 634  */
 635 static void r4k_copy_page_s16(void * to, void * from)
 636 {
 637         unsigned long dummy1, dummy2;
 638         unsigned long reg1, reg2, reg3, reg4;
 639
 640         __asm__ __volatile__(
 641                 ".set\tnoreorder\n\t"
 642                 ".set\tnoat\n\t"
 643                 ".set\tmips3\n\t"
 644                 "daddiu\t$1,%0,%8\n"
 645                 "1:\tcache\t%9,(%0)\n\t"
 646                 "lw\t%2,(%1)\n\t"
 647                 "lw\t%3,4(%1)\n\t"
 648                 "lw\t%4,8(%1)\n\t"
 649                 "lw\t%5,12(%1)\n\t"
 650                 "sw\t%2,(%0)\n\t"
 651                 "sw\t%3,4(%0)\n\t"
 652                 "sw\t%4,8(%0)\n\t"
 653                 "sw\t%5,12(%0)\n\t"
 654                 "cache\t%9,16(%0)\n\t"
 655                 "lw\t%2,16(%1)\n\t"
 656                 "lw\t%3,20(%1)\n\t"
 657                 "lw\t%4,24(%1)\n\t"
 658                 "lw\t%5,28(%1)\n\t"
 659                 "sw\t%2,16(%0)\n\t"
 660                 "sw\t%3,20(%0)\n\t"
 661                 "sw\t%4,24(%0)\n\t"
 662                 "sw\t%5,28(%0)\n\t"
 663                 "cache\t%9,32(%0)\n\t"
 664                 "daddiu\t%0,64\n\t"
 665                 "daddiu\t%1,64\n\t"
 666                 "lw\t%2,-32(%1)\n\t"
 667                 "lw\t%3,-28(%1)\n\t"
 668                 "lw\t%4,-24(%1)\n\t"
 669                 "lw\t%5,-20(%1)\n\t"
 670                 "sw\t%2,-32(%0)\n\t"
 671                 "sw\t%3,-28(%0)\n\t"
 672                 "sw\t%4,-24(%0)\n\t"
 673                 "sw\t%5,-20(%0)\n\t"
 674                 "cache\t%9,-16(%0)\n\t"
 675                 "lw\t%2,-16(%1)\n\t"
 676                 "lw\t%3,-12(%1)\n\t"
 677                 "lw\t%4,-8(%1)\n\t"
 678                 "lw\t%5,-4(%1)\n\t"
 679                 "sw\t%2,-16(%0)\n\t"
 680                 "sw\t%3,-12(%0)\n\t"
 681                 "sw\t%4,-8(%0)\n\t"
 682                 "bne\t$1,%0,1b\n\t"
 683                 "sw\t%5,-4(%0)\n\t"
 684                 ".set\tmips0\n\t"
 685                 ".set\tat\n\t"
 686                 ".set\treorder"
 687                 :"=r" (dummy1), "=r" (dummy2),
 688                  "=&r" (reg1), "=&r" (reg2), "=&r" (reg3), "=&r" (reg4)
 689                 :"0" (to), "1" (from),
 690                  "I" (PAGE_SIZE),
 691                  "i" (Create_Dirty_Excl_SD));
 692 }
 693
 694 static void r4k_copy_page_s32(void * to, void * from)
 695 {
 696         unsigned long dummy1, dummy2;
 697         unsigned long reg1, reg2, reg3, reg4;
 698
 699         __asm__ __volatile__(
 700                 ".set\tnoreorder\n\t"
 701                 ".set\tnoat\n\t"
 702                 ".set\tmips3\n\t"
 703                 "daddiu\t$1,%0,%8\n"
 704                 "1:\tcache\t%9,(%0)\n\t"
 705                 "lw\t%2,(%1)\n\t"
 706                 "lw\t%3,4(%1)\n\t"
 707                 "lw\t%4,8(%1)\n\t"
 708                 "lw\t%5,12(%1)\n\t"
 709                 "sw\t%2,(%0)\n\t"
 710                 "sw\t%3,4(%0)\n\t"
 711                 "sw\t%4,8(%0)\n\t"
 712                 "sw\t%5,12(%0)\n\t"
 713                 "lw\t%2,16(%1)\n\t"
 714                 "lw\t%3,20(%1)\n\t"
 715                 "lw\t%4,24(%1)\n\t"
 716                 "lw\t%5,28(%1)\n\t"
 717                 "sw\t%2,16(%0)\n\t"
 718                 "sw\t%3,20(%0)\n\t"
 719                 "sw\t%4,24(%0)\n\t"
 720                 "sw\t%5,28(%0)\n\t"
 721                 "cache\t%9,32(%0)\n\t"
 722                 "daddiu\t%0,64\n\t"
 723                 "daddiu\t%1,64\n\t"
 724                 "lw\t%2,-32(%1)\n\t"
 725                 "lw\t%3,-28(%1)\n\t"
 726                 "lw\t%4,-24(%1)\n\t"
 727                 "lw\t%5,-20(%1)\n\t"
 728                 "sw\t%2,-32(%0)\n\t"
 729                 "sw\t%3,-28(%0)\n\t"
 730                 "sw\t%4,-24(%0)\n\t"
 731                 "sw\t%5,-20(%0)\n\t"
 732                 "lw\t%2,-16(%1)\n\t"
 733                 "lw\t%3,-12(%1)\n\t"
 734                 "lw\t%4,-8(%1)\n\t"
 735                 "lw\t%5,-4(%1)\n\t"
 736                 "sw\t%2,-16(%0)\n\t"
 737                 "sw\t%3,-12(%0)\n\t"
 738                 "sw\t%4,-8(%0)\n\t"
 739                 "bne\t$1,%0,1b\n\t"
 740                 "sw\t%5,-4(%0)\n\t"
 741                 ".set\tmips0\n\t"
 742                 ".set\tat\n\t"
 743                 ".set\treorder"
 744                 :"=r" (dummy1), "=r" (dummy2),
 745                  "=&r" (reg1), "=&r" (reg2), "=&r" (reg3), "=&r" (reg4)
 746                 :"0" (to), "1" (from),
 747                  "I" (PAGE_SIZE),
 748                  "i" (Create_Dirty_Excl_SD));
 749 }
 750
 751 static void r4k_copy_page_s64(void * to, void * from)
 752 {
 753         unsigned long dummy1, dummy2;
 754         unsigned long reg1, reg2, reg3, reg4;
 755
 756         __asm__ __volatile__(
 757                 ".set\tnoreorder\n\t"
 758                 ".set\tnoat\n\t"
 759                 ".set\tmips3\n\t"
 760                 "daddiu\t$1,%0,%8\n"
 761                 "1:\tcache\t%9,(%0)\n\t"
 762                 "lw\t%2,(%1)\n\t"
 763                 "lw\t%3,4(%1)\n\t"
 764                 "lw\t%4,8(%1)\n\t"
 765                 "lw\t%5,12(%1)\n\t"
 766                 "sw\t%2,(%0)\n\t"
 767                 "sw\t%3,4(%0)\n\t"
 768                 "sw\t%4,8(%0)\n\t"
 769                 "sw\t%5,12(%0)\n\t"
 770                 "lw\t%2,16(%1)\n\t"
 771                 "lw\t%3,20(%1)\n\t"
 772                 "lw\t%4,24(%1)\n\t"
 773                 "lw\t%5,28(%1)\n\t"
 774                 "sw\t%2,16(%0)\n\t"
 775                 "sw\t%3,20(%0)\n\t"
 776                 "sw\t%4,24(%0)\n\t"
 777                 "sw\t%5,28(%0)\n\t"
 778                 "daddiu\t%0,64\n\t"
 779                 "daddiu\t%1,64\n\t"
 780                 "lw\t%2,-32(%1)\n\t"
 781                 "lw\t%3,-28(%1)\n\t"
 782                 "lw\t%4,-24(%1)\n\t"
 783                 "lw\t%5,-20(%1)\n\t"
 784                 "sw\t%2,-32(%0)\n\t"
 785                 "sw\t%3,-28(%0)\n\t"
 786                 "sw\t%4,-24(%0)\n\t"
 787                 "sw\t%5,-20(%0)\n\t"
 788                 "lw\t%2,-16(%1)\n\t"
 789                 "lw\t%3,-12(%1)\n\t"
 790                 "lw\t%4,-8(%1)\n\t"
 791                 "lw\t%5,-4(%1)\n\t"
 792                 "sw\t%2,-16(%0)\n\t"
 793                 "sw\t%3,-12(%0)\n\t"
 794                 "sw\t%4,-8(%0)\n\t"
 795                 "bne\t$1,%0,1b\n\t"
 796                 "sw\t%5,-4(%0)\n\t"
 797                 ".set\tmips0\n\t"
 798                 ".set\tat\n\t"
 799                 ".set\treorder"
 800                 :"=r" (dummy1), "=r" (dummy2),
 801                  "=&r" (reg1), "=&r" (reg2), "=&r" (reg3), "=&r" (reg4)
 802                 :"0" (to), "1" (from),
 803                  "I" (PAGE_SIZE),
 804                  "i" (Create_Dirty_Excl_SD));
 805 }
 806
 807 static void r4k_copy_page_s128(void * to, void * from)
 808 {
 809         unsigned long dummy1, dummy2;
 810         unsigned long reg1, reg2, reg3, reg4;
 811
 812         __asm__ __volatile__(
 813                 ".set\tnoreorder\n\t"
 814                 ".set\tnoat\n\t"
 815                 ".set\tmips3\n\t"
 816                 "daddiu\t$1,%0,%8\n"
 817                 "1:\tcache\t%9,(%0)\n\t"
 818                 "lw\t%2,(%1)\n\t"
 819                 "lw\t%3,4(%1)\n\t"
 820                 "lw\t%4,8(%1)\n\t"
 821                 "lw\t%5,12(%1)\n\t"
 822                 "sw\t%2,(%0)\n\t"
 823                 "sw\t%3,4(%0)\n\t"
 824                 "sw\t%4,8(%0)\n\t"
 825                 "sw\t%5,12(%0)\n\t"
 826                 "lw\t%2,16(%1)\n\t"
 827                 "lw\t%3,20(%1)\n\t"
 828                 "lw\t%4,24(%1)\n\t"
 829                 "lw\t%5,28(%1)\n\t"
 830                 "sw\t%2,16(%0)\n\t"
 831                 "sw\t%3,20(%0)\n\t"
 832                 "sw\t%4,24(%0)\n\t"
 833                 "sw\t%5,28(%0)\n\t"
 834                 "lw\t%2,32(%1)\n\t"
 835                 "lw\t%3,36(%1)\n\t"
 836                 "lw\t%4,40(%1)\n\t"
 837                 "lw\t%5,44(%1)\n\t"
 838                 "sw\t%2,32(%0)\n\t"
 839                 "sw\t%3,36(%0)\n\t"
 840                 "sw\t%4,40(%0)\n\t"
 841                 "sw\t%5,44(%0)\n\t"
 842                 "lw\t%2,48(%1)\n\t"
 843                 "lw\t%3,52(%1)\n\t"
 844                 "lw\t%4,56(%1)\n\t"
 845                 "lw\t%5,60(%1)\n\t"
 846                 "sw\t%2,48(%0)\n\t"
 847                 "sw\t%3,52(%0)\n\t"
 848                 "sw\t%4,56(%0)\n\t"
 849                 "sw\t%5,60(%0)\n\t"
 850                 "daddiu\t%0,128\n\t"
 851                 "daddiu\t%1,128\n\t"
 852                 "lw\t%2,-64(%1)\n\t"
 853                 "lw\t%3,-60(%1)\n\t"
 854                 "lw\t%4,-56(%1)\n\t"
 855                 "lw\t%5,-52(%1)\n\t"
 856                 "sw\t%2,-64(%0)\n\t"
 857                 "sw\t%3,-60(%0)\n\t"
 858                 "sw\t%4,-56(%0)\n\t"
 859                 "sw\t%5,-52(%0)\n\t"
 860                 "lw\t%2,-48(%1)\n\t"
 861                 "lw\t%3,-44(%1)\n\t"
 862                 "lw\t%4,-40(%1)\n\t"
 863                 "lw\t%5,-36(%1)\n\t"
 864                 "sw\t%2,-48(%0)\n\t"
 865                 "sw\t%3,-44(%0)\n\t"
 866                 "sw\t%4,-40(%0)\n\t"
 867                 "sw\t%5,-36(%0)\n\t"
 868                 "lw\t%2,-32(%1)\n\t"
 869                 "lw\t%3,-28(%1)\n\t"
 870                 "lw\t%4,-24(%1)\n\t"
 871                 "lw\t%5,-20(%1)\n\t"
 872                 "sw\t%2,-32(%0)\n\t"
 873                 "sw\t%3,-28(%0)\n\t"
 874                 "sw\t%4,-24(%0)\n\t"
 875                 "sw\t%5,-20(%0)\n\t"
 876                 "lw\t%2,-16(%1)\n\t"
 877                 "lw\t%3,-12(%1)\n\t"
 878                 "lw\t%4,-8(%1)\n\t"
 879                 "lw\t%5,-4(%1)\n\t"
 880                 "sw\t%2,-16(%0)\n\t"
 881                 "sw\t%3,-12(%0)\n\t"
 882                 "sw\t%4,-8(%0)\n\t"
 883                 "bne\t$1,%0,1b\n\t"
 884                 "sw\t%5,-4(%0)\n\t"
 885                 ".set\tmips0\n\t"
 886                 ".set\tat\n\t"
 887                 ".set\treorder"
 888                 :"=r" (dummy1), "=r" (dummy2),
 889                  "=&r" (reg1), "=&r" (reg2), "=&r" (reg3), "=&r" (reg4)
 890                 :"0" (to), "1" (from),
 891                  "I" (PAGE_SIZE),
 892                  "i" (Create_Dirty_Excl_SD));
 893 }
 894
 895
 896 /*
 897  * If you think for one second that this stuff coming up is a lot
 898  * of bulky code eating too many kernel cache lines.  Think _again_.
 899  *
 900  * Consider:
 901  * 1) Taken branches have a 3 cycle penalty on R4k
 902  * 2) The branch itself is a real dead cycle on even R4600/R5000.
 903  * 3) Only one of the following variants of each type is even used by
 904  *    the kernel based upon the cache parameters we detect at boot time.
 905  *
 906  * QED.
 907  */
 908
 909 static inline void r4k_flush_cache_all_s16d16i16(void)
 910 {
 911         unsigned long flags;
 912
 913         save_and_cli(flags);
 914         blast_dcache16(); blast_icache16(); blast_scache16();
 915         restore_flags(flags);
 916 }
 917
 918 static inline void r4k_flush_cache_all_s32d16i16(void)
 919 {
 920         unsigned long flags;
 921
 922         save_and_cli(flags);
 923         blast_dcache16(); blast_icache16(); blast_scache32();
 924         restore_flags(flags);
 925 }
 926
 927 static inline void r4k_flush_cache_all_s64d16i16(void)
 928 {
 929         unsigned long flags;
 930
 931         save_and_cli(flags);
 932         blast_dcache16(); blast_icache16(); blast_scache64();
 933         restore_flags(flags);
 934 }
 935
 936 static inline void r4k_flush_cache_all_s128d16i16(void)
 937 {
 938         unsigned long flags;
 939
 940         save_and_cli(flags);
 941         blast_dcache16(); blast_icache16(); blast_scache128();
 942         restore_flags(flags);
 943 }
 944
 945 static inline void r4k_flush_cache_all_s32d32i32(void)
 946 {
 947         unsigned long flags;
 948
 949         save_and_cli(flags);
 950         blast_dcache32(); blast_icache32(); blast_scache32();
 951         restore_flags(flags);
 952 }
 953
 954 static inline void r4k_flush_cache_all_s64d32i32(void)
 955 {
 956         unsigned long flags;
 957
 958         save_and_cli(flags);
 959         blast_dcache32(); blast_icache32(); blast_scache64();
 960         restore_flags(flags);
 961 }
 962
 963 static inline void r4k_flush_cache_all_s128d32i32(void)
 964 {
 965         unsigned long flags;
 966
 967         save_and_cli(flags);
 968         blast_dcache32(); blast_icache32(); blast_scache128();
 969         restore_flags(flags);
 970 }
 971
 972 static inline void r4k_flush_cache_all_d16i16(void)
 973 {
 974         unsigned long flags;
 975
 976         save_and_cli(flags);
 977         blast_dcache16(); blast_icache16();
 978         restore_flags(flags);
 979 }
 980
 981 static inline void r4k_flush_cache_all_d32i32(void)
 982 {
 983         unsigned long flags;
 984
 985         save_and_cli(flags);
 986         blast_dcache32(); blast_icache32();
 987         restore_flags(flags);
 988 }
 989
 990 static void
 991 r4k_flush_cache_range_s16d16i16(struct mm_struct *mm,
 992                                 unsigned long start,
 993                                 unsigned long end)
 994 {
 995         struct vm_area_struct *vma;
 996         unsigned long flags;
 997
 998         if (mm->context == 0)
 999                 return;
1000
1001         start &= PAGE_MASK;
1002 #ifdef DEBUG_CACHE
1003         printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end);
1004 #endif
1005         vma = find_vma(mm, start);
1006         if (vma) {
1007                 if (mm->context != current->active_mm->context) {
1008                         r4k_flush_cache_all_s16d16i16();
1009                 } else {
1010                         pgd_t *pgd;
1011                         pmd_t *pmd;
1012                         pte_t *pte;
1013                         int text;
1014
1015                         save_and_cli(flags);
1016                         text = vma->vm_flags & VM_EXEC;
1017                         while(start < end) {
1018                                 pgd = pgd_offset(mm, start);
1019                                 pmd = pmd_offset(pgd, start);
1020                                 pte = pte_offset(pmd, start);
1021
1022                                 if(pte_val(*pte) & _PAGE_VALID)
1023                                         blast_scache16_page(start);
1024                                 start += PAGE_SIZE;
1025                         }
1026                         restore_flags(flags);
1027                 }
1028         }
1029 }
1030
1031 static void
1032 r4k_flush_cache_range_s32d16i16(struct mm_struct *mm,
1033                                 unsigned long start,
1034                                 unsigned long end)
1035 {
1036         struct vm_area_struct *vma;
1037         unsigned long flags;
1038
1039         if (mm->context == 0)
1040                 return;
1041
1042         start &= PAGE_MASK;
1043 #ifdef DEBUG_CACHE
1044         printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end);
1045 #endif
1046         vma = find_vma(mm, start);
1047         if (vma) {
1048                 if (mm->context != current->active_mm->context) {
1049                         r4k_flush_cache_all_s32d16i16();
1050                 } else {
1051                         pgd_t *pgd;
1052                         pmd_t *pmd;
1053                         pte_t *pte;
1054                         int text;
1055
1056                         save_and_cli(flags);
1057                         text = vma->vm_flags & VM_EXEC;
1058                         while(start < end) {
1059                                 pgd = pgd_offset(mm, start);
1060                                 pmd = pmd_offset(pgd, start);
1061                                 pte = pte_offset(pmd, start);
1062
1063                                 if(pte_val(*pte) & _PAGE_VALID)
1064                                         blast_scache32_page(start);
1065                                 start += PAGE_SIZE;
1066                         }
1067                         restore_flags(flags);
1068                 }
1069         }
1070 }
1071
1072 static void r4k_flush_cache_range_s64d16i16(struct mm_struct *mm,
1073                                             unsigned long start,
1074                                             unsigned long end)
1075 {
1076         struct vm_area_struct *vma;
1077         unsigned long flags;
1078
1079         if (mm->context == 0)
1080                 return;
1081
1082         start &= PAGE_MASK;
1083 #ifdef DEBUG_CACHE
1084         printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end);
1085 #endif
1086         vma = find_vma(mm, start);
1087         if(vma) {
1088                 if (mm->context != current->active_mm->context) {
1089                         r4k_flush_cache_all_s64d16i16();
1090                 } else {
1091                         pgd_t *pgd;
1092                         pmd_t *pmd;
1093                         pte_t *pte;
1094                         int text;
1095
1096                         save_and_cli(flags);
1097                         text = vma->vm_flags & VM_EXEC;
1098                         while(start < end) {
1099                                 pgd = pgd_offset(mm, start);
1100                                 pmd = pmd_offset(pgd, start);
1101                                 pte = pte_offset(pmd, start);
1102
1103                                 if(pte_val(*pte) & _PAGE_VALID)
1104                                         blast_scache64_page(start);
1105                                 start += PAGE_SIZE;
1106                         }
1107                         restore_flags(flags);
1108                 }
1109         }
1110 }
1111
1112 static void r4k_flush_cache_range_s128d16i16(struct mm_struct *mm,
1113                                              unsigned long start,
1114                                              unsigned long end)
1115 {
1116         struct vm_area_struct *vma;
1117         unsigned long flags;
1118
1119         if (mm->context == 0)
1120                 return;
1121
1122         start &= PAGE_MASK;
1123 #ifdef DEBUG_CACHE
1124         printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end);
1125 #endif
1126         vma = find_vma(mm, start);
1127         if (vma) {
1128                 if (mm->context != current->active_mm->context) {
1129                         r4k_flush_cache_all_s128d16i16();
1130                 } else {
1131                         pgd_t *pgd;
1132                         pmd_t *pmd;
1133                         pte_t *pte;
1134                         int text;
1135
1136                         save_and_cli(flags);
1137                         text = vma->vm_flags & VM_EXEC;
1138                         while(start < end) {
1139                                 pgd = pgd_offset(mm, start);
1140                                 pmd = pmd_offset(pgd, start);
1141                                 pte = pte_offset(pmd, start);
1142
1143                                 if(pte_val(*pte) & _PAGE_VALID)
1144                                         blast_scache128_page(start);
1145                                 start += PAGE_SIZE;
1146                         }
1147                         restore_flags(flags);
1148                 }
1149         }
1150 }
1151
1152 static void r4k_flush_cache_range_s32d32i32(struct mm_struct *mm,
1153                                             unsigned long start,
1154                                             unsigned long end)
1155 {
1156         struct vm_area_struct *vma;
1157         unsigned long flags;
1158
1159         if (mm->context == 0)
1160                 return;
1161
1162         start &= PAGE_MASK;
1163 #ifdef DEBUG_CACHE
1164         printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end);
1165 #endif
1166         vma = find_vma(mm, start);
1167         if (vma) {
1168                 if (mm->context != current->active_mm->context) {
1169                         r4k_flush_cache_all_s32d32i32();
1170                 } else {
1171                         pgd_t *pgd;
1172                         pmd_t *pmd;
1173                         pte_t *pte;
1174                         int text;
1175
1176                         save_and_cli(flags);
1177                         text = vma->vm_flags & VM_EXEC;
1178                         while(start < end) {
1179                                 pgd = pgd_offset(mm, start);
1180                                 pmd = pmd_offset(pgd, start);
1181                                 pte = pte_offset(pmd, start);
1182
1183                                 if(pte_val(*pte) & _PAGE_VALID)
1184                                         blast_scache32_page(start);
1185                                 start += PAGE_SIZE;
1186                         }
1187                         restore_flags(flags);
1188                 }
1189         }
1190 }
1191
1192 static void r4k_flush_cache_range_s64d32i32(struct mm_struct *mm,
1193                                             unsigned long start,
1194                                             unsigned long end)
1195 {
1196         struct vm_area_struct *vma;
1197         unsigned long flags;
1198
1199         if (mm->context == 0)
1200                 return;
1201
1202         start &= PAGE_MASK;
1203 #ifdef DEBUG_CACHE
1204         printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end);
1205 #endif
1206         vma = find_vma(mm, start);
1207         if (vma) {
1208                 if (mm->context != current->active_mm->context) {
1209                         r4k_flush_cache_all_s64d32i32();
1210                 } else {
1211                         pgd_t *pgd;
1212                         pmd_t *pmd;
1213                         pte_t *pte;
1214                         int text;
1215
1216                         save_and_cli(flags);
1217                         text = vma->vm_flags & VM_EXEC;
1218                         while(start < end) {
1219                                 pgd = pgd_offset(mm, start);
1220                                 pmd = pmd_offset(pgd, start);
1221                                 pte = pte_offset(pmd, start);
1222
1223                                 if(pte_val(*pte) & _PAGE_VALID)
1224                                         blast_scache64_page(start);
1225                                 start += PAGE_SIZE;
1226                         }
1227                         restore_flags(flags);
1228                 }
1229         }
1230 }
1231
1232 static void r4k_flush_cache_range_s128d32i32(struct mm_struct *mm,
1233                                              unsigned long start,
1234                                              unsigned long end)
1235 {
1236         struct vm_area_struct *vma;
1237         unsigned long flags;
1238
1239         if (mm->context == 0)
1240                 return;
1241
1242         start &= PAGE_MASK;
1243 #ifdef DEBUG_CACHE
1244         printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end);
1245 #endif
1246         vma = find_vma(mm, start);
1247         if (vma) {
1248                 if (mm->context != current->active_mm->context) {
1249                         r4k_flush_cache_all_s128d32i32();
1250                 } else {
1251                         pgd_t *pgd;
1252                         pmd_t *pmd;
1253                         pte_t *pte;
1254                         int text;
1255
1256                         save_and_cli(flags);
1257                         text = vma->vm_flags & VM_EXEC;
1258                         while(start < end) {
1259                                 pgd = pgd_offset(mm, start);
1260                                 pmd = pmd_offset(pgd, start);
1261                                 pte = pte_offset(pmd, start);
1262
1263                                 if(pte_val(*pte) & _PAGE_VALID)
1264                                         blast_scache128_page(start);
1265                                 start += PAGE_SIZE;
1266                         }
1267                         restore_flags(flags);
1268                 }
1269         }
1270 }
1271
1272 static void r4k_flush_cache_range_d16i16(struct mm_struct *mm,
1273                                          unsigned long start,
1274                                          unsigned long end)
1275 {
1276         if (mm->context != 0) {
1277                 unsigned long flags;
1278
1279 #ifdef DEBUG_CACHE
1280                 printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end);
1281 #endif
1282                 save_and_cli(flags);
1283                 blast_dcache16(); blast_icache16();
1284                 restore_flags(flags);
1285         }
1286 }
1287
1288 static void r4k_flush_cache_range_d32i32(struct mm_struct *mm,
1289                                          unsigned long start,
1290                                          unsigned long end)
1291 {
1292         if (mm->context != 0) {
1293                 unsigned long flags;
1294
1295 #ifdef DEBUG_CACHE
1296                 printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end);
1297 #endif
1298                 save_and_cli(flags);
1299                 blast_dcache32(); blast_icache32();
1300                 restore_flags(flags);
1301         }
1302 }
1303
1304 /*
1305  * On architectures like the Sparc, we could get rid of lines in
1306  * the cache created only by a certain context, but on the MIPS
1307  * (and actually certain Sparc's) we cannot.
1308  */
1309 static void r4k_flush_cache_mm_s16d16i16(struct mm_struct *mm)
1310 {
1311         if (mm->context != 0) {
1312 #ifdef DEBUG_CACHE
1313                 printk("cmm[%d]", (int)mm->context);
1314 #endif
1315                 r4k_flush_cache_all_s16d16i16();
1316         }
1317 }
1318
1319 static void r4k_flush_cache_mm_s32d16i16(struct mm_struct *mm)
1320 {
1321         if (mm->context != 0) {
1322 #ifdef DEBUG_CACHE
1323                 printk("cmm[%d]", (int)mm->context);
1324 #endif
1325                 r4k_flush_cache_all_s32d16i16();
1326         }
1327 }
1328
1329 static void r4k_flush_cache_mm_s64d16i16(struct mm_struct *mm)
1330 {
1331         if (mm->context != 0) {
1332 #ifdef DEBUG_CACHE
1333                 printk("cmm[%d]", (int)mm->context);
1334 #endif
1335                 r4k_flush_cache_all_s64d16i16();
1336         }
1337 }
1338
1339 static void r4k_flush_cache_mm_s128d16i16(struct mm_struct *mm)
1340 {
1341         if (mm->context != 0) {
1342 #ifdef DEBUG_CACHE
1343                 printk("cmm[%d]", (int)mm->context);
1344 #endif
1345                 r4k_flush_cache_all_s128d16i16();
1346         }
1347 }
1348
1349 static void r4k_flush_cache_mm_s32d32i32(struct mm_struct *mm)
1350 {
1351         if (mm->context != 0) {
1352 #ifdef DEBUG_CACHE
1353                 printk("cmm[%d]", (int)mm->context);
1354 #endif
1355                 r4k_flush_cache_all_s32d32i32();
1356         }
1357 }
1358
1359 static void r4k_flush_cache_mm_s64d32i32(struct mm_struct *mm)
1360 {
1361         if (mm->context != 0) {
1362 #ifdef DEBUG_CACHE
1363                 printk("cmm[%d]", (int)mm->context);
1364 #endif
1365                 r4k_flush_cache_all_s64d32i32();
1366         }
1367 }
1368
1369 static void r4k_flush_cache_mm_s128d32i32(struct mm_struct *mm)
1370 {
1371         if (mm->context != 0) {
1372 #ifdef DEBUG_CACHE
1373                 printk("cmm[%d]", (int)mm->context);
1374 #endif
1375                 r4k_flush_cache_all_s128d32i32();
1376         }
1377 }
1378
1379 static void r4k_flush_cache_mm_d16i16(struct mm_struct *mm)
1380 {
1381         if (mm->context != 0) {
1382 #ifdef DEBUG_CACHE
1383                 printk("cmm[%d]", (int)mm->context);
1384 #endif
1385                 r4k_flush_cache_all_d16i16();
1386         }
1387 }
1388
1389 static void r4k_flush_cache_mm_d32i32(struct mm_struct *mm)
1390 {
1391         if (mm->context != 0) {
1392 #ifdef DEBUG_CACHE
1393                 printk("cmm[%d]", (int)mm->context);
1394 #endif
1395                 r4k_flush_cache_all_d32i32();
1396         }
1397 }
1398
1399 static void r4k_flush_cache_page_s16d16i16(struct vm_area_struct *vma,
1400                                            unsigned long page)
1401 {
1402         struct mm_struct *mm = vma->vm_mm;
1403         unsigned long flags;
1404         pgd_t *pgdp;
1405         pmd_t *pmdp;
1406         pte_t *ptep;
1407         int text;
1408
1409         /*
1410          * If ownes no valid ASID yet, cannot possibly have gotten
1411          * this page into the cache.
1412          */
1413         if (mm->context == 0)
1414                 return;
1415
1416 #ifdef DEBUG_CACHE
1417         printk("cpage[%d,%08lx]", (int)mm->context, page);
1418 #endif
1419         save_and_cli(flags);
1420         page &= PAGE_MASK;
1421         pgdp = pgd_offset(mm, page);
1422         pmdp = pmd_offset(pgdp, page);
1423         ptep = pte_offset(pmdp, page);
1424
1425         /*
1426          * If the page isn't marked valid, the page cannot possibly be
1427          * in the cache.
1428          */
1429         if (!(pte_val(*ptep) & _PAGE_VALID))
1430                 goto out;
1431
1432         text = (vma->vm_flags & VM_EXEC);
1433         /* Doing flushes for another ASID than the current one is
1434          * too difficult since stupid R4k caches do a TLB translation
1435          * for every cache flush operation.  So we do indexed flushes
1436          * in that case, which doesn't overly flush the cache too much.
1437          */
1438         if (mm->context != current->active_mm->context) {
1439                 /* Do indexed flush, too much work to get the (possible)
1440                  * tlb refills to work correctly.
1441                  */
1442                 page = (KSEG0 + (page & (scache_size - 1)));
1443                 blast_dcache16_page_indexed(page);
1444                 if(text)
1445                         blast_icache16_page_indexed(page);
1446                 blast_scache16_page_indexed(page);
1447         } else
1448                 blast_scache16_page(page);
1449 out:
1450         restore_flags(flags);
1451 }
1452
1453 static void r4k_flush_cache_page_s32d16i16(struct vm_area_struct *vma,
1454                                            unsigned long page)
1455 {
1456         struct mm_struct *mm = vma->vm_mm;
1457         unsigned long flags;
1458         pgd_t *pgdp;
1459         pmd_t *pmdp;
1460         pte_t *ptep;
1461         int text;
1462
1463         /*
1464          * If ownes no valid ASID yet, cannot possibly have gotten
1465          * this page into the cache.
1466          */
1467         if (mm->context == 0)
1468                 return;
1469
1470 #ifdef DEBUG_CACHE
1471         printk("cpage[%d,%08lx]", (int)mm->context, page);
1472 #endif
1473         save_and_cli(flags);
1474         page &= PAGE_MASK;
1475         pgdp = pgd_offset(mm, page);
1476         pmdp = pmd_offset(pgdp, page);
1477         ptep = pte_offset(pmdp, page);
1478
1479         /* If the page isn't marked valid, the page cannot possibly be
1480          * in the cache.
1481          */
1482         if (!(pte_val(*ptep) & _PAGE_VALID))
1483                 goto out;
1484
1485         text = (vma->vm_flags & VM_EXEC);
1486         /* Doing flushes for another ASID than the current one is
1487          * too difficult since stupid R4k caches do a TLB translation
1488          * for every cache flush operation.  So we do indexed flushes
1489          * in that case, which doesn't overly flush the cache too much.
1490          */
1491         if (mm->context != current->active_mm->context) {
1492                 /* Do indexed flush, too much work to get the (possible)
1493                  * tlb refills to work correctly.
1494                  */
1495                 page = (KSEG0 + (page & (scache_size - 1)));
1496                 blast_dcache16_page_indexed(page);
1497                 if(text)
1498                         blast_icache16_page_indexed(page);
1499                 blast_scache32_page_indexed(page);
1500         } else
1501                 blast_scache32_page(page);
1502 out:
1503         restore_flags(flags);
1504 }
1505
1506 static void r4k_flush_cache_page_s64d16i16(struct vm_area_struct *vma,
1507                                            unsigned long page)
1508 {
1509         struct mm_struct *mm = vma->vm_mm;
1510         unsigned long flags;
1511         pgd_t *pgdp;
1512         pmd_t *pmdp;
1513         pte_t *ptep;
1514         int text;
1515
1516         /*
1517          * If ownes no valid ASID yet, cannot possibly have gotten
1518          * this page into the cache.
1519          */
1520         if (mm->context == 0)
1521                 return;
1522
1523 #ifdef DEBUG_CACHE
1524         printk("cpage[%d,%08lx]", (int)mm->context, page);
1525 #endif
1526         save_and_cli(flags);
1527         page &= PAGE_MASK;
1528         pgdp = pgd_offset(mm, page);
1529         pmdp = pmd_offset(pgdp, page);
1530         ptep = pte_offset(pmdp, page);
1531
1532         /* If the page isn't marked valid, the page cannot possibly be
1533          * in the cache.
1534          */
1535         if (!(pte_val(*ptep) & _PAGE_VALID))
1536                 goto out;
1537
1538         text = (vma->vm_flags & VM_EXEC);
1539         /*
1540          * Doing flushes for another ASID than the current one is
1541          * too difficult since stupid R4k caches do a TLB translation
1542          * for every cache flush operation.  So we do indexed flushes
1543          * in that case, which doesn't overly flush the cache too much.
1544          */
1545         if (mm->context != current->active_mm->context) {
1546                 /* Do indexed flush, too much work to get the (possible)
1547                  * tlb refills to work correctly.
1548                  */
1549                 page = (KSEG0 + (page & (scache_size - 1)));
1550                 blast_dcache16_page_indexed(page);
1551                 if(text)
1552                         blast_icache16_page_indexed(page);
1553                 blast_scache64_page_indexed(page);
1554         } else
1555                 blast_scache64_page(page);
1556 out:
1557         restore_flags(flags);
1558 }
1559
1560 static void r4k_flush_cache_page_s128d16i16(struct vm_area_struct *vma,
1561                                             unsigned long page)
1562 {
1563         struct mm_struct *mm = vma->vm_mm;
1564         unsigned long flags;
1565         pgd_t *pgdp;
1566         pmd_t *pmdp;
1567         pte_t *ptep;
1568         int text;
1569
1570         /*
1571          * If ownes no valid ASID yet, cannot possibly have gotten
1572          * this page into the cache.
1573          */
1574         if (mm->context == 0)
1575                 return;
1576
1577 #ifdef DEBUG_CACHE
1578         printk("cpage[%d,%08lx]", (int)mm->context, page);
1579 #endif
1580         save_and_cli(flags);
1581         page &= PAGE_MASK;
1582         pgdp = pgd_offset(mm, page);
1583         pmdp = pmd_offset(pgdp, page);
1584         ptep = pte_offset(pmdp, page);
1585
1586         /*
1587          * If the page isn't marked valid, the page cannot possibly be
1588          * in the cache.
1589          */
1590         if (!(pte_val(*ptep) & _PAGE_VALID))
1591                 goto out;
1592
1593         text = (vma->vm_flags & VM_EXEC);
1594         /* Doing flushes for another ASID than the current one is
1595          * too difficult since stupid R4k caches do a TLB translation
1596          * for every cache flush operation.  So we do indexed flushes
1597          * in that case, which doesn't overly flush the cache too much.
1598          */
1599         if (mm->context != current->active_mm->context) {
1600                 /*
1601                  * Do indexed flush, too much work to get the (possible)
1602                  * tlb refills to work correctly.
1603                  */
1604                 page = (KSEG0 + (page & (scache_size - 1)));
1605                 blast_dcache16_page_indexed(page);
1606                 if(text)
1607                         blast_icache16_page_indexed(page);
1608                 blast_scache128_page_indexed(page);
1609         } else
1610                 blast_scache128_page(page);
1611 out:
1612         restore_flags(flags);
1613 }
1614
1615 static void r4k_flush_cache_page_s32d32i32(struct vm_area_struct *vma,
1616                                            unsigned long page)
1617 {
1618         struct mm_struct *mm = vma->vm_mm;
1619         unsigned long flags;
1620         pgd_t *pgdp;
1621         pmd_t *pmdp;
1622         pte_t *ptep;
1623         int text;
1624
1625         /*
1626          * If ownes no valid ASID yet, cannot possibly have gotten
1627          * this page into the cache.
1628          */
1629         if (mm->context == 0)
1630                 return;
1631
1632 #ifdef DEBUG_CACHE
1633         printk("cpage[%d,%08lx]", (int)mm->context, page);
1634 #endif
1635         save_and_cli(flags);
1636         page &= PAGE_MASK;
1637         pgdp = pgd_offset(mm, page);
1638         pmdp = pmd_offset(pgdp, page);
1639         ptep = pte_offset(pmdp, page);
1640
1641         /*
1642          * If the page isn't marked valid, the page cannot possibly be
1643          * in the cache.
1644          */
1645         if (!(pte_val(*ptep) & _PAGE_VALID))
1646                 goto out;
1647
1648         text = (vma->vm_flags & VM_EXEC);
1649         /*
1650          * Doing flushes for another ASID than the current one is
1651          * too difficult since stupid R4k caches do a TLB translation
1652          * for every cache flush operation.  So we do indexed flushes
1653          * in that case, which doesn't overly flush the cache too much.
1654          */
1655         if (mm->context != current->active_mm->context) {
1656                 /*
1657                  * Do indexed flush, too much work to get the (possible)
1658                  * tlb refills to work correctly.
1659                  */
1660                 page = (KSEG0 + (page & (scache_size - 1)));
1661                 blast_dcache32_page_indexed(page);
1662                 if(text)
1663                         blast_icache32_page_indexed(page);
1664                 blast_scache32_page_indexed(page);
1665         } else
1666                 blast_scache32_page(page);
1667 out:
1668         restore_flags(flags);
1669 }
1670
1671 static void r4k_flush_cache_page_s64d32i32(struct vm_area_struct *vma,
1672                                            unsigned long page)
1673 {
1674         struct mm_struct *mm = vma->vm_mm;
1675         unsigned long flags;
1676         pgd_t *pgdp;
1677         pmd_t *pmdp;
1678         pte_t *ptep;
1679         int text;
1680
1681         /*
1682          * If ownes no valid ASID yet, cannot possibly have gotten
1683          * this page into the cache.
1684          */
1685         if (mm->context == 0)
1686                 return;
1687
1688 #ifdef DEBUG_CACHE
1689         printk("cpage[%d,%08lx]", (int)mm->context, page);
1690 #endif
1691         save_and_cli(flags);
1692         page &= PAGE_MASK;
1693         pgdp = pgd_offset(mm, page);
1694         pmdp = pmd_offset(pgdp, page);
1695         ptep = pte_offset(pmdp, page);
1696
1697         /*
1698          * If the page isn't marked valid, the page cannot possibly be
1699          * in the cache.
1700          */
1701         if (!(pte_val(*ptep) & _PAGE_VALID))
1702                 goto out;
1703
1704         text = (vma->vm_flags & VM_EXEC);
1705         /*
1706          * Doing flushes for another ASID than the current one is
1707          * too difficult since stupid R4k caches do a TLB translation
1708          * for every cache flush operation.  So we do indexed flushes
1709          * in that case, which doesn't overly flush the cache too much.
1710          */
1711         if (mm->context != current->active_mm->context) {
1712                 /*
1713                  * Do indexed flush, too much work to get the (possible)
1714                  * tlb refills to work correctly.
1715                  */
1716                 page = (KSEG0 + (page & (scache_size - 1)));
1717                 blast_dcache32_page_indexed(page);
1718                 if(text)
1719                         blast_icache32_page_indexed(page);
1720                 blast_scache64_page_indexed(page);
1721         } else
1722                 blast_scache64_page(page);
1723 out:
1724         restore_flags(flags);
1725 }
1726
1727 static void r4k_flush_cache_page_s128d32i32(struct vm_area_struct *vma,
1728                                             unsigned long page)
1729 {
1730         struct mm_struct *mm = vma->vm_mm;
1731         unsigned long flags;
1732         pgd_t *pgdp;
1733         pmd_t *pmdp;
1734         pte_t *ptep;
1735         int text;
1736
1737         /*
1738          * If ownes no valid ASID yet, cannot possibly have gotten
1739          * this page into the cache.
1740          */
1741         if (mm->context == 0)
1742                 return;
1743
1744 #ifdef DEBUG_CACHE
1745         printk("cpage[%d,%08lx]", (int)mm->context, page);
1746 #endif
1747         save_and_cli(flags);
1748         page &= PAGE_MASK;
1749         pgdp = pgd_offset(mm, page);
1750         pmdp = pmd_offset(pgdp, page);
1751         ptep = pte_offset(pmdp, page);
1752
1753         /* If the page isn't marked valid, the page cannot possibly be
1754          * in the cache.
1755          */
1756         if (!(pte_val(*ptep) & _PAGE_VALID))
1757                 goto out;
1758
1759         text = (vma->vm_flags & VM_EXEC);
1760         /*
1761          * Doing flushes for another ASID than the current one is
1762          * too difficult since stupid R4k caches do a TLB translation
1763          * for every cache flush operation.  So we do indexed flushes
1764          * in that case, which doesn't overly flush the cache too much.
1765          */
1766         if (mm->context != current->active_mm->context) {
1767                 /* Do indexed flush, too much work to get the (possible)
1768                  * tlb refills to work correctly.
1769                  */
1770                 page = (KSEG0 + (page & (scache_size - 1)));
1771                 blast_dcache32_page_indexed(page);
1772                 if(text)
1773                         blast_icache32_page_indexed(page);
1774                 blast_scache128_page_indexed(page);
1775         } else
1776                 blast_scache128_page(page);
1777 out:
1778         restore_flags(flags);
1779 }
1780
1781 static void r4k_flush_cache_page_d16i16(struct vm_area_struct *vma,
1782                                         unsigned long page)
1783 {
1784         struct mm_struct *mm = vma->vm_mm;
1785         unsigned long flags;
1786         pgd_t *pgdp;
1787         pmd_t *pmdp;
1788         pte_t *ptep;
1789         int text;
1790
1791         /*
1792          * If ownes no valid ASID yet, cannot possibly have gotten
1793          * this page into the cache.
1794          */
1795         if (mm->context == 0)
1796                 return;
1797
1798 #ifdef DEBUG_CACHE
1799         printk("cpage[%d,%08lx]", (int)mm->context, page);
1800 #endif
1801         save_and_cli(flags);
1802         page &= PAGE_MASK;
1803         pgdp = pgd_offset(mm, page);
1804         pmdp = pmd_offset(pgdp, page);
1805         ptep = pte_offset(pmdp, page);
1806
1807         /* If the page isn't marked valid, the page cannot possibly be
1808          * in the cache.
1809          */
1810         if (!(pte_val(*ptep) & _PAGE_VALID))
1811                 goto out;
1812
1813         text = (vma->vm_flags & VM_EXEC);
1814         /*
1815          * Doing flushes for another ASID than the current one is
1816          * too difficult since stupid R4k caches do a TLB translation
1817          * for every cache flush operation.  So we do indexed flushes
1818          * in that case, which doesn't overly flush the cache too much.
1819          */
1820         if (mm == current->active_mm) {
1821                 blast_dcache16_page(page);
1822                 if(text)
1823                         blast_icache16_page(page);
1824         } else {
1825                 /* Do indexed flush, too much work to get the (possible)
1826                  * tlb refills to work correctly.
1827                  */
1828                 page = (KSEG0 + (page & (dcache_size - 1)));
1829                 blast_dcache16_page_indexed(page);
1830                 if(text)
1831                         blast_icache16_page_indexed(page);
1832         }
1833 out:
1834         restore_flags(flags);
1835 }
1836
1837 static void r4k_flush_cache_page_d32i32(struct vm_area_struct *vma,
1838                                         unsigned long page)
1839 {
1840         struct mm_struct *mm = vma->vm_mm;
1841         unsigned long flags;
1842         pgd_t *pgdp;
1843         pmd_t *pmdp;
1844         pte_t *ptep;
1845         int text;
1846
1847         /*
1848          * If ownes no valid ASID yet, cannot possibly have gotten
1849          * this page into the cache.
1850          */
1851         if (mm->context == 0)
1852                 return;
1853
1854 #ifdef DEBUG_CACHE
1855         printk("cpage[%d,%08lx]", (int)mm->context, page);
1856 #endif
1857         save_and_cli(flags);
1858         page &= PAGE_MASK;
1859         pgdp = pgd_offset(mm, page);
1860         pmdp = pmd_offset(pgdp, page);
1861         ptep = pte_offset(pmdp, page);
1862
1863         /*
1864          * If the page isn't marked valid, the page cannot possibly be
1865          * in the cache.
1866          */
1867         if (!(pte_val(*ptep) & _PAGE_PRESENT))
1868                 goto out;
1869
1870         text = (vma->vm_flags & VM_EXEC);
1871         /*
1872          * Doing flushes for another ASID than the current one is
1873          * too difficult since stupid R4k caches do a TLB translation
1874          * for every cache flush operation.  So we do indexed flushes
1875          * in that case, which doesn't overly flush the cache too much.
1876          */
1877         if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID)) {
1878                 blast_dcache32_page(page);
1879                 if(text)
1880                         blast_icache32_page(page);
1881         } else {
1882                 /*
1883                  * Do indexed flush, too much work to get the (possible)
1884                  * tlb refills to work correctly.
1885                  */
1886                 page = (KSEG0 + (page & (dcache_size - 1)));
1887                 blast_dcache32_page_indexed(page);
1888                 if(text)
1889                         blast_icache32_page_indexed(page);
1890         }
1891 out:
1892         restore_flags(flags);
1893 }
1894
1895 static void r4k_flush_cache_page_d32i32_r4600(struct vm_area_struct *vma,
1896                                               unsigned long page)
1897 {
1898         struct mm_struct *mm = vma->vm_mm;
1899         unsigned long flags;
1900         pgd_t *pgdp;
1901         pmd_t *pmdp;
1902         pte_t *ptep;
1903         int text;
1904
1905         /*
1906          * If ownes no valid ASID yet, cannot possibly have gotten
1907          * this page into the cache.
1908          */
1909         if (mm->context == 0)
1910                 return;
1911
1912 #ifdef DEBUG_CACHE
1913         printk("cpage[%d,%08lx]", (int)mm->context, page);
1914 #endif
1915         save_and_cli(flags);
1916         page &= PAGE_MASK;
1917         pgdp = pgd_offset(mm, page);
1918         pmdp = pmd_offset(pgdp, page);
1919         ptep = pte_offset(pmdp, page);
1920
1921         /*
1922          * If the page isn't marked valid, the page cannot possibly be
1923          * in the cache.
1924          */
1925         if (!(pte_val(*ptep) & _PAGE_PRESENT))
1926                 goto out;
1927
1928         text = (vma->vm_flags & VM_EXEC);
1929         /*
1930          * Doing flushes for another ASID than the current one is
1931          * too difficult since stupid R4k caches do a TLB translation
1932          * for every cache flush operation.  So we do indexed flushes
1933          * in that case, which doesn't overly flush the cache too much.
1934          */
1935         if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID)) {
1936                 blast_dcache32_page(page);
1937                 if(text)
1938                         blast_icache32_page(page);
1939         } else {
1940                 /* Do indexed flush, too much work to get the (possible)
1941                  * tlb refills to work correctly.
1942                  */
1943                 page = (KSEG0 + (page & (dcache_size - 1)));
1944                 blast_dcache32_page_indexed(page);
1945                 blast_dcache32_page_indexed(page ^ dcache_waybit);
1946                 if(text) {
1947                         blast_icache32_page_indexed(page);
1948                         blast_icache32_page_indexed(page ^ icache_waybit);
1949                 }
1950         }
1951 out:
1952         restore_flags(flags);
1953 }
1954
1955 /* If the addresses passed to these routines are valid, they are
1956  * either:
1957  *
1958  * 1) In KSEG0, so we can do a direct flush of the page.
1959  * 2) In KSEG2, and since every process can translate those
1960  *    addresses all the time in kernel mode we can do a direct
1961  *    flush.
1962  * 3) In KSEG1, no flush necessary.
1963  */
1964 static void r4k_flush_page_to_ram_s16d16i16(struct page * page)
1965 {
1966         unsigned long addr = (unsigned long) page_address(page) & PAGE_MASK;
1967
1968         if ((addr >= KSEG0 && addr < KSEG1) || (addr >= KSEG2)) {
1969 #ifdef DEBUG_CACHE
1970                 printk("cram[%08lx]", addr);
1971 #endif
1972                 blast_scache16_page(addr);
1973         }
1974 }
1975
1976 static void r4k_flush_page_to_ram_s32d16i16(struct page * page)
1977 {
1978         unsigned long addr = (unsigned long) page_address(page) & PAGE_MASK;
1979
1980         if ((addr >= KSEG0 && addr < KSEG1) || (addr >= KSEG2)) {
1981 #ifdef DEBUG_CACHE
1982                 printk("cram[%08lx]", addr);
1983 #endif
1984                 blast_scache32_page(addr);
1985         }
1986 }
1987
1988 static void r4k_flush_page_to_ram_s64d16i16(struct page * page)
1989 {
1990         unsigned long addr = (unsigned long) page_address(page) & PAGE_MASK;
1991
1992         if ((addr >= KSEG0 && addr < KSEG1) || (addr >= KSEG2)) {
1993 #ifdef DEBUG_CACHE
1994                 printk("cram[%08lx]", addr);
1995 #endif
1996                 blast_scache64_page(addr);
1997         }
1998 }
1999
2000 static void r4k_flush_page_to_ram_s128d16i16(struct page * page)
2001 {
2002         unsigned long addr = (unsigned long) page_address(page) & PAGE_MASK;
2003
2004         if ((addr >= KSEG0 && addr < KSEG1) || (addr >= KSEG2)) {
2005 #ifdef DEBUG_CACHE
2006                 printk("cram[%08lx]", addr);
2007 #endif
2008                 blast_scache128_page(addr);
2009         }
2010 }
2011
2012 static void r4k_flush_page_to_ram_s32d32i32(struct page * page)
2013 {
2014         unsigned long addr = (unsigned long) page_address(page) & PAGE_MASK;
2015
2016         if ((addr >= KSEG0 && addr < KSEG1) || (addr >= KSEG2)) {
2017 #ifdef DEBUG_CACHE
2018                 printk("cram[%08lx]", addr);
2019 #endif
2020                 blast_scache32_page(addr);
2021         }
2022 }
2023
2024 static void r4k_flush_page_to_ram_s64d32i32(struct page * page)
2025 {
2026         unsigned long addr = (unsigned long) page_address(page) & PAGE_MASK;
2027
2028         if ((addr >= KSEG0 && addr < KSEG1) || (addr >= KSEG2)) {
2029 #ifdef DEBUG_CACHE
2030                 printk("cram[%08lx]", addr);
2031 #endif
2032                 blast_scache64_page(addr);
2033         }
2034 }
2035
2036 static void r4k_flush_page_to_ram_s128d32i32(struct page * page)
2037 {
2038         unsigned long addr = (unsigned long) page_address(page) & PAGE_MASK;
2039
2040         if ((addr >= KSEG0 && addr < KSEG1) || (addr >= KSEG2)) {
2041 #ifdef DEBUG_CACHE
2042                 printk("cram[%08lx]", addr);
2043 #endif
2044                 blast_scache128_page(addr);
2045         }
2046 }
2047
2048 static void r4k_flush_page_to_ram_d16i16(struct page * page)
2049 {
2050         unsigned long addr = (unsigned long) page_address(page) & PAGE_MASK;
2051
2052         if ((addr >= KSEG0 && addr < KSEG1) || (addr >= KSEG2)) {
2053                 unsigned long flags;
2054
2055 #ifdef DEBUG_CACHE
2056                 printk("cram[%08lx]", addr);
2057 #endif
2058                 __save_and_cli(flags);
2059                 blast_dcache16_page(addr);
2060                 __restore_flags(flags);
2061         }
2062 }
2063
2064 static void r4k_flush_page_to_ram_d32i32(struct page * page)
2065 {
2066         unsigned long addr = (unsigned long) page_address(page) & PAGE_MASK;
2067
2068         if ((addr >= KSEG0 && addr < KSEG1) || (addr >= KSEG2)) {
2069                 unsigned long flags;
2070
2071 #ifdef DEBUG_CACHE
2072                 printk("cram[%08lx]", addr);
2073 #endif
2074                 __save_and_cli(flags);
2075                 blast_dcache32_page(addr);
2076                 __restore_flags(flags);
2077         }
2078 }
2079
2080 /*
2081  * Writeback and invalidate the primary cache dcache before DMA.
2082  *
2083  * R4600 v2.0 bug: "The CACHE instructions Hit_Writeback_Inv_D,
2084  * Hit_Writeback_D, Hit_Invalidate_D and Create_Dirty_Exclusive_D will only
2085  * operate correctly if the internal data cache refill buffer is empty.  These
2086  * CACHE instructions should be separated from any potential data cache miss
2087  * by a load instruction to an uncached address to empty the response buffer."
2088  * (Revision 2.0 device errata from IDT available on http://www.idt.com/
2089  * in .pdf format.)
2090  */
2091 static void
2092 r4k_dma_cache_wback_inv_pc(unsigned long addr, unsigned long size)
2093 {
2094         unsigned long end, a;
2095         unsigned int flags;
2096
2097         if (size >= dcache_size) {
2098                 flush_cache_all();
2099         } else {
2100                 /* Workaround for R4600 bug.  See comment above. */
2101                 save_and_cli(flags);
2102                 *(volatile unsigned long *)KSEG1;
2103
2104                 a = addr & ~(dc_lsize - 1);
2105                 end = (addr + size) & ~(dc_lsize - 1);
2106                 while (1) {
2107                         flush_dcache_line(a); /* Hit_Writeback_Inv_D */
2108                         if (a == end) break;
2109                         a += dc_lsize;
2110                 }
2111                 restore_flags(flags);
2112         }
2113         bcops->bc_wback_inv(addr, size);
2114 }
2115
2116 static void
2117 r4k_dma_cache_wback_inv_sc(unsigned long addr, unsigned long size)
2118 {
2119         unsigned long end, a;
2120
2121         if (size >= scache_size) {
2122                 flush_cache_all();
2123                 return;
2124         }
2125
2126         a = addr & ~(sc_lsize - 1);
2127         end = (addr + size) & ~(sc_lsize - 1);
2128         while (1) {
2129                 flush_scache_line(a);   /* Hit_Writeback_Inv_SD */
2130                 if (a == end) break;
2131                 a += sc_lsize;
2132         }
2133 }
2134
2135 static void
2136 r4k_dma_cache_inv_pc(unsigned long addr, unsigned long size)
2137 {
2138         unsigned long end, a;
2139         unsigned int flags;
2140
2141         if (size >= dcache_size) {
2142                 flush_cache_all();
2143         } else {
2144                 /* Workaround for R4600 bug.  See comment above. */
2145                 save_and_cli(flags);
2146                 *(volatile unsigned long *)KSEG1;
2147
2148                 a = addr & ~(dc_lsize - 1);
2149                 end = (addr + size) & ~(dc_lsize - 1);
2150                 while (1) {
2151                         flush_dcache_line(a); /* Hit_Writeback_Inv_D */
2152                         if (a == end) break;
2153                         a += dc_lsize;
2154                 }
2155                 restore_flags(flags);
2156         }
2157
2158         bcops->bc_inv(addr, size);
2159 }
2160
2161 static void
2162 r4k_dma_cache_inv_sc(unsigned long addr, unsigned long size)
2163 {
2164         unsigned long end, a;
2165
2166         if (size >= scache_size) {
2167                 flush_cache_all();
2168                 return;
2169         }
2170
2171         a = addr & ~(sc_lsize - 1);
2172         end = (addr + size) & ~(sc_lsize - 1);
2173         while (1) {
2174                 flush_scache_line(a); /* Hit_Writeback_Inv_SD */
2175                 if (a == end) break;
2176                 a += sc_lsize;
2177         }
2178 }
2179
2180 static void
2181 r4k_dma_cache_wback(unsigned long addr, unsigned long size)
2182 {
2183         panic("r4k_dma_cache called - should not happen.\n");
2184 }
2185
2186 /*
2187  * While we're protected against bad userland addresses we don't care
2188  * very much about what happens in that case.  Usually a segmentation
2189  * fault will dump the process later on anyway ...
2190  */
2191 static void r4k_flush_cache_sigtramp(unsigned long addr)
2192 {
2193         unsigned long daddr, iaddr;
2194
2195         daddr = addr & ~(dc_lsize - 1);
2196         __asm__ __volatile__("nop;nop;nop;nop");        /* R4600 V1.7 */
2197         protected_writeback_dcache_line(daddr);
2198         protected_writeback_dcache_line(daddr + dc_lsize);
2199         iaddr = addr & ~(ic_lsize - 1);
2200         protected_flush_icache_line(iaddr);
2201         protected_flush_icache_line(iaddr + ic_lsize);
2202 }
2203
2204 static void r4600v20k_flush_cache_sigtramp(unsigned long addr)
2205 {
2206         unsigned long daddr, iaddr;
2207         unsigned int flags;
2208
2209         daddr = addr & ~(dc_lsize - 1);
2210         __save_and_cli(flags);
2211
2212         /* Clear internal cache refill buffer */
2213         *(volatile unsigned int *)KSEG1;
2214
2215         protected_writeback_dcache_line(daddr);
2216         protected_writeback_dcache_line(daddr + dc_lsize);
2217         iaddr = addr & ~(ic_lsize - 1);
2218         protected_flush_icache_line(iaddr);
2219         protected_flush_icache_line(iaddr + ic_lsize);
2220         __restore_flags(flags);
2221 }
2222
2223 #undef DEBUG_TLB
2224 #undef DEBUG_TLBUPDATE
2225
2226 #define NTLB_ENTRIES       48  /* Fixed on all R4XX0 variants... */
2227
2228 #define NTLB_ENTRIES_HALF  24  /* Fixed on all R4XX0 variants... */
2229
2230 void flush_tlb_all(void)
2231 {
2232         unsigned long flags;
2233         unsigned long old_ctx;
2234         int entry;
2235
2236 #ifdef DEBUG_TLB
2237         printk("[tlball]");
2238 #endif
2239
2240         save_and_cli(flags);
2241         /* Save old context and create impossible VPN2 value */
2242         old_ctx = (get_entryhi() & 0xff);
2243         set_entryhi(KSEG0);
2244         set_entrylo0(0);
2245         set_entrylo1(0);
2246         BARRIER;
2247
2248         entry = get_wired();
2249
2250         /* Blast 'em all away. */
2251         while(entry < NTLB_ENTRIES) {
2252                 set_index(entry);
2253                 BARRIER;
2254                 tlb_write_indexed();
2255                 BARRIER;
2256                 entry++;
2257         }
2258         BARRIER;
2259         set_entryhi(old_ctx);
2260         restore_flags(flags);
2261 }
2262
2263 void flush_tlb_mm(struct mm_struct *mm)
2264 {
2265         if (mm->context != 0) {
2266                 unsigned long flags;
2267
2268 #ifdef DEBUG_TLB
2269                 printk("[tlbmm<%d>]", mm->context);
2270 #endif
2271                 save_and_cli(flags);
2272                 get_new_mmu_context(mm, asid_cache);
2273                 if (mm == current->active_mm)
2274                         set_entryhi(mm->context & 0xff);
2275                 restore_flags(flags);
2276         }
2277 }
2278
2279 void flush_tlb_range(struct mm_struct *mm, unsigned long start,
2280                                 unsigned long end)
2281 {
2282         if(mm->context != 0) {
2283                 unsigned long flags;
2284                 int size;
2285
2286 #ifdef DEBUG_TLB
2287                 printk("[tlbrange<%02x,%08lx,%08lx>]", (mm->context & 0xff),
2288                        start, end);
2289 #endif
2290                 save_and_cli(flags);
2291                 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
2292                 size = (size + 1) >> 1;
2293                 if(size <= NTLB_ENTRIES_HALF) {
2294                         int oldpid = (get_entryhi() & 0xff);
2295                         int newpid = (mm->context & 0xff);
2296
2297                         start &= (PAGE_MASK << 1);
2298                         end += ((PAGE_SIZE << 1) - 1);
2299                         end &= (PAGE_MASK << 1);
2300                         while(start < end) {
2301                                 int idx;
2302
2303                                 set_entryhi(start | newpid);
2304                                 start += (PAGE_SIZE << 1);
2305                                 BARRIER;
2306                                 tlb_probe();
2307                                 BARRIER;
2308                                 idx = get_index();
2309                                 set_entrylo0(0);
2310                                 set_entrylo1(0);
2311                                 set_entryhi(KSEG0);
2312                                 BARRIER;
2313                                 if(idx < 0)
2314                                         continue;
2315                                 tlb_write_indexed();
2316                                 BARRIER;
2317                         }
2318                         set_entryhi(oldpid);
2319                 } else {
2320                         get_new_mmu_context(mm, asid_cache);
2321                         if (mm == current->active_mm)
2322                                 set_entryhi(mm->context & 0xff);
2323                 }
2324                 restore_flags(flags);
2325         }
2326 }
2327
2328 void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
2329 {
2330         if (vma->vm_mm->context != 0) {
2331                 unsigned long flags;
2332                 int oldpid, newpid, idx;
2333
2334 #ifdef DEBUG_TLB
2335                 printk("[tlbpage<%d,%08lx>]", vma->vm_mm->context, page);
2336 #endif
2337                 newpid = (vma->vm_mm->context & 0xff);
2338                 page &= (PAGE_MASK << 1);
2339                 save_and_cli(flags);
2340                 oldpid = (get_entryhi() & 0xff);
2341                 set_entryhi(page | newpid);
2342                 BARRIER;
2343                 tlb_probe();
2344                 BARRIER;
2345                 idx = get_index();
2346                 set_entrylo0(0);
2347                 set_entrylo1(0);
2348                 set_entryhi(KSEG0);
2349                 if(idx < 0)
2350                         goto finish;
2351                 BARRIER;
2352                 tlb_write_indexed();
2353
2354         finish:
2355                 BARRIER;
2356                 set_entryhi(oldpid);
2357                 restore_flags(flags);
2358         }
2359 }
2360
2361 /* Load a new root pointer into the TLB. */
2362 void load_pgd(unsigned long pg_dir)
2363 {
2364 }
2365
2366 void pgd_init(unsigned long page)
2367 {
2368         unsigned long *p = (unsigned long *) page;
2369         int i;
2370
2371         for(i = 0; i < USER_PTRS_PER_PGD; i+=8) {
2372                 p[i + 0] = (unsigned long) invalid_pte_table;
2373                 p[i + 1] = (unsigned long) invalid_pte_table;
2374                 p[i + 2] = (unsigned long) invalid_pte_table;
2375                 p[i + 3] = (unsigned long) invalid_pte_table;
2376                 p[i + 4] = (unsigned long) invalid_pte_table;
2377                 p[i + 5] = (unsigned long) invalid_pte_table;
2378                 p[i + 6] = (unsigned long) invalid_pte_table;
2379                 p[i + 7] = (unsigned long) invalid_pte_table;
2380         }
2381 }
2382
2383 #ifdef DEBUG_TLBUPDATE
2384 static unsigned long ehi_debug[NTLB_ENTRIES];
2385 static unsigned long el0_debug[NTLB_ENTRIES];
2386 static unsigned long el1_debug[NTLB_ENTRIES];
2387 #endif
2388
2389 /* We will need multiple versions of update_mmu_cache(), one that just
2390  * updates the TLB with the new pte(s), and another which also checks
2391  * for the R4k "end of page" hardware bug and does the needy.
2392  */
2393 void update_mmu_cache(struct vm_area_struct * vma,
2394                                  unsigned long address, pte_t pte)
2395 {
2396         unsigned long flags;
2397         pgd_t *pgdp;
2398         pmd_t *pmdp;
2399         pte_t *ptep;
2400         int idx, pid;
2401
2402         /*
2403          * Handle debugger faulting in for debugee.
2404          */
2405         if (current->active_mm != vma->vm_mm)
2406                 return;
2407
2408         pid = get_entryhi() & 0xff;
2409
2410 #ifdef DEBUG_TLB
2411         if((pid != (vma->vm_mm->context & 0xff)) || (vma->vm_mm->context == 0)) {
2412                 printk("update_mmu_cache: Wheee, bogus tlbpid mmpid=%d tlbpid=%d\n",
2413                        (int) (vma->vm_mm->context & 0xff), pid);
2414         }
2415 #endif
2416
2417         save_and_cli(flags);
2418         address &= (PAGE_MASK << 1);
2419         set_entryhi(address | (pid));
2420         pgdp = pgd_offset(vma->vm_mm, address);
2421         BARRIER;
2422         tlb_probe();
2423         BARRIER;
2424         pmdp = pmd_offset(pgdp, address);
2425         idx = get_index();
2426         ptep = pte_offset(pmdp, address);
2427         BARRIER;
2428         set_entrylo0(pte_val(*ptep++) >> 6);
2429         set_entrylo1(pte_val(*ptep) >> 6);
2430         set_entryhi(address | (pid));
2431         BARRIER;
2432         if(idx < 0) {
2433                 tlb_write_random();
2434         } else {
2435                 tlb_write_indexed();
2436         }
2437         BARRIER;
2438         set_entryhi(pid);
2439         BARRIER;
2440         restore_flags(flags);
2441 }
2442
2443 #if 0
2444 static void r4k_update_mmu_cache_hwbug(struct vm_area_struct * vma,
2445                                        unsigned long address, pte_t pte)
2446 {
2447         unsigned long flags;
2448         pgd_t *pgdp;
2449         pmd_t *pmdp;
2450         pte_t *ptep;
2451         int idx;
2452
2453         save_and_cli(flags);
2454         address &= (PAGE_MASK << 1);
2455         set_entryhi(address | (get_entryhi() & 0xff));
2456         pgdp = pgd_offset(vma->vm_mm, address);
2457         tlb_probe();
2458         pmdp = pmd_offset(pgdp, address);
2459         idx = get_index();
2460         ptep = pte_offset(pmdp, address);
2461         set_entrylo0(pte_val(*ptep++) >> 6);
2462         set_entrylo1(pte_val(*ptep) >> 6);
2463         BARRIER;
2464         if(idx < 0)
2465                 tlb_write_random();
2466         else
2467                 tlb_write_indexed();
2468         BARRIER;
2469         restore_flags(flags);
2470 }
2471 #endif
2472
2473 void show_regs(struct pt_regs * regs)
2474 {
2475         /* Saved main processor registers. */
2476         printk("$0 : %08lx %08lx %08lx %08lx\n",
2477                0UL, regs->regs[1], regs->regs[2], regs->regs[3]);
2478         printk("$4 : %08lx %08lx %08lx %08lx\n",
2479                regs->regs[4], regs->regs[5], regs->regs[6], regs->regs[7]);
2480         printk("$8 : %08lx %08lx %08lx %08lx\n",
2481                regs->regs[8], regs->regs[9], regs->regs[10], regs->regs[11]);
2482         printk("$12: %08lx %08lx %08lx %08lx\n",
2483                regs->regs[12], regs->regs[13], regs->regs[14], regs->regs[15]);
2484         printk("$16: %08lx %08lx %08lx %08lx\n",
2485                regs->regs[16], regs->regs[17], regs->regs[18], regs->regs[19]);
2486         printk("$20: %08lx %08lx %08lx %08lx\n",
2487                regs->regs[20], regs->regs[21], regs->regs[22], regs->regs[23]);
2488         printk("$24: %08lx %08lx\n",
2489                regs->regs[24], regs->regs[25]);
2490         printk("$28: %08lx %08lx %08lx %08lx\n",
2491                regs->regs[28], regs->regs[29], regs->regs[30], regs->regs[31]);
2492
2493         /* Saved cp0 registers. */
2494         printk("epc   : %08lx\nStatus: %08lx\nCause : %08lx\n",
2495                regs->cp0_epc, regs->cp0_status, regs->cp0_cause);
2496 }
2497
2498 void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
2499                                       unsigned long entryhi, unsigned long pagemask)
2500 {
2501         unsigned long flags;
2502         unsigned long wired;
2503         unsigned long old_pagemask;
2504         unsigned long old_ctx;
2505
2506         save_and_cli(flags);
2507         /* Save old context and create impossible VPN2 value */
2508         old_ctx = (get_entryhi() & 0xff);
2509         old_pagemask = get_pagemask();
2510         wired = get_wired();
2511         set_wired (wired + 1);
2512         set_index (wired);
2513         BARRIER;
2514         set_pagemask (pagemask);
2515         set_entryhi(entryhi);
2516         set_entrylo0(entrylo0);
2517         set_entrylo1(entrylo1);
2518         BARRIER;
2519         tlb_write_indexed();
2520         BARRIER;
2521
2522         set_entryhi(old_ctx);
2523         BARRIER;
2524         set_pagemask (old_pagemask);
2525         flush_tlb_all();
2526         restore_flags(flags);
2527 }
2528
2529 /* Detect and size the various r4k caches. */
2530 static void __init probe_icache(unsigned long config)
2531 {
2532         icache_size = 1 << (12 + ((config >> 9) & 7));
2533         ic_lsize = 16 << ((config >> 5) & 1);
2534
2535         printk("Primary instruction cache %dkb, linesize %d bytes)\n",
2536                icache_size >> 10, ic_lsize);
2537 }
2538
2539 static void __init probe_dcache(unsigned long config)
2540 {
2541         dcache_size = 1 << (12 + ((config >> 6) & 7));
2542         dc_lsize = 16 << ((config >> 4) & 1);
2543
2544         printk("Primary data cache %dkb, linesize %d bytes)\n",
2545                dcache_size >> 10, dc_lsize);
2546 }
2547
2548
2549 /* If you even _breathe_ on this function, look at the gcc output
2550  * and make sure it does not pop things on and off the stack for
2551  * the cache sizing loop that executes in KSEG1 space or else
2552  * you will crash and burn badly.  You have been warned.
2553  */
2554 static int __init probe_scache(unsigned long config)
2555 {
2556         extern unsigned long stext;
2557         unsigned long flags, addr, begin, end, pow2;
2558         int tmp;
2559
2560         tmp = ((config >> 17) & 1);
2561         if(tmp)
2562                 return 0;
2563         tmp = ((config >> 22) & 3);
2564         switch(tmp) {
2565         case 0:
2566                 sc_lsize = 16;
2567                 break;
2568         case 1:
2569                 sc_lsize = 32;
2570                 break;
2571         case 2:
2572                 sc_lsize = 64;
2573                 break;
2574         case 3:
2575                 sc_lsize = 128;
2576                 break;
2577         }
2578
2579         begin = (unsigned long) &stext;
2580         begin &= ~((4 * 1024 * 1024) - 1);
2581         end = begin + (4 * 1024 * 1024);
2582
2583         /* This is such a bitch, you'd think they would make it
2584          * easy to do this.  Away you daemons of stupidity!
2585          */
2586         save_and_cli(flags);
2587
2588         /* Fill each size-multiple cache line with a valid tag. */
2589         pow2 = (64 * 1024);
2590         for(addr = begin; addr < end; addr = (begin + pow2)) {
2591                 unsigned long *p = (unsigned long *) addr;
2592                 __asm__ __volatile__("nop" : : "r" (*p)); /* whee... */
2593                 pow2 <<= 1;
2594         }
2595
2596         /* Load first line with zero (therefore invalid) tag. */
2597         set_taglo(0);
2598         set_taghi(0);
2599         __asm__ __volatile__("nop; nop; nop; nop;"); /* avoid the hazard */
2600         __asm__ __volatile__("\n\t.set noreorder\n\t"
2601                              ".set mips3\n\t"
2602                              "cache 8, (%0)\n\t"
2603                              ".set mips0\n\t"
2604                              ".set reorder\n\t" : : "r" (begin));
2605         __asm__ __volatile__("\n\t.set noreorder\n\t"
2606                              ".set mips3\n\t"
2607                              "cache 9, (%0)\n\t"
2608                              ".set mips0\n\t"
2609                              ".set reorder\n\t" : : "r" (begin));
2610         __asm__ __volatile__("\n\t.set noreorder\n\t"
2611                              ".set mips3\n\t"
2612                              "cache 11, (%0)\n\t"
2613                              ".set mips0\n\t"
2614                              ".set reorder\n\t" : : "r" (begin));
2615
2616         /* Now search for the wrap around point. */
2617         pow2 = (128 * 1024);
2618         tmp = 0;
2619         for(addr = (begin + (128 * 1024)); addr < (end); addr = (begin + pow2)) {
2620                 __asm__ __volatile__("\n\t.set noreorder\n\t"
2621                                      ".set mips3\n\t"
2622                                      "cache 7, (%0)\n\t"
2623                                      ".set mips0\n\t"
2624                                      ".set reorder\n\t" : : "r" (addr));
2625                 __asm__ __volatile__("nop; nop; nop; nop;"); /* hazard... */
2626                 if(!get_taglo())
2627                         break;
2628                 pow2 <<= 1;
2629         }
2630         restore_flags(flags);
2631         addr -= begin;
2632         printk("Secondary cache sized at %dK linesize %d\n",
2633                (int) (addr >> 10), sc_lsize);
2634         scache_size = addr;
2635         return 1;
2636 }
2637
2638 static void __init setup_noscache_funcs(void)
2639 {
2640         unsigned int prid;
2641
2642         switch(dc_lsize) {
2643         case 16:
2644                 _clear_page = r4k_clear_page_d16;
2645                 _copy_page = r4k_copy_page_d16;
2646                 _flush_cache_all = r4k_flush_cache_all_d16i16;
2647                 _flush_cache_mm = r4k_flush_cache_mm_d16i16;
2648                 _flush_cache_range = r4k_flush_cache_range_d16i16;
2649                 _flush_cache_page = r4k_flush_cache_page_d16i16;
2650                 _flush_page_to_ram = r4k_flush_page_to_ram_d16i16;
2651                 break;
2652         case 32:
2653                 prid = read_32bit_cp0_register(CP0_PRID) & 0xfff0;
2654                 if (prid == 0x2010) {                   /* R4600 V1.7 */
2655                         _clear_page = r4k_clear_page_r4600_v1;
2656                         _copy_page = r4k_copy_page_r4600_v1;
2657                 } else if (prid == 0x2020) {            /* R4600 V2.0 */
2658                         _clear_page = r4k_clear_page_r4600_v2;
2659                         _copy_page = r4k_copy_page_r4600_v2;
2660                 } else {
2661                         _clear_page = r4k_clear_page_d32;
2662                         _copy_page = r4k_copy_page_d32;
2663                 }
2664                 _flush_cache_all = r4k_flush_cache_all_d32i32;
2665                 _flush_cache_mm = r4k_flush_cache_mm_d32i32;
2666                 _flush_cache_range = r4k_flush_cache_range_d32i32;
2667                 _flush_cache_page = r4k_flush_cache_page_d32i32;
2668                 _flush_page_to_ram = r4k_flush_page_to_ram_d32i32;
2669                 break;
2670         }
2671         _dma_cache_wback_inv = r4k_dma_cache_wback_inv_pc;
2672         _dma_cache_wback = r4k_dma_cache_wback;
2673         _dma_cache_inv = r4k_dma_cache_inv_pc;
2674 }
2675
2676 static void __init setup_scache_funcs(void)
2677 {
2678         switch(sc_lsize) {
2679         case 16:
2680                 switch(dc_lsize) {
2681                 case 16:
2682                         _flush_cache_all = r4k_flush_cache_all_s16d16i16;
2683                         _flush_cache_mm = r4k_flush_cache_mm_s16d16i16;
2684                         _flush_cache_range = r4k_flush_cache_range_s16d16i16;
2685                         _flush_cache_page = r4k_flush_cache_page_s16d16i16;
2686                         _flush_page_to_ram = r4k_flush_page_to_ram_s16d16i16;
2687                         break;
2688                 case 32:
2689                         panic("Invalid cache configuration detected");
2690                 };
2691                 _clear_page = r4k_clear_page_s16;
2692                 _copy_page = r4k_copy_page_s16;
2693                 break;
2694         case 32:
2695                 switch(dc_lsize) {
2696                 case 16:
2697                         _flush_cache_all = r4k_flush_cache_all_s32d16i16;
2698                         _flush_cache_mm = r4k_flush_cache_mm_s32d16i16;
2699                         _flush_cache_range = r4k_flush_cache_range_s32d16i16;
2700                         _flush_cache_page = r4k_flush_cache_page_s32d16i16;
2701                         _flush_page_to_ram = r4k_flush_page_to_ram_s32d16i16;
2702                         break;
2703                 case 32:
2704                         _flush_cache_all = r4k_flush_cache_all_s32d32i32;
2705                         _flush_cache_mm = r4k_flush_cache_mm_s32d32i32;
2706                         _flush_cache_range = r4k_flush_cache_range_s32d32i32;
2707                         _flush_cache_page = r4k_flush_cache_page_s32d32i32;
2708                         _flush_page_to_ram = r4k_flush_page_to_ram_s32d32i32;
2709                         break;
2710                 };
2711                 _clear_page = r4k_clear_page_s32;
2712                 _copy_page = r4k_copy_page_s32;
2713                 break;
2714         case 64:
2715                 switch(dc_lsize) {
2716                 case 16:
2717                         _flush_cache_all = r4k_flush_cache_all_s64d16i16;
2718                         _flush_cache_mm = r4k_flush_cache_mm_s64d16i16;
2719                         _flush_cache_range = r4k_flush_cache_range_s64d16i16;
2720                         _flush_cache_page = r4k_flush_cache_page_s64d16i16;
2721                         _flush_page_to_ram = r4k_flush_page_to_ram_s64d16i16;
2722                         break;
2723                 case 32:
2724                         _flush_cache_all = r4k_flush_cache_all_s64d32i32;
2725                         _flush_cache_mm = r4k_flush_cache_mm_s64d32i32;
2726                         _flush_cache_range = r4k_flush_cache_range_s64d32i32;
2727                         _flush_cache_page = r4k_flush_cache_page_s64d32i32;
2728                         _flush_page_to_ram = r4k_flush_page_to_ram_s64d32i32;
2729                         break;
2730                 };
2731                 _clear_page = r4k_clear_page_s64;
2732                 _copy_page = r4k_copy_page_s64;
2733                 break;
2734         case 128:
2735                 switch(dc_lsize) {
2736                 case 16:
2737                         _flush_cache_all = r4k_flush_cache_all_s128d16i16;
2738                         _flush_cache_mm = r4k_flush_cache_mm_s128d16i16;
2739                         _flush_cache_range = r4k_flush_cache_range_s128d16i16;
2740                         _flush_cache_page = r4k_flush_cache_page_s128d16i16;
2741                         _flush_page_to_ram = r4k_flush_page_to_ram_s128d16i16;
2742                         break;
2743                 case 32:
2744                         _flush_cache_all = r4k_flush_cache_all_s128d32i32;
2745                         _flush_cache_mm = r4k_flush_cache_mm_s128d32i32;
2746                         _flush_cache_range = r4k_flush_cache_range_s128d32i32;
2747                         _flush_cache_page = r4k_flush_cache_page_s128d32i32;
2748                         _flush_page_to_ram = r4k_flush_page_to_ram_s128d32i32;
2749                         break;
2750                 };
2751                 _clear_page = r4k_clear_page_s128;
2752                 _copy_page = r4k_copy_page_s128;
2753                 break;
2754         }
2755         _dma_cache_wback_inv = r4k_dma_cache_wback_inv_sc;
2756         _dma_cache_wback = r4k_dma_cache_wback;
2757         _dma_cache_inv = r4k_dma_cache_inv_sc;
2758 }
2759
2760 typedef int (*probe_func_t)(unsigned long);
2761
2762 static inline void __init setup_scache(unsigned int config)
2763 {
2764         probe_func_t probe_scache_kseg1;
2765         int sc_present = 0;
2766
2767         /* Maybe the cpu knows about a l2 cache? */
2768         probe_scache_kseg1 = (probe_func_t) (KSEG1ADDR(&probe_scache));
2769         sc_present = probe_scache_kseg1(config);
2770
2771         if (sc_present) {
2772                 setup_scache_funcs();
2773                 return;
2774         }
2775
2776         setup_noscache_funcs();
2777 }
2778
2779 void __init ld_mmu_r4xx0(void)
2780 {
2781         unsigned long config = read_32bit_cp0_register(CP0_CONFIG);
2782
2783         printk("CPU revision is: %08x\n", read_32bit_cp0_register(CP0_PRID));
2784
2785         set_cp0_config(CONF_CM_CMASK, CONF_CM_CACHABLE_NONCOHERENT);
2786
2787         probe_icache(config);
2788         probe_dcache(config);
2789         setup_scache(config);
2790
2791         switch(mips_cputype) {
2792         case CPU_R4600:                 /* QED style two way caches? */
2793         case CPU_R4700:
2794         case CPU_R5000:
2795         case CPU_NEVADA:
2796                 _flush_cache_page = r4k_flush_cache_page_d32i32_r4600;
2797         }
2798
2799         _flush_cache_sigtramp = r4k_flush_cache_sigtramp;
2800         if ((read_32bit_cp0_register(CP0_PRID) & 0xfff0) == 0x2020) {
2801                 _flush_cache_sigtramp = r4600v20k_flush_cache_sigtramp;
2802         }
2803
2804         flush_cache_all();
2805         write_32bit_cp0_register(CP0_WIRED, 0);
2806
2807         /*
2808          * You should never change this register:
2809          *   - On R4600 1.7 the tlbp never hits for pages smaller than
2810          *     the value in the c0_pagemask register.
2811          *   - The entire mm handling assumes the c0_pagemask register to
2812          *     be set for 4kb pages.
2813          */
2814         write_32bit_cp0_register(CP0_PAGEMASK, PM_4K);
2815         flush_tlb_all();
2816 }