arch/mips/cavium-octeon/executive/cvmx-l2c.c

   1 /***********************license start***************
   2  * Author: Cavium Networks
   3  *
   4  * Contact: support@caviumnetworks.com
   5  * This file is part of the OCTEON SDK
   6  *
   7  * Copyright (c) 2003-2017 Cavium, Inc.
   8  *
   9  * This file is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License, Version 2, as
  11  * published by the Free Software Foundation.
  12  *
  13  * This file is distributed in the hope that it will be useful, but
  14  * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
  15  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
  16  * NONINFRINGEMENT.  See the GNU General Public License for more
  17  * details.
  18  *
  19  * You should have received a copy of the GNU General Public License
  20  * along with this file; if not, write to the Free Software
  21  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  22  * or visit http://www.gnu.org/licenses/.
  23  *
  24  * This file may also be available under a different license from Cavium.
  25  * Contact Cavium Networks for more information
  26  ***********************license end**************************************/
  27
  28 /*
  29  * Implementation of the Level 2 Cache (L2C) control,
  30  * measurement, and debugging facilities.
  31  */
  32
  33 #include <linux/compiler.h>
  34 #include <linux/irqflags.h>
  35 #include <asm/octeon/cvmx.h>
  36 #include <asm/octeon/cvmx-l2c.h>
  37 #include <asm/octeon/cvmx-spinlock.h>
  38
  39 /*
  40  * This spinlock is used internally to ensure that only one core is
  41  * performing certain L2 operations at a time.
  42  *
  43  * NOTE: This only protects calls from within a single application -
  44  * if multiple applications or operating systems are running, then it
  45  * is up to the user program to coordinate between them.
  46  */
  47 static cvmx_spinlock_t cvmx_l2c_spinlock;
  48
  49 int cvmx_l2c_get_core_way_partition(uint32_t core)
  50 {
  51         uint32_t field;
  52
  53         /* Validate the core number */
  54         if (core >= cvmx_octeon_num_cores())
  55                 return -1;
  56
  57         if (OCTEON_IS_MODEL(OCTEON_CN63XX))
  58                 return cvmx_read_csr(CVMX_L2C_WPAR_PPX(core)) & 0xffff;
  59
  60         /*
  61          * Use the lower two bits of the coreNumber to determine the
  62          * bit offset of the UMSK[] field in the L2C_SPAR register.
  63          */
  64         field = (core & 0x3) * 8;
  65
  66         /*
  67          * Return the UMSK[] field from the appropriate L2C_SPAR
  68          * register based on the coreNumber.
  69          */
  70
  71         switch (core & 0xC) {
  72         case 0x0:
  73                 return (cvmx_read_csr(CVMX_L2C_SPAR0) & (0xFF << field)) >> field;
  74         case 0x4:
  75                 return (cvmx_read_csr(CVMX_L2C_SPAR1) & (0xFF << field)) >> field;
  76         case 0x8:
  77                 return (cvmx_read_csr(CVMX_L2C_SPAR2) & (0xFF << field)) >> field;
  78         case 0xC:
  79                 return (cvmx_read_csr(CVMX_L2C_SPAR3) & (0xFF << field)) >> field;
  80         }
  81         return 0;
  82 }
  83
  84 int cvmx_l2c_set_core_way_partition(uint32_t core, uint32_t mask)
  85 {
  86         uint32_t field;
  87         uint32_t valid_mask;
  88
  89         valid_mask = (0x1 << cvmx_l2c_get_num_assoc()) - 1;
  90
  91         mask &= valid_mask;
  92
  93         /* A UMSK setting which blocks all L2C Ways is an error on some chips */
  94         if (mask == valid_mask && !OCTEON_IS_MODEL(OCTEON_CN63XX))
  95                 return -1;
  96
  97         /* Validate the core number */
  98         if (core >= cvmx_octeon_num_cores())
  99                 return -1;
 100
 101         if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
 102                 cvmx_write_csr(CVMX_L2C_WPAR_PPX(core), mask);
 103                 return 0;
 104         }
 105
 106         /*
 107          * Use the lower two bits of core to determine the bit offset of the
 108          * UMSK[] field in the L2C_SPAR register.
 109          */
 110         field = (core & 0x3) * 8;
 111
 112         /*
 113          * Assign the new mask setting to the UMSK[] field in the appropriate
 114          * L2C_SPAR register based on the core_num.
 115          *
 116          */
 117         switch (core & 0xC) {
 118         case 0x0:
 119                 cvmx_write_csr(CVMX_L2C_SPAR0,
 120                                (cvmx_read_csr(CVMX_L2C_SPAR0) & ~(0xFF << field)) |
 121                                mask << field);
 122                 break;
 123         case 0x4:
 124                 cvmx_write_csr(CVMX_L2C_SPAR1,
 125                                (cvmx_read_csr(CVMX_L2C_SPAR1) & ~(0xFF << field)) |
 126                                mask << field);
 127                 break;
 128         case 0x8:
 129                 cvmx_write_csr(CVMX_L2C_SPAR2,
 130                                (cvmx_read_csr(CVMX_L2C_SPAR2) & ~(0xFF << field)) |
 131                                mask << field);
 132                 break;
 133         case 0xC:
 134                 cvmx_write_csr(CVMX_L2C_SPAR3,
 135                                (cvmx_read_csr(CVMX_L2C_SPAR3) & ~(0xFF << field)) |
 136                                mask << field);
 137                 break;
 138         }
 139         return 0;
 140 }
 141
 142 int cvmx_l2c_set_hw_way_partition(uint32_t mask)
 143 {
 144         uint32_t valid_mask;
 145
 146         valid_mask = (0x1 << cvmx_l2c_get_num_assoc()) - 1;
 147         mask &= valid_mask;
 148
 149         /* A UMSK setting which blocks all L2C Ways is an error on some chips */
 150         if (mask == valid_mask  && !OCTEON_IS_MODEL(OCTEON_CN63XX))
 151                 return -1;
 152
 153         if (OCTEON_IS_MODEL(OCTEON_CN63XX))
 154                 cvmx_write_csr(CVMX_L2C_WPAR_IOBX(0), mask);
 155         else
 156                 cvmx_write_csr(CVMX_L2C_SPAR4,
 157                                (cvmx_read_csr(CVMX_L2C_SPAR4) & ~0xFF) | mask);
 158         return 0;
 159 }
 160
 161 int cvmx_l2c_get_hw_way_partition(void)
 162 {
 163         if (OCTEON_IS_MODEL(OCTEON_CN63XX))
 164                 return cvmx_read_csr(CVMX_L2C_WPAR_IOBX(0)) & 0xffff;
 165         else
 166                 return cvmx_read_csr(CVMX_L2C_SPAR4) & (0xFF);
 167 }
 168
 169 void cvmx_l2c_config_perf(uint32_t counter, enum cvmx_l2c_event event,
 170                           uint32_t clear_on_read)
 171 {
 172         if (OCTEON_IS_MODEL(OCTEON_CN5XXX) || OCTEON_IS_MODEL(OCTEON_CN3XXX)) {
 173                 union cvmx_l2c_pfctl pfctl;
 174
 175                 pfctl.u64 = cvmx_read_csr(CVMX_L2C_PFCTL);
 176
 177                 switch (counter) {
 178                 case 0:
 179                         pfctl.s.cnt0sel = event;
 180                         pfctl.s.cnt0ena = 1;
 181                         pfctl.s.cnt0rdclr = clear_on_read;
 182                         break;
 183                 case 1:
 184                         pfctl.s.cnt1sel = event;
 185                         pfctl.s.cnt1ena = 1;
 186                         pfctl.s.cnt1rdclr = clear_on_read;
 187                         break;
 188                 case 2:
 189                         pfctl.s.cnt2sel = event;
 190                         pfctl.s.cnt2ena = 1;
 191                         pfctl.s.cnt2rdclr = clear_on_read;
 192                         break;
 193                 case 3:
 194                 default:
 195                         pfctl.s.cnt3sel = event;
 196                         pfctl.s.cnt3ena = 1;
 197                         pfctl.s.cnt3rdclr = clear_on_read;
 198                         break;
 199                 }
 200
 201                 cvmx_write_csr(CVMX_L2C_PFCTL, pfctl.u64);
 202         } else {
 203                 union cvmx_l2c_tadx_prf l2c_tadx_prf;
 204                 int tad;
 205
 206                 cvmx_dprintf("L2C performance counter events are different for this chip, mapping 'event' to cvmx_l2c_tad_event_t\n");
 207                 if (clear_on_read)
 208                         cvmx_dprintf("L2C counters don't support clear on read for this chip\n");
 209
 210                 l2c_tadx_prf.u64 = cvmx_read_csr(CVMX_L2C_TADX_PRF(0));
 211
 212                 switch (counter) {
 213                 case 0:
 214                         l2c_tadx_prf.s.cnt0sel = event;
 215                         break;
 216                 case 1:
 217                         l2c_tadx_prf.s.cnt1sel = event;
 218                         break;
 219                 case 2:
 220                         l2c_tadx_prf.s.cnt2sel = event;
 221                         break;
 222                 default:
 223                 case 3:
 224                         l2c_tadx_prf.s.cnt3sel = event;
 225                         break;
 226                 }
 227                 for (tad = 0; tad < CVMX_L2C_TADS; tad++)
 228                         cvmx_write_csr(CVMX_L2C_TADX_PRF(tad),
 229                                        l2c_tadx_prf.u64);
 230         }
 231 }
 232
 233 uint64_t cvmx_l2c_read_perf(uint32_t counter)
 234 {
 235         switch (counter) {
 236         case 0:
 237                 if (OCTEON_IS_MODEL(OCTEON_CN5XXX) || OCTEON_IS_MODEL(OCTEON_CN3XXX))
 238                         return cvmx_read_csr(CVMX_L2C_PFC0);
 239                 else {
 240                         uint64_t counter = 0;
 241                         int tad;
 242
 243                         for (tad = 0; tad < CVMX_L2C_TADS; tad++)
 244                                 counter += cvmx_read_csr(CVMX_L2C_TADX_PFC0(tad));
 245                         return counter;
 246                 }
 247         case 1:
 248                 if (OCTEON_IS_MODEL(OCTEON_CN5XXX) || OCTEON_IS_MODEL(OCTEON_CN3XXX))
 249                         return cvmx_read_csr(CVMX_L2C_PFC1);
 250                 else {
 251                         uint64_t counter = 0;
 252                         int tad;
 253
 254                         for (tad = 0; tad < CVMX_L2C_TADS; tad++)
 255                                 counter += cvmx_read_csr(CVMX_L2C_TADX_PFC1(tad));
 256                         return counter;
 257                 }
 258         case 2:
 259                 if (OCTEON_IS_MODEL(OCTEON_CN5XXX) || OCTEON_IS_MODEL(OCTEON_CN3XXX))
 260                         return cvmx_read_csr(CVMX_L2C_PFC2);
 261                 else {
 262                         uint64_t counter = 0;
 263                         int tad;
 264
 265                         for (tad = 0; tad < CVMX_L2C_TADS; tad++)
 266                                 counter += cvmx_read_csr(CVMX_L2C_TADX_PFC2(tad));
 267                         return counter;
 268                 }
 269         case 3:
 270         default:
 271                 if (OCTEON_IS_MODEL(OCTEON_CN5XXX) || OCTEON_IS_MODEL(OCTEON_CN3XXX))
 272                         return cvmx_read_csr(CVMX_L2C_PFC3);
 273                 else {
 274                         uint64_t counter = 0;
 275                         int tad;
 276
 277                         for (tad = 0; tad < CVMX_L2C_TADS; tad++)
 278                                 counter += cvmx_read_csr(CVMX_L2C_TADX_PFC3(tad));
 279                         return counter;
 280                 }
 281         }
 282 }
 283
 284 /**
 285  * @INTERNAL
 286  * Helper function use to fault in cache lines for L2 cache locking
 287  *
 288  * @addr:   Address of base of memory region to read into L2 cache
 289  * @len:    Length (in bytes) of region to fault in
 290  */
 291 static void fault_in(uint64_t addr, int len)
 292 {
 293         char *ptr;
 294
 295         /*
 296          * Adjust addr and length so we get all cache lines even for
 297          * small ranges spanning two cache lines.
 298          */
 299         len += addr & CVMX_CACHE_LINE_MASK;
 300         addr &= ~CVMX_CACHE_LINE_MASK;
 301         ptr = cvmx_phys_to_ptr(addr);
 302         /*
 303          * Invalidate L1 cache to make sure all loads result in data
 304          * being in L2.
 305          */
 306         CVMX_DCACHE_INVALIDATE;
 307         while (len > 0) {
 308                 READ_ONCE(*ptr);
 309                 len -= CVMX_CACHE_LINE_SIZE;
 310                 ptr += CVMX_CACHE_LINE_SIZE;
 311         }
 312 }
 313
 314 int cvmx_l2c_lock_line(uint64_t addr)
 315 {
 316         if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
 317                 int shift = CVMX_L2C_TAG_ADDR_ALIAS_SHIFT;
 318                 uint64_t assoc = cvmx_l2c_get_num_assoc();
 319                 uint64_t tag = addr >> shift;
 320                 uint64_t index = CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, cvmx_l2c_address_to_index(addr) << CVMX_L2C_IDX_ADDR_SHIFT);
 321                 uint64_t way;
 322                 union cvmx_l2c_tadx_tag l2c_tadx_tag;
 323
 324                 CVMX_CACHE_LCKL2(CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, addr), 0);
 325
 326                 /* Make sure we were able to lock the line */
 327                 for (way = 0; way < assoc; way++) {
 328                         CVMX_CACHE_LTGL2I(index | (way << shift), 0);
 329                         /* make sure CVMX_L2C_TADX_TAG is updated */
 330                         CVMX_SYNC;
 331                         l2c_tadx_tag.u64 = cvmx_read_csr(CVMX_L2C_TADX_TAG(0));
 332                         if (l2c_tadx_tag.s.valid && l2c_tadx_tag.s.tag == tag)
 333                                 break;
 334                 }
 335
 336                 /* Check if a valid line is found */
 337                 if (way >= assoc) {
 338                         /* cvmx_dprintf("ERROR: cvmx_l2c_lock_line: line not found for locking at 0x%llx address\n", (unsigned long long)addr); */
 339                         return -1;
 340                 }
 341
 342                 /* Check if lock bit is not set */
 343                 if (!l2c_tadx_tag.s.lock) {
 344                         /* cvmx_dprintf("ERROR: cvmx_l2c_lock_line: Not able to lock at 0x%llx address\n", (unsigned long long)addr); */
 345                         return -1;
 346                 }
 347                 return way;
 348         } else {
 349                 int retval = 0;
 350                 union cvmx_l2c_dbg l2cdbg;
 351                 union cvmx_l2c_lckbase lckbase;
 352                 union cvmx_l2c_lckoff lckoff;
 353                 union cvmx_l2t_err l2t_err;
 354
 355                 cvmx_spinlock_lock(&cvmx_l2c_spinlock);
 356
 357                 l2cdbg.u64 = 0;
 358                 lckbase.u64 = 0;
 359                 lckoff.u64 = 0;
 360
 361                 /* Clear l2t error bits if set */
 362                 l2t_err.u64 = cvmx_read_csr(CVMX_L2T_ERR);
 363                 l2t_err.s.lckerr = 1;
 364                 l2t_err.s.lckerr2 = 1;
 365                 cvmx_write_csr(CVMX_L2T_ERR, l2t_err.u64);
 366
 367                 addr &= ~CVMX_CACHE_LINE_MASK;
 368
 369                 /* Set this core as debug core */
 370                 l2cdbg.s.ppnum = cvmx_get_core_num();
 371                 CVMX_SYNC;
 372                 cvmx_write_csr(CVMX_L2C_DBG, l2cdbg.u64);
 373                 cvmx_read_csr(CVMX_L2C_DBG);
 374
 375                 lckoff.s.lck_offset = 0; /* Only lock 1 line at a time */
 376                 cvmx_write_csr(CVMX_L2C_LCKOFF, lckoff.u64);
 377                 cvmx_read_csr(CVMX_L2C_LCKOFF);
 378
 379                 if (((union cvmx_l2c_cfg)(cvmx_read_csr(CVMX_L2C_CFG))).s.idxalias) {
 380                         int alias_shift = CVMX_L2C_IDX_ADDR_SHIFT + 2 * CVMX_L2_SET_BITS - 1;
 381                         uint64_t addr_tmp = addr ^ (addr & ((1 << alias_shift) - 1)) >> CVMX_L2_SET_BITS;
 382
 383                         lckbase.s.lck_base = addr_tmp >> 7;
 384
 385                 } else {
 386                         lckbase.s.lck_base = addr >> 7;
 387                 }
 388
 389                 lckbase.s.lck_ena = 1;
 390                 cvmx_write_csr(CVMX_L2C_LCKBASE, lckbase.u64);
 391                 /* Make sure it gets there */
 392                 cvmx_read_csr(CVMX_L2C_LCKBASE);
 393
 394                 fault_in(addr, CVMX_CACHE_LINE_SIZE);
 395
 396                 lckbase.s.lck_ena = 0;
 397                 cvmx_write_csr(CVMX_L2C_LCKBASE, lckbase.u64);
 398                 /* Make sure it gets there */
 399                 cvmx_read_csr(CVMX_L2C_LCKBASE);
 400
 401                 /* Stop being debug core */
 402                 cvmx_write_csr(CVMX_L2C_DBG, 0);
 403                 cvmx_read_csr(CVMX_L2C_DBG);
 404
 405                 l2t_err.u64 = cvmx_read_csr(CVMX_L2T_ERR);
 406                 if (l2t_err.s.lckerr || l2t_err.s.lckerr2)
 407                         retval = 1;  /* We were unable to lock the line */
 408
 409                 cvmx_spinlock_unlock(&cvmx_l2c_spinlock);
 410                 return retval;
 411         }
 412 }
 413
 414 int cvmx_l2c_lock_mem_region(uint64_t start, uint64_t len)
 415 {
 416         int retval = 0;
 417
 418         /* Round start/end to cache line boundaries */
 419         len += start & CVMX_CACHE_LINE_MASK;
 420         start &= ~CVMX_CACHE_LINE_MASK;
 421         len = (len + CVMX_CACHE_LINE_MASK) & ~CVMX_CACHE_LINE_MASK;
 422
 423         while (len) {
 424                 retval += cvmx_l2c_lock_line(start);
 425                 start += CVMX_CACHE_LINE_SIZE;
 426                 len -= CVMX_CACHE_LINE_SIZE;
 427         }
 428         return retval;
 429 }
 430
 431 void cvmx_l2c_flush(void)
 432 {
 433         uint64_t assoc, set;
 434         uint64_t n_assoc, n_set;
 435
 436         n_set = cvmx_l2c_get_num_sets();
 437         n_assoc = cvmx_l2c_get_num_assoc();
 438
 439         if (OCTEON_IS_MODEL(OCTEON_CN6XXX)) {
 440                 uint64_t address;
 441                 /* These may look like constants, but they aren't... */
 442                 int assoc_shift = CVMX_L2C_TAG_ADDR_ALIAS_SHIFT;
 443                 int set_shift = CVMX_L2C_IDX_ADDR_SHIFT;
 444
 445                 for (set = 0; set < n_set; set++) {
 446                         for (assoc = 0; assoc < n_assoc; assoc++) {
 447                                 address = CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS,
 448                                                        (assoc << assoc_shift) | (set << set_shift));
 449                                 CVMX_CACHE_WBIL2I(address, 0);
 450                         }
 451                 }
 452         } else {
 453                 for (set = 0; set < n_set; set++)
 454                         for (assoc = 0; assoc < n_assoc; assoc++)
 455                                 cvmx_l2c_flush_line(assoc, set);
 456         }
 457 }
 458
 459
 460 int cvmx_l2c_unlock_line(uint64_t address)
 461 {
 462
 463         if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
 464                 int assoc;
 465                 union cvmx_l2c_tag tag;
 466                 uint32_t tag_addr;
 467                 uint32_t index = cvmx_l2c_address_to_index(address);
 468
 469                 tag_addr = ((address >> CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) & ((1 << CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) - 1));
 470
 471                 /*
 472                  * For 63XX, we can flush a line by using the physical
 473                  * address directly, so finding the cache line used by
 474                  * the address is only required to provide the proper
 475                  * return value for the function.
 476                  */
 477                 for (assoc = 0; assoc < CVMX_L2_ASSOC; assoc++) {
 478                         tag = cvmx_l2c_get_tag(assoc, index);
 479
 480                         if (tag.s.V && (tag.s.addr == tag_addr)) {
 481                                 CVMX_CACHE_WBIL2(CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, address), 0);
 482                                 return tag.s.L;
 483                         }
 484                 }
 485         } else {
 486                 int assoc;
 487                 union cvmx_l2c_tag tag;
 488                 uint32_t tag_addr;
 489
 490                 uint32_t index = cvmx_l2c_address_to_index(address);
 491
 492                 /* Compute portion of address that is stored in tag */
 493                 tag_addr = ((address >> CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) & ((1 << CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) - 1));
 494                 for (assoc = 0; assoc < CVMX_L2_ASSOC; assoc++) {
 495                         tag = cvmx_l2c_get_tag(assoc, index);
 496
 497                         if (tag.s.V && (tag.s.addr == tag_addr)) {
 498                                 cvmx_l2c_flush_line(assoc, index);
 499                                 return tag.s.L;
 500                         }
 501                 }
 502         }
 503         return 0;
 504 }
 505
 506 int cvmx_l2c_unlock_mem_region(uint64_t start, uint64_t len)
 507 {
 508         int num_unlocked = 0;
 509         /* Round start/end to cache line boundaries */
 510         len += start & CVMX_CACHE_LINE_MASK;
 511         start &= ~CVMX_CACHE_LINE_MASK;
 512         len = (len + CVMX_CACHE_LINE_MASK) & ~CVMX_CACHE_LINE_MASK;
 513         while (len > 0) {
 514                 num_unlocked += cvmx_l2c_unlock_line(start);
 515                 start += CVMX_CACHE_LINE_SIZE;
 516                 len -= CVMX_CACHE_LINE_SIZE;
 517         }
 518
 519         return num_unlocked;
 520 }
 521
 522 /*
 523  * Internal l2c tag types.  These are converted to a generic structure
 524  * that can be used on all chips.
 525  */
 526 union __cvmx_l2c_tag {
 527         uint64_t u64;
 528         struct cvmx_l2c_tag_cn50xx {
 529                 __BITFIELD_FIELD(uint64_t reserved:40,
 530                 __BITFIELD_FIELD(uint64_t V:1,          /* Line valid */
 531                 __BITFIELD_FIELD(uint64_t D:1,          /* Line dirty */
 532                 __BITFIELD_FIELD(uint64_t L:1,          /* Line locked */
 533                 __BITFIELD_FIELD(uint64_t U:1,          /* Use, LRU eviction */
 534                 __BITFIELD_FIELD(uint64_t addr:20,      /* Phys addr (33..14) */
 535                 ;))))))
 536         } cn50xx;
 537         struct cvmx_l2c_tag_cn30xx {
 538                 __BITFIELD_FIELD(uint64_t reserved:41,
 539                 __BITFIELD_FIELD(uint64_t V:1,          /* Line valid */
 540                 __BITFIELD_FIELD(uint64_t D:1,          /* Line dirty */
 541                 __BITFIELD_FIELD(uint64_t L:1,          /* Line locked */
 542                 __BITFIELD_FIELD(uint64_t U:1,          /* Use, LRU eviction */
 543                 __BITFIELD_FIELD(uint64_t addr:19,      /* Phys addr (33..15) */
 544                 ;))))))
 545         } cn30xx;
 546         struct cvmx_l2c_tag_cn31xx {
 547                 __BITFIELD_FIELD(uint64_t reserved:42,
 548                 __BITFIELD_FIELD(uint64_t V:1,          /* Line valid */
 549                 __BITFIELD_FIELD(uint64_t D:1,          /* Line dirty */
 550                 __BITFIELD_FIELD(uint64_t L:1,          /* Line locked */
 551                 __BITFIELD_FIELD(uint64_t U:1,          /* Use, LRU eviction */
 552                 __BITFIELD_FIELD(uint64_t addr:18,      /* Phys addr (33..16) */
 553                 ;))))))
 554         } cn31xx;
 555         struct cvmx_l2c_tag_cn38xx {
 556                 __BITFIELD_FIELD(uint64_t reserved:43,
 557                 __BITFIELD_FIELD(uint64_t V:1,          /* Line valid */
 558                 __BITFIELD_FIELD(uint64_t D:1,          /* Line dirty */
 559                 __BITFIELD_FIELD(uint64_t L:1,          /* Line locked */
 560                 __BITFIELD_FIELD(uint64_t U:1,          /* Use, LRU eviction */
 561                 __BITFIELD_FIELD(uint64_t addr:17,      /* Phys addr (33..17) */
 562                 ;))))))
 563         } cn38xx;
 564         struct cvmx_l2c_tag_cn58xx {
 565                 __BITFIELD_FIELD(uint64_t reserved:44,
 566                 __BITFIELD_FIELD(uint64_t V:1,          /* Line valid */
 567                 __BITFIELD_FIELD(uint64_t D:1,          /* Line dirty */
 568                 __BITFIELD_FIELD(uint64_t L:1,          /* Line locked */
 569                 __BITFIELD_FIELD(uint64_t U:1,          /* Use, LRU eviction */
 570                 __BITFIELD_FIELD(uint64_t addr:16,      /* Phys addr (33..18) */
 571                 ;))))))
 572         } cn58xx;
 573         struct cvmx_l2c_tag_cn58xx cn56xx;      /* 2048 sets */
 574         struct cvmx_l2c_tag_cn31xx cn52xx;      /* 512 sets */
 575 };
 576
 577
 578 /**
 579  * @INTERNAL
 580  * Function to read a L2C tag.  This code make the current core
 581  * the 'debug core' for the L2.  This code must only be executed by
 582  * 1 core at a time.
 583  *
 584  * @assoc:  Association (way) of the tag to dump
 585  * @index:  Index of the cacheline
 586  *
 587  * Returns The Octeon model specific tag structure.  This is
 588  *         translated by a wrapper function to a generic form that is
 589  *         easier for applications to use.
 590  */
 591 static union __cvmx_l2c_tag __read_l2_tag(uint64_t assoc, uint64_t index)
 592 {
 593
 594         uint64_t debug_tag_addr = CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, (index << 7) + 96);
 595         uint64_t core = cvmx_get_core_num();
 596         union __cvmx_l2c_tag tag_val;
 597         uint64_t dbg_addr = CVMX_L2C_DBG;
 598         unsigned long flags;
 599         union cvmx_l2c_dbg debug_val;
 600
 601         debug_val.u64 = 0;
 602         /*
 603          * For low core count parts, the core number is always small
 604          * enough to stay in the correct field and not set any
 605          * reserved bits.
 606          */
 607         debug_val.s.ppnum = core;
 608         debug_val.s.l2t = 1;
 609         debug_val.s.set = assoc;
 610
 611         local_irq_save(flags);
 612         /*
 613          * Make sure core is quiet (no prefetches, etc.) before
 614          * entering debug mode.
 615          */
 616         CVMX_SYNC;
 617         /* Flush L1 to make sure debug load misses L1 */
 618         CVMX_DCACHE_INVALIDATE;
 619
 620         /*
 621          * The following must be done in assembly as when in debug
 622          * mode all data loads from L2 return special debug data, not
 623          * normal memory contents.  Also, interrupts must be disabled,
 624          * since if an interrupt occurs while in debug mode the ISR
 625          * will get debug data from all its memory * reads instead of
 626          * the contents of memory.
 627          */
 628
 629         asm volatile (
 630                 ".set push\n\t"
 631                 ".set mips64\n\t"
 632                 ".set noreorder\n\t"
 633                 "sd    %[dbg_val], 0(%[dbg_addr])\n\t"   /* Enter debug mode, wait for store */
 634                 "ld    $0, 0(%[dbg_addr])\n\t"
 635                 "ld    %[tag_val], 0(%[tag_addr])\n\t"   /* Read L2C tag data */
 636                 "sd    $0, 0(%[dbg_addr])\n\t"          /* Exit debug mode, wait for store */
 637                 "ld    $0, 0(%[dbg_addr])\n\t"
 638                 "cache 9, 0($0)\n\t"             /* Invalidate dcache to discard debug data */
 639                 ".set pop"
 640                 : [tag_val] "=r" (tag_val)
 641                 : [dbg_addr] "r" (dbg_addr), [dbg_val] "r" (debug_val), [tag_addr] "r" (debug_tag_addr)
 642                 : "memory");
 643
 644         local_irq_restore(flags);
 645
 646         return tag_val;
 647 }
 648
 649
 650 union cvmx_l2c_tag cvmx_l2c_get_tag(uint32_t association, uint32_t index)
 651 {
 652         union cvmx_l2c_tag tag;
 653
 654         tag.u64 = 0;
 655         if ((int)association >= cvmx_l2c_get_num_assoc()) {
 656                 cvmx_dprintf("ERROR: cvmx_l2c_get_tag association out of range\n");
 657                 return tag;
 658         }
 659         if ((int)index >= cvmx_l2c_get_num_sets()) {
 660                 cvmx_dprintf("ERROR: cvmx_l2c_get_tag index out of range (arg: %d, max: %d)\n",
 661                              (int)index, cvmx_l2c_get_num_sets());
 662                 return tag;
 663         }
 664         if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
 665                 union cvmx_l2c_tadx_tag l2c_tadx_tag;
 666                 uint64_t address = CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS,
 667                                                 (association << CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) |
 668                                                 (index << CVMX_L2C_IDX_ADDR_SHIFT));
 669                 /*
 670                  * Use L2 cache Index load tag cache instruction, as
 671                  * hardware loads the virtual tag for the L2 cache
 672                  * block with the contents of L2C_TAD0_TAG
 673                  * register.
 674                  */
 675                 CVMX_CACHE_LTGL2I(address, 0);
 676                 CVMX_SYNC;   /* make sure CVMX_L2C_TADX_TAG is updated */
 677                 l2c_tadx_tag.u64 = cvmx_read_csr(CVMX_L2C_TADX_TAG(0));
 678
 679                 tag.s.V     = l2c_tadx_tag.s.valid;
 680                 tag.s.D     = l2c_tadx_tag.s.dirty;
 681                 tag.s.L     = l2c_tadx_tag.s.lock;
 682                 tag.s.U     = l2c_tadx_tag.s.use;
 683                 tag.s.addr  = l2c_tadx_tag.s.tag;
 684         } else {
 685                 union __cvmx_l2c_tag tmp_tag;
 686                 /* __read_l2_tag is intended for internal use only */
 687                 tmp_tag = __read_l2_tag(association, index);
 688
 689                 /*
 690                  * Convert all tag structure types to generic version,
 691                  * as it can represent all models.
 692                  */
 693                 if (OCTEON_IS_MODEL(OCTEON_CN58XX) || OCTEON_IS_MODEL(OCTEON_CN56XX)) {
 694                         tag.s.V    = tmp_tag.cn58xx.V;
 695                         tag.s.D    = tmp_tag.cn58xx.D;
 696                         tag.s.L    = tmp_tag.cn58xx.L;
 697                         tag.s.U    = tmp_tag.cn58xx.U;
 698                         tag.s.addr = tmp_tag.cn58xx.addr;
 699                 } else if (OCTEON_IS_MODEL(OCTEON_CN38XX)) {
 700                         tag.s.V    = tmp_tag.cn38xx.V;
 701                         tag.s.D    = tmp_tag.cn38xx.D;
 702                         tag.s.L    = tmp_tag.cn38xx.L;
 703                         tag.s.U    = tmp_tag.cn38xx.U;
 704                         tag.s.addr = tmp_tag.cn38xx.addr;
 705                 } else if (OCTEON_IS_MODEL(OCTEON_CN31XX) || OCTEON_IS_MODEL(OCTEON_CN52XX)) {
 706                         tag.s.V    = tmp_tag.cn31xx.V;
 707                         tag.s.D    = tmp_tag.cn31xx.D;
 708                         tag.s.L    = tmp_tag.cn31xx.L;
 709                         tag.s.U    = tmp_tag.cn31xx.U;
 710                         tag.s.addr = tmp_tag.cn31xx.addr;
 711                 } else if (OCTEON_IS_MODEL(OCTEON_CN30XX)) {
 712                         tag.s.V    = tmp_tag.cn30xx.V;
 713                         tag.s.D    = tmp_tag.cn30xx.D;
 714                         tag.s.L    = tmp_tag.cn30xx.L;
 715                         tag.s.U    = tmp_tag.cn30xx.U;
 716                         tag.s.addr = tmp_tag.cn30xx.addr;
 717                 } else if (OCTEON_IS_MODEL(OCTEON_CN50XX)) {
 718                         tag.s.V    = tmp_tag.cn50xx.V;
 719                         tag.s.D    = tmp_tag.cn50xx.D;
 720                         tag.s.L    = tmp_tag.cn50xx.L;
 721                         tag.s.U    = tmp_tag.cn50xx.U;
 722                         tag.s.addr = tmp_tag.cn50xx.addr;
 723                 } else {
 724                         cvmx_dprintf("Unsupported OCTEON Model in %s\n", __func__);
 725                 }
 726         }
 727         return tag;
 728 }
 729
 730 uint32_t cvmx_l2c_address_to_index(uint64_t addr)
 731 {
 732         uint64_t idx = addr >> CVMX_L2C_IDX_ADDR_SHIFT;
 733         int indxalias = 0;
 734
 735         if (OCTEON_IS_MODEL(OCTEON_CN6XXX)) {
 736                 union cvmx_l2c_ctl l2c_ctl;
 737
 738                 l2c_ctl.u64 = cvmx_read_csr(CVMX_L2C_CTL);
 739                 indxalias = !l2c_ctl.s.disidxalias;
 740         } else {
 741                 union cvmx_l2c_cfg l2c_cfg;
 742
 743                 l2c_cfg.u64 = cvmx_read_csr(CVMX_L2C_CFG);
 744                 indxalias = l2c_cfg.s.idxalias;
 745         }
 746
 747         if (indxalias) {
 748                 if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
 749                         uint32_t a_14_12 = (idx / (CVMX_L2C_MEMBANK_SELECT_SIZE/(1<<CVMX_L2C_IDX_ADDR_SHIFT))) & 0x7;
 750
 751                         idx ^= idx / cvmx_l2c_get_num_sets();
 752                         idx ^= a_14_12;
 753                 } else {
 754                         idx ^= ((addr & CVMX_L2C_ALIAS_MASK) >> CVMX_L2C_TAG_ADDR_ALIAS_SHIFT);
 755                 }
 756         }
 757         idx &= CVMX_L2C_IDX_MASK;
 758         return idx;
 759 }
 760
 761 int cvmx_l2c_get_cache_size_bytes(void)
 762 {
 763         return cvmx_l2c_get_num_sets() * cvmx_l2c_get_num_assoc() *
 764                 CVMX_CACHE_LINE_SIZE;
 765 }
 766
 767 /**
 768  * Return log base 2 of the number of sets in the L2 cache
 769  * Returns
 770  */
 771 int cvmx_l2c_get_set_bits(void)
 772 {
 773         int l2_set_bits;
 774
 775         if (OCTEON_IS_MODEL(OCTEON_CN56XX) || OCTEON_IS_MODEL(OCTEON_CN58XX))
 776                 l2_set_bits = 11;       /* 2048 sets */
 777         else if (OCTEON_IS_MODEL(OCTEON_CN38XX) || OCTEON_IS_MODEL(OCTEON_CN63XX))
 778                 l2_set_bits = 10;       /* 1024 sets */
 779         else if (OCTEON_IS_MODEL(OCTEON_CN31XX) || OCTEON_IS_MODEL(OCTEON_CN52XX))
 780                 l2_set_bits = 9;        /* 512 sets */
 781         else if (OCTEON_IS_MODEL(OCTEON_CN30XX))
 782                 l2_set_bits = 8;        /* 256 sets */
 783         else if (OCTEON_IS_MODEL(OCTEON_CN50XX))
 784                 l2_set_bits = 7;        /* 128 sets */
 785         else {
 786                 cvmx_dprintf("Unsupported OCTEON Model in %s\n", __func__);
 787                 l2_set_bits = 11;       /* 2048 sets */
 788         }
 789         return l2_set_bits;
 790 }
 791
 792 /* Return the number of sets in the L2 Cache */
 793 int cvmx_l2c_get_num_sets(void)
 794 {
 795         return 1 << cvmx_l2c_get_set_bits();
 796 }
 797
 798 /* Return the number of associations in the L2 Cache */
 799 int cvmx_l2c_get_num_assoc(void)
 800 {
 801         int l2_assoc;
 802
 803         if (OCTEON_IS_MODEL(OCTEON_CN56XX) ||
 804             OCTEON_IS_MODEL(OCTEON_CN52XX) ||
 805             OCTEON_IS_MODEL(OCTEON_CN58XX) ||
 806             OCTEON_IS_MODEL(OCTEON_CN50XX) ||
 807             OCTEON_IS_MODEL(OCTEON_CN38XX))
 808                 l2_assoc = 8;
 809         else if (OCTEON_IS_MODEL(OCTEON_CN63XX))
 810                 l2_assoc = 16;
 811         else if (OCTEON_IS_MODEL(OCTEON_CN31XX) ||
 812                  OCTEON_IS_MODEL(OCTEON_CN30XX))
 813                 l2_assoc = 4;
 814         else {
 815                 cvmx_dprintf("Unsupported OCTEON Model in %s\n", __func__);
 816                 l2_assoc = 8;
 817         }
 818
 819         /* Check to see if part of the cache is disabled */
 820         if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
 821                 union cvmx_mio_fus_dat3 mio_fus_dat3;
 822
 823                 mio_fus_dat3.u64 = cvmx_read_csr(CVMX_MIO_FUS_DAT3);
 824                 /*
 825                  * cvmx_mio_fus_dat3.s.l2c_crip fuses map as follows
 826                  * <2> will be not used for 63xx
 827                  * <1> disables 1/2 ways
 828                  * <0> disables 1/4 ways
 829                  * They are cumulative, so for 63xx:
 830                  * <1> <0>
 831                  * 0 0 16-way 2MB cache
 832                  * 0 1 12-way 1.5MB cache
 833                  * 1 0 8-way 1MB cache
 834                  * 1 1 4-way 512KB cache
 835                  */
 836
 837                 if (mio_fus_dat3.s.l2c_crip == 3)
 838                         l2_assoc = 4;
 839                 else if (mio_fus_dat3.s.l2c_crip == 2)
 840                         l2_assoc = 8;
 841                 else if (mio_fus_dat3.s.l2c_crip == 1)
 842                         l2_assoc = 12;
 843         } else {
 844                 uint64_t l2d_fus3;
 845
 846                 l2d_fus3 = cvmx_read_csr(CVMX_L2D_FUS3);
 847                 /*
 848                  * Using shifts here, as bit position names are
 849                  * different for each model but they all mean the
 850                  * same.
 851                  */
 852                 if ((l2d_fus3 >> 35) & 0x1)
 853                         l2_assoc = l2_assoc >> 2;
 854                 else if ((l2d_fus3 >> 34) & 0x1)
 855                         l2_assoc = l2_assoc >> 1;
 856         }
 857         return l2_assoc;
 858 }
 859
 860 /**
 861  * Flush a line from the L2 cache
 862  * This should only be called from one core at a time, as this routine
 863  * sets the core to the 'debug' core in order to flush the line.
 864  *
 865  * @assoc:  Association (or way) to flush
 866  * @index:  Index to flush
 867  */
 868 void cvmx_l2c_flush_line(uint32_t assoc, uint32_t index)
 869 {
 870         /* Check the range of the index. */
 871         if (index > (uint32_t)cvmx_l2c_get_num_sets()) {
 872                 cvmx_dprintf("ERROR: cvmx_l2c_flush_line index out of range.\n");
 873                 return;
 874         }
 875
 876         /* Check the range of association. */
 877         if (assoc > (uint32_t)cvmx_l2c_get_num_assoc()) {
 878                 cvmx_dprintf("ERROR: cvmx_l2c_flush_line association out of range.\n");
 879                 return;
 880         }
 881
 882         if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
 883                 uint64_t address;
 884                 /* Create the address based on index and association.
 885                  * Bits<20:17> select the way of the cache block involved in
 886                  *             the operation
 887                  * Bits<16:7> of the effect address select the index
 888                  */
 889                 address = CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS,
 890                                 (assoc << CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) |
 891                                 (index << CVMX_L2C_IDX_ADDR_SHIFT));
 892                 CVMX_CACHE_WBIL2I(address, 0);
 893         } else {
 894                 union cvmx_l2c_dbg l2cdbg;
 895
 896                 l2cdbg.u64 = 0;
 897                 if (!OCTEON_IS_MODEL(OCTEON_CN30XX))
 898                         l2cdbg.s.ppnum = cvmx_get_core_num();
 899                 l2cdbg.s.finv = 1;
 900
 901                 l2cdbg.s.set = assoc;
 902                 cvmx_spinlock_lock(&cvmx_l2c_spinlock);
 903                 /*
 904                  * Enter debug mode, and make sure all other writes
 905                  * complete before we enter debug mode
 906                  */
 907                 CVMX_SYNC;
 908                 cvmx_write_csr(CVMX_L2C_DBG, l2cdbg.u64);
 909                 cvmx_read_csr(CVMX_L2C_DBG);
 910
 911                 CVMX_PREPARE_FOR_STORE(CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS,
 912                                                     index * CVMX_CACHE_LINE_SIZE),
 913                                        0);
 914                 /* Exit debug mode */
 915                 CVMX_SYNC;
 916                 cvmx_write_csr(CVMX_L2C_DBG, 0);
 917                 cvmx_read_csr(CVMX_L2C_DBG);
 918                 cvmx_spinlock_unlock(&cvmx_l2c_spinlock);
 919         }
 920 }