drivers/gpu/drm/amd/amdkfd/kfd_topology.c

   1 /*
   2  * Copyright 2014 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22
  23 #include <linux/types.h>
  24 #include <linux/kernel.h>
  25 #include <linux/pci.h>
  26 #include <linux/errno.h>
  27 #include <linux/acpi.h>
  28 #include <linux/hash.h>
  29 #include <linux/cpufreq.h>
  30 #include <linux/log2.h>
  31 #include <linux/dmi.h>
  32 #include <linux/atomic.h>
  33
  34 #include "kfd_priv.h"
  35 #include "kfd_crat.h"
  36 #include "kfd_topology.h"
  37 #include "kfd_device_queue_manager.h"
  38 #include "kfd_iommu.h"
  39 #include "kfd_svm.h"
  40 #include "amdgpu_amdkfd.h"
  41 #include "amdgpu_ras.h"
  42
  43 /* topology_device_list - Master list of all topology devices */
  44 static struct list_head topology_device_list;
  45 static struct kfd_system_properties sys_props;
  46
  47 static DECLARE_RWSEM(topology_lock);
  48 static atomic_t topology_crat_proximity_domain;
  49
  50 struct kfd_topology_device *kfd_topology_device_by_proximity_domain(
  51                                                 uint32_t proximity_domain)
  52 {
  53         struct kfd_topology_device *top_dev;
  54         struct kfd_topology_device *device = NULL;
  55
  56         down_read(&topology_lock);
  57
  58         list_for_each_entry(top_dev, &topology_device_list, list)
  59                 if (top_dev->proximity_domain == proximity_domain) {
  60                         device = top_dev;
  61                         break;
  62                 }
  63
  64         up_read(&topology_lock);
  65
  66         return device;
  67 }
  68
  69 struct kfd_topology_device *kfd_topology_device_by_id(uint32_t gpu_id)
  70 {
  71         struct kfd_topology_device *top_dev = NULL;
  72         struct kfd_topology_device *ret = NULL;
  73
  74         down_read(&topology_lock);
  75
  76         list_for_each_entry(top_dev, &topology_device_list, list)
  77                 if (top_dev->gpu_id == gpu_id) {
  78                         ret = top_dev;
  79                         break;
  80                 }
  81
  82         up_read(&topology_lock);
  83
  84         return ret;
  85 }
  86
  87 struct kfd_dev *kfd_device_by_id(uint32_t gpu_id)
  88 {
  89         struct kfd_topology_device *top_dev;
  90
  91         top_dev = kfd_topology_device_by_id(gpu_id);
  92         if (!top_dev)
  93                 return NULL;
  94
  95         return top_dev->gpu;
  96 }
  97
  98 struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev)
  99 {
 100         struct kfd_topology_device *top_dev;
 101         struct kfd_dev *device = NULL;
 102
 103         down_read(&topology_lock);
 104
 105         list_for_each_entry(top_dev, &topology_device_list, list)
 106                 if (top_dev->gpu && top_dev->gpu->pdev == pdev) {
 107                         device = top_dev->gpu;
 108                         break;
 109                 }
 110
 111         up_read(&topology_lock);
 112
 113         return device;
 114 }
 115
 116 struct kfd_dev *kfd_device_by_kgd(const struct kgd_dev *kgd)
 117 {
 118         struct kfd_topology_device *top_dev;
 119         struct kfd_dev *device = NULL;
 120
 121         down_read(&topology_lock);
 122
 123         list_for_each_entry(top_dev, &topology_device_list, list)
 124                 if (top_dev->gpu && top_dev->gpu->kgd == kgd) {
 125                         device = top_dev->gpu;
 126                         break;
 127                 }
 128
 129         up_read(&topology_lock);
 130
 131         return device;
 132 }
 133
 134 /* Called with write topology_lock acquired */
 135 static void kfd_release_topology_device(struct kfd_topology_device *dev)
 136 {
 137         struct kfd_mem_properties *mem;
 138         struct kfd_cache_properties *cache;
 139         struct kfd_iolink_properties *iolink;
 140         struct kfd_perf_properties *perf;
 141
 142         list_del(&dev->list);
 143
 144         while (dev->mem_props.next != &dev->mem_props) {
 145                 mem = container_of(dev->mem_props.next,
 146                                 struct kfd_mem_properties, list);
 147                 list_del(&mem->list);
 148                 kfree(mem);
 149         }
 150
 151         while (dev->cache_props.next != &dev->cache_props) {
 152                 cache = container_of(dev->cache_props.next,
 153                                 struct kfd_cache_properties, list);
 154                 list_del(&cache->list);
 155                 kfree(cache);
 156         }
 157
 158         while (dev->io_link_props.next != &dev->io_link_props) {
 159                 iolink = container_of(dev->io_link_props.next,
 160                                 struct kfd_iolink_properties, list);
 161                 list_del(&iolink->list);
 162                 kfree(iolink);
 163         }
 164
 165         while (dev->perf_props.next != &dev->perf_props) {
 166                 perf = container_of(dev->perf_props.next,
 167                                 struct kfd_perf_properties, list);
 168                 list_del(&perf->list);
 169                 kfree(perf);
 170         }
 171
 172         kfree(dev);
 173 }
 174
 175 void kfd_release_topology_device_list(struct list_head *device_list)
 176 {
 177         struct kfd_topology_device *dev;
 178
 179         while (!list_empty(device_list)) {
 180                 dev = list_first_entry(device_list,
 181                                        struct kfd_topology_device, list);
 182                 kfd_release_topology_device(dev);
 183         }
 184 }
 185
 186 static void kfd_release_live_view(void)
 187 {
 188         kfd_release_topology_device_list(&topology_device_list);
 189         memset(&sys_props, 0, sizeof(sys_props));
 190 }
 191
 192 struct kfd_topology_device *kfd_create_topology_device(
 193                                 struct list_head *device_list)
 194 {
 195         struct kfd_topology_device *dev;
 196
 197         dev = kfd_alloc_struct(dev);
 198         if (!dev) {
 199                 pr_err("No memory to allocate a topology device");
 200                 return NULL;
 201         }
 202
 203         INIT_LIST_HEAD(&dev->mem_props);
 204         INIT_LIST_HEAD(&dev->cache_props);
 205         INIT_LIST_HEAD(&dev->io_link_props);
 206         INIT_LIST_HEAD(&dev->perf_props);
 207
 208         list_add_tail(&dev->list, device_list);
 209
 210         return dev;
 211 }
 212
 213
 214 #define sysfs_show_gen_prop(buffer, offs, fmt, ...)             \
 215                 (offs += snprintf(buffer+offs, PAGE_SIZE-offs,  \
 216                                   fmt, __VA_ARGS__))
 217 #define sysfs_show_32bit_prop(buffer, offs, name, value) \
 218                 sysfs_show_gen_prop(buffer, offs, "%s %u\n", name, value)
 219 #define sysfs_show_64bit_prop(buffer, offs, name, value) \
 220                 sysfs_show_gen_prop(buffer, offs, "%s %llu\n", name, value)
 221 #define sysfs_show_32bit_val(buffer, offs, value) \
 222                 sysfs_show_gen_prop(buffer, offs, "%u\n", value)
 223 #define sysfs_show_str_val(buffer, offs, value) \
 224                 sysfs_show_gen_prop(buffer, offs, "%s\n", value)
 225
 226 static ssize_t sysprops_show(struct kobject *kobj, struct attribute *attr,
 227                 char *buffer)
 228 {
 229         int offs = 0;
 230
 231         /* Making sure that the buffer is an empty string */
 232         buffer[0] = 0;
 233
 234         if (attr == &sys_props.attr_genid) {
 235                 sysfs_show_32bit_val(buffer, offs,
 236                                      sys_props.generation_count);
 237         } else if (attr == &sys_props.attr_props) {
 238                 sysfs_show_64bit_prop(buffer, offs, "platform_oem",
 239                                       sys_props.platform_oem);
 240                 sysfs_show_64bit_prop(buffer, offs, "platform_id",
 241                                       sys_props.platform_id);
 242                 sysfs_show_64bit_prop(buffer, offs, "platform_rev",
 243                                       sys_props.platform_rev);
 244         } else {
 245                 offs = -EINVAL;
 246         }
 247
 248         return offs;
 249 }
 250
 251 static void kfd_topology_kobj_release(struct kobject *kobj)
 252 {
 253         kfree(kobj);
 254 }
 255
 256 static const struct sysfs_ops sysprops_ops = {
 257         .show = sysprops_show,
 258 };
 259
 260 static struct kobj_type sysprops_type = {
 261         .release = kfd_topology_kobj_release,
 262         .sysfs_ops = &sysprops_ops,
 263 };
 264
 265 static ssize_t iolink_show(struct kobject *kobj, struct attribute *attr,
 266                 char *buffer)
 267 {
 268         int offs = 0;
 269         struct kfd_iolink_properties *iolink;
 270
 271         /* Making sure that the buffer is an empty string */
 272         buffer[0] = 0;
 273
 274         iolink = container_of(attr, struct kfd_iolink_properties, attr);
 275         if (iolink->gpu && kfd_devcgroup_check_permission(iolink->gpu))
 276                 return -EPERM;
 277         sysfs_show_32bit_prop(buffer, offs, "type", iolink->iolink_type);
 278         sysfs_show_32bit_prop(buffer, offs, "version_major", iolink->ver_maj);
 279         sysfs_show_32bit_prop(buffer, offs, "version_minor", iolink->ver_min);
 280         sysfs_show_32bit_prop(buffer, offs, "node_from", iolink->node_from);
 281         sysfs_show_32bit_prop(buffer, offs, "node_to", iolink->node_to);
 282         sysfs_show_32bit_prop(buffer, offs, "weight", iolink->weight);
 283         sysfs_show_32bit_prop(buffer, offs, "min_latency", iolink->min_latency);
 284         sysfs_show_32bit_prop(buffer, offs, "max_latency", iolink->max_latency);
 285         sysfs_show_32bit_prop(buffer, offs, "min_bandwidth",
 286                               iolink->min_bandwidth);
 287         sysfs_show_32bit_prop(buffer, offs, "max_bandwidth",
 288                               iolink->max_bandwidth);
 289         sysfs_show_32bit_prop(buffer, offs, "recommended_transfer_size",
 290                               iolink->rec_transfer_size);
 291         sysfs_show_32bit_prop(buffer, offs, "flags", iolink->flags);
 292
 293         return offs;
 294 }
 295
 296 static const struct sysfs_ops iolink_ops = {
 297         .show = iolink_show,
 298 };
 299
 300 static struct kobj_type iolink_type = {
 301         .release = kfd_topology_kobj_release,
 302         .sysfs_ops = &iolink_ops,
 303 };
 304
 305 static ssize_t mem_show(struct kobject *kobj, struct attribute *attr,
 306                 char *buffer)
 307 {
 308         int offs = 0;
 309         struct kfd_mem_properties *mem;
 310
 311         /* Making sure that the buffer is an empty string */
 312         buffer[0] = 0;
 313
 314         mem = container_of(attr, struct kfd_mem_properties, attr);
 315         if (mem->gpu && kfd_devcgroup_check_permission(mem->gpu))
 316                 return -EPERM;
 317         sysfs_show_32bit_prop(buffer, offs, "heap_type", mem->heap_type);
 318         sysfs_show_64bit_prop(buffer, offs, "size_in_bytes",
 319                               mem->size_in_bytes);
 320         sysfs_show_32bit_prop(buffer, offs, "flags", mem->flags);
 321         sysfs_show_32bit_prop(buffer, offs, "width", mem->width);
 322         sysfs_show_32bit_prop(buffer, offs, "mem_clk_max",
 323                               mem->mem_clk_max);
 324
 325         return offs;
 326 }
 327
 328 static const struct sysfs_ops mem_ops = {
 329         .show = mem_show,
 330 };
 331
 332 static struct kobj_type mem_type = {
 333         .release = kfd_topology_kobj_release,
 334         .sysfs_ops = &mem_ops,
 335 };
 336
 337 static ssize_t kfd_cache_show(struct kobject *kobj, struct attribute *attr,
 338                 char *buffer)
 339 {
 340         int offs = 0;
 341         uint32_t i, j;
 342         struct kfd_cache_properties *cache;
 343
 344         /* Making sure that the buffer is an empty string */
 345         buffer[0] = 0;
 346
 347         cache = container_of(attr, struct kfd_cache_properties, attr);
 348         if (cache->gpu && kfd_devcgroup_check_permission(cache->gpu))
 349                 return -EPERM;
 350         sysfs_show_32bit_prop(buffer, offs, "processor_id_low",
 351                         cache->processor_id_low);
 352         sysfs_show_32bit_prop(buffer, offs, "level", cache->cache_level);
 353         sysfs_show_32bit_prop(buffer, offs, "size", cache->cache_size);
 354         sysfs_show_32bit_prop(buffer, offs, "cache_line_size",
 355                               cache->cacheline_size);
 356         sysfs_show_32bit_prop(buffer, offs, "cache_lines_per_tag",
 357                               cache->cachelines_per_tag);
 358         sysfs_show_32bit_prop(buffer, offs, "association", cache->cache_assoc);
 359         sysfs_show_32bit_prop(buffer, offs, "latency", cache->cache_latency);
 360         sysfs_show_32bit_prop(buffer, offs, "type", cache->cache_type);
 361         offs += snprintf(buffer+offs, PAGE_SIZE-offs, "sibling_map ");
 362         for (i = 0; i < CRAT_SIBLINGMAP_SIZE; i++)
 363                 for (j = 0; j < sizeof(cache->sibling_map[0])*8; j++)
 364                         /* Check each bit */
 365                         offs += snprintf(buffer+offs, PAGE_SIZE-offs, "%d,",
 366                                          (cache->sibling_map[i] >> j) & 1);
 367
 368         /* Replace the last "," with end of line */
 369         buffer[offs-1] = '\n';
 370         return offs;
 371 }
 372
 373 static const struct sysfs_ops cache_ops = {
 374         .show = kfd_cache_show,
 375 };
 376
 377 static struct kobj_type cache_type = {
 378         .release = kfd_topology_kobj_release,
 379         .sysfs_ops = &cache_ops,
 380 };
 381
 382 /****** Sysfs of Performance Counters ******/
 383
 384 struct kfd_perf_attr {
 385         struct kobj_attribute attr;
 386         uint32_t data;
 387 };
 388
 389 static ssize_t perf_show(struct kobject *kobj, struct kobj_attribute *attrs,
 390                         char *buf)
 391 {
 392         int offs = 0;
 393         struct kfd_perf_attr *attr;
 394
 395         buf[0] = 0;
 396         attr = container_of(attrs, struct kfd_perf_attr, attr);
 397         if (!attr->data) /* invalid data for PMC */
 398                 return 0;
 399         else
 400                 return sysfs_show_32bit_val(buf, offs, attr->data);
 401 }
 402
 403 #define KFD_PERF_DESC(_name, _data)                     \
 404 {                                                       \
 405         .attr  = __ATTR(_name, 0444, perf_show, NULL),  \
 406         .data = _data,                                  \
 407 }
 408
 409 static struct kfd_perf_attr perf_attr_iommu[] = {
 410         KFD_PERF_DESC(max_concurrent, 0),
 411         KFD_PERF_DESC(num_counters, 0),
 412         KFD_PERF_DESC(counter_ids, 0),
 413 };
 414 /****************************************/
 415
 416 static ssize_t node_show(struct kobject *kobj, struct attribute *attr,
 417                 char *buffer)
 418 {
 419         int offs = 0;
 420         struct kfd_topology_device *dev;
 421         uint32_t log_max_watch_addr;
 422
 423         /* Making sure that the buffer is an empty string */
 424         buffer[0] = 0;
 425
 426         if (strcmp(attr->name, "gpu_id") == 0) {
 427                 dev = container_of(attr, struct kfd_topology_device,
 428                                 attr_gpuid);
 429                 if (dev->gpu && kfd_devcgroup_check_permission(dev->gpu))
 430                         return -EPERM;
 431                 return sysfs_show_32bit_val(buffer, offs, dev->gpu_id);
 432         }
 433
 434         if (strcmp(attr->name, "name") == 0) {
 435                 dev = container_of(attr, struct kfd_topology_device,
 436                                 attr_name);
 437
 438                 if (dev->gpu && kfd_devcgroup_check_permission(dev->gpu))
 439                         return -EPERM;
 440                 return sysfs_show_str_val(buffer, offs, dev->node_props.name);
 441         }
 442
 443         dev = container_of(attr, struct kfd_topology_device,
 444                         attr_props);
 445         if (dev->gpu && kfd_devcgroup_check_permission(dev->gpu))
 446                 return -EPERM;
 447         sysfs_show_32bit_prop(buffer, offs, "cpu_cores_count",
 448                               dev->node_props.cpu_cores_count);
 449         sysfs_show_32bit_prop(buffer, offs, "simd_count",
 450                               dev->gpu ? dev->node_props.simd_count : 0);
 451         sysfs_show_32bit_prop(buffer, offs, "mem_banks_count",
 452                               dev->node_props.mem_banks_count);
 453         sysfs_show_32bit_prop(buffer, offs, "caches_count",
 454                               dev->node_props.caches_count);
 455         sysfs_show_32bit_prop(buffer, offs, "io_links_count",
 456                               dev->node_props.io_links_count);
 457         sysfs_show_32bit_prop(buffer, offs, "cpu_core_id_base",
 458                               dev->node_props.cpu_core_id_base);
 459         sysfs_show_32bit_prop(buffer, offs, "simd_id_base",
 460                               dev->node_props.simd_id_base);
 461         sysfs_show_32bit_prop(buffer, offs, "max_waves_per_simd",
 462                               dev->node_props.max_waves_per_simd);
 463         sysfs_show_32bit_prop(buffer, offs, "lds_size_in_kb",
 464                               dev->node_props.lds_size_in_kb);
 465         sysfs_show_32bit_prop(buffer, offs, "gds_size_in_kb",
 466                               dev->node_props.gds_size_in_kb);
 467         sysfs_show_32bit_prop(buffer, offs, "num_gws",
 468                               dev->node_props.num_gws);
 469         sysfs_show_32bit_prop(buffer, offs, "wave_front_size",
 470                               dev->node_props.wave_front_size);
 471         sysfs_show_32bit_prop(buffer, offs, "array_count",
 472                               dev->node_props.array_count);
 473         sysfs_show_32bit_prop(buffer, offs, "simd_arrays_per_engine",
 474                               dev->node_props.simd_arrays_per_engine);
 475         sysfs_show_32bit_prop(buffer, offs, "cu_per_simd_array",
 476                               dev->node_props.cu_per_simd_array);
 477         sysfs_show_32bit_prop(buffer, offs, "simd_per_cu",
 478                               dev->node_props.simd_per_cu);
 479         sysfs_show_32bit_prop(buffer, offs, "max_slots_scratch_cu",
 480                               dev->node_props.max_slots_scratch_cu);
 481         sysfs_show_32bit_prop(buffer, offs, "gfx_target_version",
 482                               dev->node_props.gfx_target_version);
 483         sysfs_show_32bit_prop(buffer, offs, "vendor_id",
 484                               dev->node_props.vendor_id);
 485         sysfs_show_32bit_prop(buffer, offs, "device_id",
 486                               dev->node_props.device_id);
 487         sysfs_show_32bit_prop(buffer, offs, "location_id",
 488                               dev->node_props.location_id);
 489         sysfs_show_32bit_prop(buffer, offs, "domain",
 490                               dev->node_props.domain);
 491         sysfs_show_32bit_prop(buffer, offs, "drm_render_minor",
 492                               dev->node_props.drm_render_minor);
 493         sysfs_show_64bit_prop(buffer, offs, "hive_id",
 494                               dev->node_props.hive_id);
 495         sysfs_show_32bit_prop(buffer, offs, "num_sdma_engines",
 496                               dev->node_props.num_sdma_engines);
 497         sysfs_show_32bit_prop(buffer, offs, "num_sdma_xgmi_engines",
 498                               dev->node_props.num_sdma_xgmi_engines);
 499         sysfs_show_32bit_prop(buffer, offs, "num_sdma_queues_per_engine",
 500                               dev->node_props.num_sdma_queues_per_engine);
 501         sysfs_show_32bit_prop(buffer, offs, "num_cp_queues",
 502                               dev->node_props.num_cp_queues);
 503
 504         if (dev->gpu) {
 505                 log_max_watch_addr =
 506                         __ilog2_u32(dev->gpu->device_info->num_of_watch_points);
 507
 508                 if (log_max_watch_addr) {
 509                         dev->node_props.capability |=
 510                                         HSA_CAP_WATCH_POINTS_SUPPORTED;
 511
 512                         dev->node_props.capability |=
 513                                 ((log_max_watch_addr <<
 514                                         HSA_CAP_WATCH_POINTS_TOTALBITS_SHIFT) &
 515                                 HSA_CAP_WATCH_POINTS_TOTALBITS_MASK);
 516                 }
 517
 518                 if (dev->gpu->device_info->asic_family == CHIP_TONGA)
 519                         dev->node_props.capability |=
 520                                         HSA_CAP_AQL_QUEUE_DOUBLE_MAP;
 521
 522                 sysfs_show_32bit_prop(buffer, offs, "max_engine_clk_fcompute",
 523                         dev->node_props.max_engine_clk_fcompute);
 524
 525                 sysfs_show_64bit_prop(buffer, offs, "local_mem_size", 0ULL);
 526
 527                 sysfs_show_32bit_prop(buffer, offs, "fw_version",
 528                                       dev->gpu->mec_fw_version);
 529                 sysfs_show_32bit_prop(buffer, offs, "capability",
 530                                       dev->node_props.capability);
 531                 sysfs_show_32bit_prop(buffer, offs, "sdma_fw_version",
 532                                       dev->gpu->sdma_fw_version);
 533                 sysfs_show_64bit_prop(buffer, offs, "unique_id",
 534                                       amdgpu_amdkfd_get_unique_id(dev->gpu->kgd));
 535
 536         }
 537
 538         return sysfs_show_32bit_prop(buffer, offs, "max_engine_clk_ccompute",
 539                                      cpufreq_quick_get_max(0)/1000);
 540 }
 541
 542 static const struct sysfs_ops node_ops = {
 543         .show = node_show,
 544 };
 545
 546 static struct kobj_type node_type = {
 547         .release = kfd_topology_kobj_release,
 548         .sysfs_ops = &node_ops,
 549 };
 550
 551 static void kfd_remove_sysfs_file(struct kobject *kobj, struct attribute *attr)
 552 {
 553         sysfs_remove_file(kobj, attr);
 554         kobject_del(kobj);
 555         kobject_put(kobj);
 556 }
 557
 558 static void kfd_remove_sysfs_node_entry(struct kfd_topology_device *dev)
 559 {
 560         struct kfd_iolink_properties *iolink;
 561         struct kfd_cache_properties *cache;
 562         struct kfd_mem_properties *mem;
 563         struct kfd_perf_properties *perf;
 564
 565         if (dev->kobj_iolink) {
 566                 list_for_each_entry(iolink, &dev->io_link_props, list)
 567                         if (iolink->kobj) {
 568                                 kfd_remove_sysfs_file(iolink->kobj,
 569                                                         &iolink->attr);
 570                                 iolink->kobj = NULL;
 571                         }
 572                 kobject_del(dev->kobj_iolink);
 573                 kobject_put(dev->kobj_iolink);
 574                 dev->kobj_iolink = NULL;
 575         }
 576
 577         if (dev->kobj_cache) {
 578                 list_for_each_entry(cache, &dev->cache_props, list)
 579                         if (cache->kobj) {
 580                                 kfd_remove_sysfs_file(cache->kobj,
 581                                                         &cache->attr);
 582                                 cache->kobj = NULL;
 583                         }
 584                 kobject_del(dev->kobj_cache);
 585                 kobject_put(dev->kobj_cache);
 586                 dev->kobj_cache = NULL;
 587         }
 588
 589         if (dev->kobj_mem) {
 590                 list_for_each_entry(mem, &dev->mem_props, list)
 591                         if (mem->kobj) {
 592                                 kfd_remove_sysfs_file(mem->kobj, &mem->attr);
 593                                 mem->kobj = NULL;
 594                         }
 595                 kobject_del(dev->kobj_mem);
 596                 kobject_put(dev->kobj_mem);
 597                 dev->kobj_mem = NULL;
 598         }
 599
 600         if (dev->kobj_perf) {
 601                 list_for_each_entry(perf, &dev->perf_props, list) {
 602                         kfree(perf->attr_group);
 603                         perf->attr_group = NULL;
 604                 }
 605                 kobject_del(dev->kobj_perf);
 606                 kobject_put(dev->kobj_perf);
 607                 dev->kobj_perf = NULL;
 608         }
 609
 610         if (dev->kobj_node) {
 611                 sysfs_remove_file(dev->kobj_node, &dev->attr_gpuid);
 612                 sysfs_remove_file(dev->kobj_node, &dev->attr_name);
 613                 sysfs_remove_file(dev->kobj_node, &dev->attr_props);
 614                 kobject_del(dev->kobj_node);
 615                 kobject_put(dev->kobj_node);
 616                 dev->kobj_node = NULL;
 617         }
 618 }
 619
 620 static int kfd_build_sysfs_node_entry(struct kfd_topology_device *dev,
 621                 uint32_t id)
 622 {
 623         struct kfd_iolink_properties *iolink;
 624         struct kfd_cache_properties *cache;
 625         struct kfd_mem_properties *mem;
 626         struct kfd_perf_properties *perf;
 627         int ret;
 628         uint32_t i, num_attrs;
 629         struct attribute **attrs;
 630
 631         if (WARN_ON(dev->kobj_node))
 632                 return -EEXIST;
 633
 634         /*
 635          * Creating the sysfs folders
 636          */
 637         dev->kobj_node = kfd_alloc_struct(dev->kobj_node);
 638         if (!dev->kobj_node)
 639                 return -ENOMEM;
 640
 641         ret = kobject_init_and_add(dev->kobj_node, &node_type,
 642                         sys_props.kobj_nodes, "%d", id);
 643         if (ret < 0) {
 644                 kobject_put(dev->kobj_node);
 645                 return ret;
 646         }
 647
 648         dev->kobj_mem = kobject_create_and_add("mem_banks", dev->kobj_node);
 649         if (!dev->kobj_mem)
 650                 return -ENOMEM;
 651
 652         dev->kobj_cache = kobject_create_and_add("caches", dev->kobj_node);
 653         if (!dev->kobj_cache)
 654                 return -ENOMEM;
 655
 656         dev->kobj_iolink = kobject_create_and_add("io_links", dev->kobj_node);
 657         if (!dev->kobj_iolink)
 658                 return -ENOMEM;
 659
 660         dev->kobj_perf = kobject_create_and_add("perf", dev->kobj_node);
 661         if (!dev->kobj_perf)
 662                 return -ENOMEM;
 663
 664         /*
 665          * Creating sysfs files for node properties
 666          */
 667         dev->attr_gpuid.name = "gpu_id";
 668         dev->attr_gpuid.mode = KFD_SYSFS_FILE_MODE;
 669         sysfs_attr_init(&dev->attr_gpuid);
 670         dev->attr_name.name = "name";
 671         dev->attr_name.mode = KFD_SYSFS_FILE_MODE;
 672         sysfs_attr_init(&dev->attr_name);
 673         dev->attr_props.name = "properties";
 674         dev->attr_props.mode = KFD_SYSFS_FILE_MODE;
 675         sysfs_attr_init(&dev->attr_props);
 676         ret = sysfs_create_file(dev->kobj_node, &dev->attr_gpuid);
 677         if (ret < 0)
 678                 return ret;
 679         ret = sysfs_create_file(dev->kobj_node, &dev->attr_name);
 680         if (ret < 0)
 681                 return ret;
 682         ret = sysfs_create_file(dev->kobj_node, &dev->attr_props);
 683         if (ret < 0)
 684                 return ret;
 685
 686         i = 0;
 687         list_for_each_entry(mem, &dev->mem_props, list) {
 688                 mem->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL);
 689                 if (!mem->kobj)
 690                         return -ENOMEM;
 691                 ret = kobject_init_and_add(mem->kobj, &mem_type,
 692                                 dev->kobj_mem, "%d", i);
 693                 if (ret < 0) {
 694                         kobject_put(mem->kobj);
 695                         return ret;
 696                 }
 697
 698                 mem->attr.name = "properties";
 699                 mem->attr.mode = KFD_SYSFS_FILE_MODE;
 700                 sysfs_attr_init(&mem->attr);
 701                 ret = sysfs_create_file(mem->kobj, &mem->attr);
 702                 if (ret < 0)
 703                         return ret;
 704                 i++;
 705         }
 706
 707         i = 0;
 708         list_for_each_entry(cache, &dev->cache_props, list) {
 709                 cache->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL);
 710                 if (!cache->kobj)
 711                         return -ENOMEM;
 712                 ret = kobject_init_and_add(cache->kobj, &cache_type,
 713                                 dev->kobj_cache, "%d", i);
 714                 if (ret < 0) {
 715                         kobject_put(cache->kobj);
 716                         return ret;
 717                 }
 718
 719                 cache->attr.name = "properties";
 720                 cache->attr.mode = KFD_SYSFS_FILE_MODE;
 721                 sysfs_attr_init(&cache->attr);
 722                 ret = sysfs_create_file(cache->kobj, &cache->attr);
 723                 if (ret < 0)
 724                         return ret;
 725                 i++;
 726         }
 727
 728         i = 0;
 729         list_for_each_entry(iolink, &dev->io_link_props, list) {
 730                 iolink->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL);
 731                 if (!iolink->kobj)
 732                         return -ENOMEM;
 733                 ret = kobject_init_and_add(iolink->kobj, &iolink_type,
 734                                 dev->kobj_iolink, "%d", i);
 735                 if (ret < 0) {
 736                         kobject_put(iolink->kobj);
 737                         return ret;
 738                 }
 739
 740                 iolink->attr.name = "properties";
 741                 iolink->attr.mode = KFD_SYSFS_FILE_MODE;
 742                 sysfs_attr_init(&iolink->attr);
 743                 ret = sysfs_create_file(iolink->kobj, &iolink->attr);
 744                 if (ret < 0)
 745                         return ret;
 746                 i++;
 747         }
 748
 749         /* All hardware blocks have the same number of attributes. */
 750         num_attrs = ARRAY_SIZE(perf_attr_iommu);
 751         list_for_each_entry(perf, &dev->perf_props, list) {
 752                 perf->attr_group = kzalloc(sizeof(struct kfd_perf_attr)
 753                         * num_attrs + sizeof(struct attribute_group),
 754                         GFP_KERNEL);
 755                 if (!perf->attr_group)
 756                         return -ENOMEM;
 757
 758                 attrs = (struct attribute **)(perf->attr_group + 1);
 759                 if (!strcmp(perf->block_name, "iommu")) {
 760                 /* Information of IOMMU's num_counters and counter_ids is shown
 761                  * under /sys/bus/event_source/devices/amd_iommu. We don't
 762                  * duplicate here.
 763                  */
 764                         perf_attr_iommu[0].data = perf->max_concurrent;
 765                         for (i = 0; i < num_attrs; i++)
 766                                 attrs[i] = &perf_attr_iommu[i].attr.attr;
 767                 }
 768                 perf->attr_group->name = perf->block_name;
 769                 perf->attr_group->attrs = attrs;
 770                 ret = sysfs_create_group(dev->kobj_perf, perf->attr_group);
 771                 if (ret < 0)
 772                         return ret;
 773         }
 774
 775         return 0;
 776 }
 777
 778 /* Called with write topology lock acquired */
 779 static int kfd_build_sysfs_node_tree(void)
 780 {
 781         struct kfd_topology_device *dev;
 782         int ret;
 783         uint32_t i = 0;
 784
 785         list_for_each_entry(dev, &topology_device_list, list) {
 786                 ret = kfd_build_sysfs_node_entry(dev, i);
 787                 if (ret < 0)
 788                         return ret;
 789                 i++;
 790         }
 791
 792         return 0;
 793 }
 794
 795 /* Called with write topology lock acquired */
 796 static void kfd_remove_sysfs_node_tree(void)
 797 {
 798         struct kfd_topology_device *dev;
 799
 800         list_for_each_entry(dev, &topology_device_list, list)
 801                 kfd_remove_sysfs_node_entry(dev);
 802 }
 803
 804 static int kfd_topology_update_sysfs(void)
 805 {
 806         int ret;
 807
 808         if (!sys_props.kobj_topology) {
 809                 sys_props.kobj_topology =
 810                                 kfd_alloc_struct(sys_props.kobj_topology);
 811                 if (!sys_props.kobj_topology)
 812                         return -ENOMEM;
 813
 814                 ret = kobject_init_and_add(sys_props.kobj_topology,
 815                                 &sysprops_type,  &kfd_device->kobj,
 816                                 "topology");
 817                 if (ret < 0) {
 818                         kobject_put(sys_props.kobj_topology);
 819                         return ret;
 820                 }
 821
 822                 sys_props.kobj_nodes = kobject_create_and_add("nodes",
 823                                 sys_props.kobj_topology);
 824                 if (!sys_props.kobj_nodes)
 825                         return -ENOMEM;
 826
 827                 sys_props.attr_genid.name = "generation_id";
 828                 sys_props.attr_genid.mode = KFD_SYSFS_FILE_MODE;
 829                 sysfs_attr_init(&sys_props.attr_genid);
 830                 ret = sysfs_create_file(sys_props.kobj_topology,
 831                                 &sys_props.attr_genid);
 832                 if (ret < 0)
 833                         return ret;
 834
 835                 sys_props.attr_props.name = "system_properties";
 836                 sys_props.attr_props.mode = KFD_SYSFS_FILE_MODE;
 837                 sysfs_attr_init(&sys_props.attr_props);
 838                 ret = sysfs_create_file(sys_props.kobj_topology,
 839                                 &sys_props.attr_props);
 840                 if (ret < 0)
 841                         return ret;
 842         }
 843
 844         kfd_remove_sysfs_node_tree();
 845
 846         return kfd_build_sysfs_node_tree();
 847 }
 848
 849 static void kfd_topology_release_sysfs(void)
 850 {
 851         kfd_remove_sysfs_node_tree();
 852         if (sys_props.kobj_topology) {
 853                 sysfs_remove_file(sys_props.kobj_topology,
 854                                 &sys_props.attr_genid);
 855                 sysfs_remove_file(sys_props.kobj_topology,
 856                                 &sys_props.attr_props);
 857                 if (sys_props.kobj_nodes) {
 858                         kobject_del(sys_props.kobj_nodes);
 859                         kobject_put(sys_props.kobj_nodes);
 860                         sys_props.kobj_nodes = NULL;
 861                 }
 862                 kobject_del(sys_props.kobj_topology);
 863                 kobject_put(sys_props.kobj_topology);
 864                 sys_props.kobj_topology = NULL;
 865         }
 866 }
 867
 868 /* Called with write topology_lock acquired */
 869 static void kfd_topology_update_device_list(struct list_head *temp_list,
 870                                         struct list_head *master_list)
 871 {
 872         while (!list_empty(temp_list)) {
 873                 list_move_tail(temp_list->next, master_list);
 874                 sys_props.num_devices++;
 875         }
 876 }
 877
 878 static void kfd_debug_print_topology(void)
 879 {
 880         struct kfd_topology_device *dev;
 881
 882         down_read(&topology_lock);
 883
 884         dev = list_last_entry(&topology_device_list,
 885                         struct kfd_topology_device, list);
 886         if (dev) {
 887                 if (dev->node_props.cpu_cores_count &&
 888                                 dev->node_props.simd_count) {
 889                         pr_info("Topology: Add APU node [0x%0x:0x%0x]\n",
 890                                 dev->node_props.device_id,
 891                                 dev->node_props.vendor_id);
 892                 } else if (dev->node_props.cpu_cores_count)
 893                         pr_info("Topology: Add CPU node\n");
 894                 else if (dev->node_props.simd_count)
 895                         pr_info("Topology: Add dGPU node [0x%0x:0x%0x]\n",
 896                                 dev->node_props.device_id,
 897                                 dev->node_props.vendor_id);
 898         }
 899         up_read(&topology_lock);
 900 }
 901
 902 /* Helper function for intializing platform_xx members of
 903  * kfd_system_properties. Uses OEM info from the last CPU/APU node.
 904  */
 905 static void kfd_update_system_properties(void)
 906 {
 907         struct kfd_topology_device *dev;
 908
 909         down_read(&topology_lock);
 910         dev = list_last_entry(&topology_device_list,
 911                         struct kfd_topology_device, list);
 912         if (dev) {
 913                 sys_props.platform_id =
 914                         (*((uint64_t *)dev->oem_id)) & CRAT_OEMID_64BIT_MASK;
 915                 sys_props.platform_oem = *((uint64_t *)dev->oem_table_id);
 916                 sys_props.platform_rev = dev->oem_revision;
 917         }
 918         up_read(&topology_lock);
 919 }
 920
 921 static void find_system_memory(const struct dmi_header *dm,
 922         void *private)
 923 {
 924         struct kfd_mem_properties *mem;
 925         u16 mem_width, mem_clock;
 926         struct kfd_topology_device *kdev =
 927                 (struct kfd_topology_device *)private;
 928         const u8 *dmi_data = (const u8 *)(dm + 1);
 929
 930         if (dm->type == DMI_ENTRY_MEM_DEVICE && dm->length >= 0x15) {
 931                 mem_width = (u16)(*(const u16 *)(dmi_data + 0x6));
 932                 mem_clock = (u16)(*(const u16 *)(dmi_data + 0x11));
 933                 list_for_each_entry(mem, &kdev->mem_props, list) {
 934                         if (mem_width != 0xFFFF && mem_width != 0)
 935                                 mem->width = mem_width;
 936                         if (mem_clock != 0)
 937                                 mem->mem_clk_max = mem_clock;
 938                 }
 939         }
 940 }
 941
 942 /*
 943  * Performance counters information is not part of CRAT but we would like to
 944  * put them in the sysfs under topology directory for Thunk to get the data.
 945  * This function is called before updating the sysfs.
 946  */
 947 static int kfd_add_perf_to_topology(struct kfd_topology_device *kdev)
 948 {
 949         /* These are the only counters supported so far */
 950         return kfd_iommu_add_perf_counters(kdev);
 951 }
 952
 953 /* kfd_add_non_crat_information - Add information that is not currently
 954  *      defined in CRAT but is necessary for KFD topology
 955  * @dev - topology device to which addition info is added
 956  */
 957 static void kfd_add_non_crat_information(struct kfd_topology_device *kdev)
 958 {
 959         /* Check if CPU only node. */
 960         if (!kdev->gpu) {
 961                 /* Add system memory information */
 962                 dmi_walk(find_system_memory, kdev);
 963         }
 964         /* TODO: For GPU node, rearrange code from kfd_topology_add_device */
 965 }
 966
 967 /* kfd_is_acpi_crat_invalid - CRAT from ACPI is valid only for AMD APU devices.
 968  *      Ignore CRAT for all other devices. AMD APU is identified if both CPU
 969  *      and GPU cores are present.
 970  * @device_list - topology device list created by parsing ACPI CRAT table.
 971  * @return - TRUE if invalid, FALSE is valid.
 972  */
 973 static bool kfd_is_acpi_crat_invalid(struct list_head *device_list)
 974 {
 975         struct kfd_topology_device *dev;
 976
 977         list_for_each_entry(dev, device_list, list) {
 978                 if (dev->node_props.cpu_cores_count &&
 979                         dev->node_props.simd_count)
 980                         return false;
 981         }
 982         pr_info("Ignoring ACPI CRAT on non-APU system\n");
 983         return true;
 984 }
 985
 986 int kfd_topology_init(void)
 987 {
 988         void *crat_image = NULL;
 989         size_t image_size = 0;
 990         int ret;
 991         struct list_head temp_topology_device_list;
 992         int cpu_only_node = 0;
 993         struct kfd_topology_device *kdev;
 994         int proximity_domain;
 995
 996         /* topology_device_list - Master list of all topology devices
 997          * temp_topology_device_list - temporary list created while parsing CRAT
 998          * or VCRAT. Once parsing is complete the contents of list is moved to
 999          * topology_device_list
1000          */
1001
1002         /* Initialize the head for the both the lists */
1003         INIT_LIST_HEAD(&topology_device_list);
1004         INIT_LIST_HEAD(&temp_topology_device_list);
1005         init_rwsem(&topology_lock);
1006
1007         memset(&sys_props, 0, sizeof(sys_props));
1008
1009         /* Proximity domains in ACPI CRAT tables start counting at
1010          * 0. The same should be true for virtual CRAT tables created
1011          * at this stage. GPUs added later in kfd_topology_add_device
1012          * use a counter.
1013          */
1014         proximity_domain = 0;
1015
1016         /*
1017          * Get the CRAT image from the ACPI. If ACPI doesn't have one
1018          * or if ACPI CRAT is invalid create a virtual CRAT.
1019          * NOTE: The current implementation expects all AMD APUs to have
1020          *      CRAT. If no CRAT is available, it is assumed to be a CPU
1021          */
1022         ret = kfd_create_crat_image_acpi(&crat_image, &image_size);
1023         if (!ret) {
1024                 ret = kfd_parse_crat_table(crat_image,
1025                                            &temp_topology_device_list,
1026                                            proximity_domain);
1027                 if (ret ||
1028                     kfd_is_acpi_crat_invalid(&temp_topology_device_list)) {
1029                         kfd_release_topology_device_list(
1030                                 &temp_topology_device_list);
1031                         kfd_destroy_crat_image(crat_image);
1032                         crat_image = NULL;
1033                 }
1034         }
1035
1036         if (!crat_image) {
1037                 ret = kfd_create_crat_image_virtual(&crat_image, &image_size,
1038                                                     COMPUTE_UNIT_CPU, NULL,
1039                                                     proximity_domain);
1040                 cpu_only_node = 1;
1041                 if (ret) {
1042                         pr_err("Error creating VCRAT table for CPU\n");
1043                         return ret;
1044                 }
1045
1046                 ret = kfd_parse_crat_table(crat_image,
1047                                            &temp_topology_device_list,
1048                                            proximity_domain);
1049                 if (ret) {
1050                         pr_err("Error parsing VCRAT table for CPU\n");
1051                         goto err;
1052                 }
1053         }
1054
1055         kdev = list_first_entry(&temp_topology_device_list,
1056                                 struct kfd_topology_device, list);
1057         kfd_add_perf_to_topology(kdev);
1058
1059         down_write(&topology_lock);
1060         kfd_topology_update_device_list(&temp_topology_device_list,
1061                                         &topology_device_list);
1062         atomic_set(&topology_crat_proximity_domain, sys_props.num_devices-1);
1063         ret = kfd_topology_update_sysfs();
1064         up_write(&topology_lock);
1065
1066         if (!ret) {
1067                 sys_props.generation_count++;
1068                 kfd_update_system_properties();
1069                 kfd_debug_print_topology();
1070         } else
1071                 pr_err("Failed to update topology in sysfs ret=%d\n", ret);
1072
1073         /* For nodes with GPU, this information gets added
1074          * when GPU is detected (kfd_topology_add_device).
1075          */
1076         if (cpu_only_node) {
1077                 /* Add additional information to CPU only node created above */
1078                 down_write(&topology_lock);
1079                 kdev = list_first_entry(&topology_device_list,
1080                                 struct kfd_topology_device, list);
1081                 up_write(&topology_lock);
1082                 kfd_add_non_crat_information(kdev);
1083         }
1084
1085 err:
1086         kfd_destroy_crat_image(crat_image);
1087         return ret;
1088 }
1089
1090 void kfd_topology_shutdown(void)
1091 {
1092         down_write(&topology_lock);
1093         kfd_topology_release_sysfs();
1094         kfd_release_live_view();
1095         up_write(&topology_lock);
1096 }
1097
1098 static uint32_t kfd_generate_gpu_id(struct kfd_dev *gpu)
1099 {
1100         uint32_t hashout;
1101         uint32_t buf[7];
1102         uint64_t local_mem_size;
1103         int i;
1104         struct kfd_local_mem_info local_mem_info;
1105
1106         if (!gpu)
1107                 return 0;
1108
1109         amdgpu_amdkfd_get_local_mem_info(gpu->kgd, &local_mem_info);
1110
1111         local_mem_size = local_mem_info.local_mem_size_private +
1112                         local_mem_info.local_mem_size_public;
1113
1114         buf[0] = gpu->pdev->devfn;
1115         buf[1] = gpu->pdev->subsystem_vendor |
1116                 (gpu->pdev->subsystem_device << 16);
1117         buf[2] = pci_domain_nr(gpu->pdev->bus);
1118         buf[3] = gpu->pdev->device;
1119         buf[4] = gpu->pdev->bus->number;
1120         buf[5] = lower_32_bits(local_mem_size);
1121         buf[6] = upper_32_bits(local_mem_size);
1122
1123         for (i = 0, hashout = 0; i < 7; i++)
1124                 hashout ^= hash_32(buf[i], KFD_GPU_ID_HASH_WIDTH);
1125
1126         return hashout;
1127 }
1128 /* kfd_assign_gpu - Attach @gpu to the correct kfd topology device. If
1129  *              the GPU device is not already present in the topology device
1130  *              list then return NULL. This means a new topology device has to
1131  *              be created for this GPU.
1132  */
1133 static struct kfd_topology_device *kfd_assign_gpu(struct kfd_dev *gpu)
1134 {
1135         struct kfd_topology_device *dev;
1136         struct kfd_topology_device *out_dev = NULL;
1137         struct kfd_mem_properties *mem;
1138         struct kfd_cache_properties *cache;
1139         struct kfd_iolink_properties *iolink;
1140
1141         down_write(&topology_lock);
1142         list_for_each_entry(dev, &topology_device_list, list) {
1143                 /* Discrete GPUs need their own topology device list
1144                  * entries. Don't assign them to CPU/APU nodes.
1145                  */
1146                 if (!gpu->use_iommu_v2 &&
1147                     dev->node_props.cpu_cores_count)
1148                         continue;
1149
1150                 if (!dev->gpu && (dev->node_props.simd_count > 0)) {
1151                         dev->gpu = gpu;
1152                         out_dev = dev;
1153
1154                         list_for_each_entry(mem, &dev->mem_props, list)
1155                                 mem->gpu = dev->gpu;
1156                         list_for_each_entry(cache, &dev->cache_props, list)
1157                                 cache->gpu = dev->gpu;
1158                         list_for_each_entry(iolink, &dev->io_link_props, list)
1159                                 iolink->gpu = dev->gpu;
1160                         break;
1161                 }
1162         }
1163         up_write(&topology_lock);
1164         return out_dev;
1165 }
1166
1167 static void kfd_notify_gpu_change(uint32_t gpu_id, int arrival)
1168 {
1169         /*
1170          * TODO: Generate an event for thunk about the arrival/removal
1171          * of the GPU
1172          */
1173 }
1174
1175 /* kfd_fill_mem_clk_max_info - Since CRAT doesn't have memory clock info,
1176  *              patch this after CRAT parsing.
1177  */
1178 static void kfd_fill_mem_clk_max_info(struct kfd_topology_device *dev)
1179 {
1180         struct kfd_mem_properties *mem;
1181         struct kfd_local_mem_info local_mem_info;
1182
1183         if (!dev)
1184                 return;
1185
1186         /* Currently, amdgpu driver (amdgpu_mc) deals only with GPUs with
1187          * single bank of VRAM local memory.
1188          * for dGPUs - VCRAT reports only one bank of Local Memory
1189          * for APUs - If CRAT from ACPI reports more than one bank, then
1190          *      all the banks will report the same mem_clk_max information
1191          */
1192         amdgpu_amdkfd_get_local_mem_info(dev->gpu->kgd, &local_mem_info);
1193
1194         list_for_each_entry(mem, &dev->mem_props, list)
1195                 mem->mem_clk_max = local_mem_info.mem_clk_max;
1196 }
1197
1198 static void kfd_set_iolink_no_atomics(struct kfd_topology_device *dev,
1199                                         struct kfd_topology_device *target_gpu_dev,
1200                                         struct kfd_iolink_properties *link)
1201 {
1202         /* xgmi always supports atomics between links. */
1203         if (link->iolink_type == CRAT_IOLINK_TYPE_XGMI)
1204                 return;
1205
1206         /* check pcie support to set cpu(dev) flags for target_gpu_dev link. */
1207         if (target_gpu_dev) {
1208                 uint32_t cap;
1209
1210                 pcie_capability_read_dword(target_gpu_dev->gpu->pdev,
1211                                 PCI_EXP_DEVCAP2, &cap);
1212
1213                 if (!(cap & (PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
1214                              PCI_EXP_DEVCAP2_ATOMIC_COMP64)))
1215                         link->flags |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT |
1216                                 CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT;
1217         /* set gpu (dev) flags. */
1218         } else {
1219                 if (!dev->gpu->pci_atomic_requested ||
1220                                 dev->gpu->device_info->asic_family ==
1221                                                         CHIP_HAWAII)
1222                         link->flags |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT |
1223                                 CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT;
1224         }
1225 }
1226
1227 static void kfd_set_iolink_non_coherent(struct kfd_topology_device *to_dev,
1228                 struct kfd_iolink_properties *outbound_link,
1229                 struct kfd_iolink_properties *inbound_link)
1230 {
1231         /* CPU -> GPU with PCIe */
1232         if (!to_dev->gpu &&
1233             inbound_link->iolink_type == CRAT_IOLINK_TYPE_PCIEXPRESS)
1234                 inbound_link->flags |= CRAT_IOLINK_FLAGS_NON_COHERENT;
1235
1236         if (to_dev->gpu) {
1237                 /* GPU <-> GPU with PCIe and
1238                  * Vega20 with XGMI
1239                  */
1240                 if (inbound_link->iolink_type == CRAT_IOLINK_TYPE_PCIEXPRESS ||
1241                     (inbound_link->iolink_type == CRAT_IOLINK_TYPE_XGMI &&
1242                     to_dev->gpu->device_info->asic_family == CHIP_VEGA20)) {
1243                         outbound_link->flags |= CRAT_IOLINK_FLAGS_NON_COHERENT;
1244                         inbound_link->flags |= CRAT_IOLINK_FLAGS_NON_COHERENT;
1245                 }
1246         }
1247 }
1248
1249 static void kfd_fill_iolink_non_crat_info(struct kfd_topology_device *dev)
1250 {
1251         struct kfd_iolink_properties *link, *inbound_link;
1252         struct kfd_topology_device *peer_dev;
1253
1254         if (!dev || !dev->gpu)
1255                 return;
1256
1257         /* GPU only creates direct links so apply flags setting to all */
1258         list_for_each_entry(link, &dev->io_link_props, list) {
1259                 link->flags = CRAT_IOLINK_FLAGS_ENABLED;
1260                 kfd_set_iolink_no_atomics(dev, NULL, link);
1261                 peer_dev = kfd_topology_device_by_proximity_domain(
1262                                 link->node_to);
1263
1264                 if (!peer_dev)
1265                         continue;
1266
1267                 list_for_each_entry(inbound_link, &peer_dev->io_link_props,
1268                                                                         list) {
1269                         if (inbound_link->node_to != link->node_from)
1270                                 continue;
1271
1272                         inbound_link->flags = CRAT_IOLINK_FLAGS_ENABLED;
1273                         kfd_set_iolink_no_atomics(peer_dev, dev, inbound_link);
1274                         kfd_set_iolink_non_coherent(peer_dev, link, inbound_link);
1275                 }
1276         }
1277 }
1278
1279 int kfd_topology_add_device(struct kfd_dev *gpu)
1280 {
1281         uint32_t gpu_id;
1282         struct kfd_topology_device *dev;
1283         struct kfd_cu_info cu_info;
1284         int res = 0;
1285         struct list_head temp_topology_device_list;
1286         void *crat_image = NULL;
1287         size_t image_size = 0;
1288         int proximity_domain;
1289         struct amdgpu_device *adev;
1290
1291         INIT_LIST_HEAD(&temp_topology_device_list);
1292
1293         gpu_id = kfd_generate_gpu_id(gpu);
1294
1295         pr_debug("Adding new GPU (ID: 0x%x) to topology\n", gpu_id);
1296
1297         proximity_domain = atomic_inc_return(&topology_crat_proximity_domain);
1298
1299         /* Check to see if this gpu device exists in the topology_device_list.
1300          * If so, assign the gpu to that device,
1301          * else create a Virtual CRAT for this gpu device and then parse that
1302          * CRAT to create a new topology device. Once created assign the gpu to
1303          * that topology device
1304          */
1305         dev = kfd_assign_gpu(gpu);
1306         if (!dev) {
1307                 res = kfd_create_crat_image_virtual(&crat_image, &image_size,
1308                                                     COMPUTE_UNIT_GPU, gpu,
1309                                                     proximity_domain);
1310                 if (res) {
1311                         pr_err("Error creating VCRAT for GPU (ID: 0x%x)\n",
1312                                gpu_id);
1313                         return res;
1314                 }
1315                 res = kfd_parse_crat_table(crat_image,
1316                                            &temp_topology_device_list,
1317                                            proximity_domain);
1318                 if (res) {
1319                         pr_err("Error parsing VCRAT for GPU (ID: 0x%x)\n",
1320                                gpu_id);
1321                         goto err;
1322                 }
1323
1324                 down_write(&topology_lock);
1325                 kfd_topology_update_device_list(&temp_topology_device_list,
1326                         &topology_device_list);
1327
1328                 /* Update the SYSFS tree, since we added another topology
1329                  * device
1330                  */
1331                 res = kfd_topology_update_sysfs();
1332                 up_write(&topology_lock);
1333
1334                 if (!res)
1335                         sys_props.generation_count++;
1336                 else
1337                         pr_err("Failed to update GPU (ID: 0x%x) to sysfs topology. res=%d\n",
1338                                                 gpu_id, res);
1339                 dev = kfd_assign_gpu(gpu);
1340                 if (WARN_ON(!dev)) {
1341                         res = -ENODEV;
1342                         goto err;
1343                 }
1344         }
1345
1346         dev->gpu_id = gpu_id;
1347         gpu->id = gpu_id;
1348
1349         /* TODO: Move the following lines to function
1350          *      kfd_add_non_crat_information
1351          */
1352
1353         /* Fill-in additional information that is not available in CRAT but
1354          * needed for the topology
1355          */
1356
1357         amdgpu_amdkfd_get_cu_info(dev->gpu->kgd, &cu_info);
1358
1359         strncpy(dev->node_props.name, gpu->device_info->asic_name,
1360                         KFD_TOPOLOGY_PUBLIC_NAME_SIZE);
1361
1362         dev->node_props.simd_arrays_per_engine =
1363                 cu_info.num_shader_arrays_per_engine;
1364
1365         dev->node_props.gfx_target_version = gpu->device_info->gfx_target_version;
1366         dev->node_props.vendor_id = gpu->pdev->vendor;
1367         dev->node_props.device_id = gpu->pdev->device;
1368         dev->node_props.capability |=
1369                 ((amdgpu_amdkfd_get_asic_rev_id(dev->gpu->kgd) <<
1370                         HSA_CAP_ASIC_REVISION_SHIFT) &
1371                         HSA_CAP_ASIC_REVISION_MASK);
1372         dev->node_props.location_id = pci_dev_id(gpu->pdev);
1373         dev->node_props.domain = pci_domain_nr(gpu->pdev->bus);
1374         dev->node_props.max_engine_clk_fcompute =
1375                 amdgpu_amdkfd_get_max_engine_clock_in_mhz(dev->gpu->kgd);
1376         dev->node_props.max_engine_clk_ccompute =
1377                 cpufreq_quick_get_max(0) / 1000;
1378         dev->node_props.drm_render_minor =
1379                 gpu->shared_resources.drm_render_minor;
1380
1381         dev->node_props.hive_id = gpu->hive_id;
1382         dev->node_props.num_sdma_engines = gpu->device_info->num_sdma_engines;
1383         dev->node_props.num_sdma_xgmi_engines =
1384                                 gpu->device_info->num_xgmi_sdma_engines;
1385         dev->node_props.num_sdma_queues_per_engine =
1386                                 gpu->device_info->num_sdma_queues_per_engine;
1387         dev->node_props.num_gws = (dev->gpu->gws &&
1388                 dev->gpu->dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) ?
1389                 amdgpu_amdkfd_get_num_gws(dev->gpu->kgd) : 0;
1390         dev->node_props.num_cp_queues = get_cp_queues_num(dev->gpu->dqm);
1391
1392         kfd_fill_mem_clk_max_info(dev);
1393         kfd_fill_iolink_non_crat_info(dev);
1394
1395         switch (dev->gpu->device_info->asic_family) {
1396         case CHIP_KAVERI:
1397         case CHIP_HAWAII:
1398         case CHIP_TONGA:
1399                 dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_PRE_1_0 <<
1400                         HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) &
1401                         HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK);
1402                 break;
1403         case CHIP_CARRIZO:
1404         case CHIP_FIJI:
1405         case CHIP_POLARIS10:
1406         case CHIP_POLARIS11:
1407         case CHIP_POLARIS12:
1408         case CHIP_VEGAM:
1409                 pr_debug("Adding doorbell packet type capability\n");
1410                 dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_1_0 <<
1411                         HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) &
1412                         HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK);
1413                 break;
1414         case CHIP_VEGA10:
1415         case CHIP_VEGA12:
1416         case CHIP_VEGA20:
1417         case CHIP_RAVEN:
1418         case CHIP_RENOIR:
1419         case CHIP_ARCTURUS:
1420         case CHIP_ALDEBARAN:
1421         case CHIP_NAVI10:
1422         case CHIP_NAVI12:
1423         case CHIP_NAVI14:
1424         case CHIP_SIENNA_CICHLID:
1425         case CHIP_NAVY_FLOUNDER:
1426         case CHIP_VANGOGH:
1427         case CHIP_DIMGREY_CAVEFISH:
1428         case CHIP_BEIGE_GOBY:
1429         case CHIP_YELLOW_CARP:
1430         case CHIP_CYAN_SKILLFISH:
1431                 dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_2_0 <<
1432                         HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) &
1433                         HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK);
1434                 break;
1435         default:
1436                 WARN(1, "Unexpected ASIC family %u",
1437                      dev->gpu->device_info->asic_family);
1438         }
1439
1440         /*
1441         * Overwrite ATS capability according to needs_iommu_device to fix
1442         * potential missing corresponding bit in CRAT of BIOS.
1443         */
1444         if (dev->gpu->use_iommu_v2)
1445                 dev->node_props.capability |= HSA_CAP_ATS_PRESENT;
1446         else
1447                 dev->node_props.capability &= ~HSA_CAP_ATS_PRESENT;
1448
1449         /* Fix errors in CZ CRAT.
1450          * simd_count: Carrizo CRAT reports wrong simd_count, probably
1451          *              because it doesn't consider masked out CUs
1452          * max_waves_per_simd: Carrizo reports wrong max_waves_per_simd
1453          */
1454         if (dev->gpu->device_info->asic_family == CHIP_CARRIZO) {
1455                 dev->node_props.simd_count =
1456                         cu_info.simd_per_cu * cu_info.cu_active_number;
1457                 dev->node_props.max_waves_per_simd = 10;
1458         }
1459
1460         adev = (struct amdgpu_device *)(dev->gpu->kgd);
1461         /* kfd only concerns sram ecc on GFX and HBM ecc on UMC */
1462         dev->node_props.capability |=
1463                 ((adev->ras_enabled & BIT(AMDGPU_RAS_BLOCK__GFX)) != 0) ?
1464                 HSA_CAP_SRAM_EDCSUPPORTED : 0;
1465         dev->node_props.capability |= ((adev->ras_enabled & BIT(AMDGPU_RAS_BLOCK__UMC)) != 0) ?
1466                 HSA_CAP_MEM_EDCSUPPORTED : 0;
1467
1468         if (adev->asic_type != CHIP_VEGA10)
1469                 dev->node_props.capability |= (adev->ras_enabled != 0) ?
1470                         HSA_CAP_RASEVENTNOTIFY : 0;
1471
1472         if (KFD_IS_SVM_API_SUPPORTED(adev->kfd.dev))
1473                 dev->node_props.capability |= HSA_CAP_SVMAPI_SUPPORTED;
1474
1475         kfd_debug_print_topology();
1476
1477         if (!res)
1478                 kfd_notify_gpu_change(gpu_id, 1);
1479 err:
1480         kfd_destroy_crat_image(crat_image);
1481         return res;
1482 }
1483
1484 int kfd_topology_remove_device(struct kfd_dev *gpu)
1485 {
1486         struct kfd_topology_device *dev, *tmp;
1487         uint32_t gpu_id;
1488         int res = -ENODEV;
1489
1490         down_write(&topology_lock);
1491
1492         list_for_each_entry_safe(dev, tmp, &topology_device_list, list)
1493                 if (dev->gpu == gpu) {
1494                         gpu_id = dev->gpu_id;
1495                         kfd_remove_sysfs_node_entry(dev);
1496                         kfd_release_topology_device(dev);
1497                         sys_props.num_devices--;
1498                         res = 0;
1499                         if (kfd_topology_update_sysfs() < 0)
1500                                 kfd_topology_release_sysfs();
1501                         break;
1502                 }
1503
1504         up_write(&topology_lock);
1505
1506         if (!res)
1507                 kfd_notify_gpu_change(gpu_id, 0);
1508
1509         return res;
1510 }
1511
1512 /* kfd_topology_enum_kfd_devices - Enumerate through all devices in KFD
1513  *      topology. If GPU device is found @idx, then valid kfd_dev pointer is
1514  *      returned through @kdev
1515  * Return -     0: On success (@kdev will be NULL for non GPU nodes)
1516  *              -1: If end of list
1517  */
1518 int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_dev **kdev)
1519 {
1520
1521         struct kfd_topology_device *top_dev;
1522         uint8_t device_idx = 0;
1523
1524         *kdev = NULL;
1525         down_read(&topology_lock);
1526
1527         list_for_each_entry(top_dev, &topology_device_list, list) {
1528                 if (device_idx == idx) {
1529                         *kdev = top_dev->gpu;
1530                         up_read(&topology_lock);
1531                         return 0;
1532                 }
1533
1534                 device_idx++;
1535         }
1536
1537         up_read(&topology_lock);
1538
1539         return -1;
1540
1541 }
1542
1543 static int kfd_cpumask_to_apic_id(const struct cpumask *cpumask)
1544 {
1545         int first_cpu_of_numa_node;
1546
1547         if (!cpumask || cpumask == cpu_none_mask)
1548                 return -1;
1549         first_cpu_of_numa_node = cpumask_first(cpumask);
1550         if (first_cpu_of_numa_node >= nr_cpu_ids)
1551                 return -1;
1552 #ifdef CONFIG_X86_64
1553         return cpu_data(first_cpu_of_numa_node).apicid;
1554 #else
1555         return first_cpu_of_numa_node;
1556 #endif
1557 }
1558
1559 /* kfd_numa_node_to_apic_id - Returns the APIC ID of the first logical processor
1560  *      of the given NUMA node (numa_node_id)
1561  * Return -1 on failure
1562  */
1563 int kfd_numa_node_to_apic_id(int numa_node_id)
1564 {
1565         if (numa_node_id == -1) {
1566                 pr_warn("Invalid NUMA Node. Use online CPU mask\n");
1567                 return kfd_cpumask_to_apic_id(cpu_online_mask);
1568         }
1569         return kfd_cpumask_to_apic_id(cpumask_of_node(numa_node_id));
1570 }
1571
1572 void kfd_double_confirm_iommu_support(struct kfd_dev *gpu)
1573 {
1574         struct kfd_topology_device *dev;
1575
1576         gpu->use_iommu_v2 = false;
1577
1578         if (!gpu->device_info->needs_iommu_device)
1579                 return;
1580
1581         down_read(&topology_lock);
1582
1583         /* Only use IOMMUv2 if there is an APU topology node with no GPU
1584          * assigned yet. This GPU will be assigned to it.
1585          */
1586         list_for_each_entry(dev, &topology_device_list, list)
1587                 if (dev->node_props.cpu_cores_count &&
1588                     dev->node_props.simd_count &&
1589                     !dev->gpu)
1590                         gpu->use_iommu_v2 = true;
1591
1592         up_read(&topology_lock);
1593 }
1594
1595 #if defined(CONFIG_DEBUG_FS)
1596
1597 int kfd_debugfs_hqds_by_device(struct seq_file *m, void *data)
1598 {
1599         struct kfd_topology_device *dev;
1600         unsigned int i = 0;
1601         int r = 0;
1602
1603         down_read(&topology_lock);
1604
1605         list_for_each_entry(dev, &topology_device_list, list) {
1606                 if (!dev->gpu) {
1607                         i++;
1608                         continue;
1609                 }
1610
1611                 seq_printf(m, "Node %u, gpu_id %x:\n", i++, dev->gpu->id);
1612                 r = dqm_debugfs_hqds(m, dev->gpu->dqm);
1613                 if (r)
1614                         break;
1615         }
1616
1617         up_read(&topology_lock);
1618
1619         return r;
1620 }
1621
1622 int kfd_debugfs_rls_by_device(struct seq_file *m, void *data)
1623 {
1624         struct kfd_topology_device *dev;
1625         unsigned int i = 0;
1626         int r = 0;
1627
1628         down_read(&topology_lock);
1629
1630         list_for_each_entry(dev, &topology_device_list, list) {
1631                 if (!dev->gpu) {
1632                         i++;
1633                         continue;
1634                 }
1635
1636                 seq_printf(m, "Node %u, gpu_id %x:\n", i++, dev->gpu->id);
1637                 r = pm_debugfs_runlist(m, &dev->gpu->dqm->packet_mgr);
1638                 if (r)
1639                         break;
1640         }
1641
1642         up_read(&topology_lock);
1643
1644         return r;
1645 }
1646
1647 #endif