drivers/gpu/drm/amd/amdkfd/kfd_topology.c

   1 /*
   2  * Copyright 2014 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22
  23 #include <linux/types.h>
  24 #include <linux/kernel.h>
  25 #include <linux/pci.h>
  26 #include <linux/errno.h>
  27 #include <linux/acpi.h>
  28 #include <linux/hash.h>
  29 #include <linux/cpufreq.h>
  30 #include <linux/log2.h>
  31 #include <linux/dmi.h>
  32 #include <linux/atomic.h>
  33
  34 #include "kfd_priv.h"
  35 #include "kfd_crat.h"
  36 #include "kfd_topology.h"
  37 #include "kfd_device_queue_manager.h"
  38 #include "kfd_iommu.h"
  39 #include "amdgpu_amdkfd.h"
  40 #include "amdgpu_ras.h"
  41
  42 /* topology_device_list - Master list of all topology devices */
  43 static struct list_head topology_device_list;
  44 static struct kfd_system_properties sys_props;
  45
  46 static DECLARE_RWSEM(topology_lock);
  47 static atomic_t topology_crat_proximity_domain;
  48
  49 struct kfd_topology_device *kfd_topology_device_by_proximity_domain(
  50                                                 uint32_t proximity_domain)
  51 {
  52         struct kfd_topology_device *top_dev;
  53         struct kfd_topology_device *device = NULL;
  54
  55         down_read(&topology_lock);
  56
  57         list_for_each_entry(top_dev, &topology_device_list, list)
  58                 if (top_dev->proximity_domain == proximity_domain) {
  59                         device = top_dev;
  60                         break;
  61                 }
  62
  63         up_read(&topology_lock);
  64
  65         return device;
  66 }
  67
  68 struct kfd_topology_device *kfd_topology_device_by_id(uint32_t gpu_id)
  69 {
  70         struct kfd_topology_device *top_dev = NULL;
  71         struct kfd_topology_device *ret = NULL;
  72
  73         down_read(&topology_lock);
  74
  75         list_for_each_entry(top_dev, &topology_device_list, list)
  76                 if (top_dev->gpu_id == gpu_id) {
  77                         ret = top_dev;
  78                         break;
  79                 }
  80
  81         up_read(&topology_lock);
  82
  83         return ret;
  84 }
  85
  86 struct kfd_dev *kfd_device_by_id(uint32_t gpu_id)
  87 {
  88         struct kfd_topology_device *top_dev;
  89
  90         top_dev = kfd_topology_device_by_id(gpu_id);
  91         if (!top_dev)
  92                 return NULL;
  93
  94         return top_dev->gpu;
  95 }
  96
  97 struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev)
  98 {
  99         struct kfd_topology_device *top_dev;
 100         struct kfd_dev *device = NULL;
 101
 102         down_read(&topology_lock);
 103
 104         list_for_each_entry(top_dev, &topology_device_list, list)
 105                 if (top_dev->gpu && top_dev->gpu->pdev == pdev) {
 106                         device = top_dev->gpu;
 107                         break;
 108                 }
 109
 110         up_read(&topology_lock);
 111
 112         return device;
 113 }
 114
 115 struct kfd_dev *kfd_device_by_kgd(const struct kgd_dev *kgd)
 116 {
 117         struct kfd_topology_device *top_dev;
 118         struct kfd_dev *device = NULL;
 119
 120         down_read(&topology_lock);
 121
 122         list_for_each_entry(top_dev, &topology_device_list, list)
 123                 if (top_dev->gpu && top_dev->gpu->kgd == kgd) {
 124                         device = top_dev->gpu;
 125                         break;
 126                 }
 127
 128         up_read(&topology_lock);
 129
 130         return device;
 131 }
 132
 133 /* Called with write topology_lock acquired */
 134 static void kfd_release_topology_device(struct kfd_topology_device *dev)
 135 {
 136         struct kfd_mem_properties *mem;
 137         struct kfd_cache_properties *cache;
 138         struct kfd_iolink_properties *iolink;
 139         struct kfd_perf_properties *perf;
 140
 141         list_del(&dev->list);
 142
 143         while (dev->mem_props.next != &dev->mem_props) {
 144                 mem = container_of(dev->mem_props.next,
 145                                 struct kfd_mem_properties, list);
 146                 list_del(&mem->list);
 147                 kfree(mem);
 148         }
 149
 150         while (dev->cache_props.next != &dev->cache_props) {
 151                 cache = container_of(dev->cache_props.next,
 152                                 struct kfd_cache_properties, list);
 153                 list_del(&cache->list);
 154                 kfree(cache);
 155         }
 156
 157         while (dev->io_link_props.next != &dev->io_link_props) {
 158                 iolink = container_of(dev->io_link_props.next,
 159                                 struct kfd_iolink_properties, list);
 160                 list_del(&iolink->list);
 161                 kfree(iolink);
 162         }
 163
 164         while (dev->perf_props.next != &dev->perf_props) {
 165                 perf = container_of(dev->perf_props.next,
 166                                 struct kfd_perf_properties, list);
 167                 list_del(&perf->list);
 168                 kfree(perf);
 169         }
 170
 171         kfree(dev);
 172 }
 173
 174 void kfd_release_topology_device_list(struct list_head *device_list)
 175 {
 176         struct kfd_topology_device *dev;
 177
 178         while (!list_empty(device_list)) {
 179                 dev = list_first_entry(device_list,
 180                                        struct kfd_topology_device, list);
 181                 kfd_release_topology_device(dev);
 182         }
 183 }
 184
 185 static void kfd_release_live_view(void)
 186 {
 187         kfd_release_topology_device_list(&topology_device_list);
 188         memset(&sys_props, 0, sizeof(sys_props));
 189 }
 190
 191 struct kfd_topology_device *kfd_create_topology_device(
 192                                 struct list_head *device_list)
 193 {
 194         struct kfd_topology_device *dev;
 195
 196         dev = kfd_alloc_struct(dev);
 197         if (!dev) {
 198                 pr_err("No memory to allocate a topology device");
 199                 return NULL;
 200         }
 201
 202         INIT_LIST_HEAD(&dev->mem_props);
 203         INIT_LIST_HEAD(&dev->cache_props);
 204         INIT_LIST_HEAD(&dev->io_link_props);
 205         INIT_LIST_HEAD(&dev->perf_props);
 206
 207         list_add_tail(&dev->list, device_list);
 208
 209         return dev;
 210 }
 211
 212
 213 #define sysfs_show_gen_prop(buffer, offs, fmt, ...)             \
 214                 (offs += snprintf(buffer+offs, PAGE_SIZE-offs,  \
 215                                   fmt, __VA_ARGS__))
 216 #define sysfs_show_32bit_prop(buffer, offs, name, value) \
 217                 sysfs_show_gen_prop(buffer, offs, "%s %u\n", name, value)
 218 #define sysfs_show_64bit_prop(buffer, offs, name, value) \
 219                 sysfs_show_gen_prop(buffer, offs, "%s %llu\n", name, value)
 220 #define sysfs_show_32bit_val(buffer, offs, value) \
 221                 sysfs_show_gen_prop(buffer, offs, "%u\n", value)
 222 #define sysfs_show_str_val(buffer, offs, value) \
 223                 sysfs_show_gen_prop(buffer, offs, "%s\n", value)
 224
 225 static ssize_t sysprops_show(struct kobject *kobj, struct attribute *attr,
 226                 char *buffer)
 227 {
 228         int offs = 0;
 229
 230         /* Making sure that the buffer is an empty string */
 231         buffer[0] = 0;
 232
 233         if (attr == &sys_props.attr_genid) {
 234                 sysfs_show_32bit_val(buffer, offs,
 235                                      sys_props.generation_count);
 236         } else if (attr == &sys_props.attr_props) {
 237                 sysfs_show_64bit_prop(buffer, offs, "platform_oem",
 238                                       sys_props.platform_oem);
 239                 sysfs_show_64bit_prop(buffer, offs, "platform_id",
 240                                       sys_props.platform_id);
 241                 sysfs_show_64bit_prop(buffer, offs, "platform_rev",
 242                                       sys_props.platform_rev);
 243         } else {
 244                 offs = -EINVAL;
 245         }
 246
 247         return offs;
 248 }
 249
 250 static void kfd_topology_kobj_release(struct kobject *kobj)
 251 {
 252         kfree(kobj);
 253 }
 254
 255 static const struct sysfs_ops sysprops_ops = {
 256         .show = sysprops_show,
 257 };
 258
 259 static struct kobj_type sysprops_type = {
 260         .release = kfd_topology_kobj_release,
 261         .sysfs_ops = &sysprops_ops,
 262 };
 263
 264 static ssize_t iolink_show(struct kobject *kobj, struct attribute *attr,
 265                 char *buffer)
 266 {
 267         int offs = 0;
 268         struct kfd_iolink_properties *iolink;
 269
 270         /* Making sure that the buffer is an empty string */
 271         buffer[0] = 0;
 272
 273         iolink = container_of(attr, struct kfd_iolink_properties, attr);
 274         if (iolink->gpu && kfd_devcgroup_check_permission(iolink->gpu))
 275                 return -EPERM;
 276         sysfs_show_32bit_prop(buffer, offs, "type", iolink->iolink_type);
 277         sysfs_show_32bit_prop(buffer, offs, "version_major", iolink->ver_maj);
 278         sysfs_show_32bit_prop(buffer, offs, "version_minor", iolink->ver_min);
 279         sysfs_show_32bit_prop(buffer, offs, "node_from", iolink->node_from);
 280         sysfs_show_32bit_prop(buffer, offs, "node_to", iolink->node_to);
 281         sysfs_show_32bit_prop(buffer, offs, "weight", iolink->weight);
 282         sysfs_show_32bit_prop(buffer, offs, "min_latency", iolink->min_latency);
 283         sysfs_show_32bit_prop(buffer, offs, "max_latency", iolink->max_latency);
 284         sysfs_show_32bit_prop(buffer, offs, "min_bandwidth",
 285                               iolink->min_bandwidth);
 286         sysfs_show_32bit_prop(buffer, offs, "max_bandwidth",
 287                               iolink->max_bandwidth);
 288         sysfs_show_32bit_prop(buffer, offs, "recommended_transfer_size",
 289                               iolink->rec_transfer_size);
 290         sysfs_show_32bit_prop(buffer, offs, "flags", iolink->flags);
 291
 292         return offs;
 293 }
 294
 295 static const struct sysfs_ops iolink_ops = {
 296         .show = iolink_show,
 297 };
 298
 299 static struct kobj_type iolink_type = {
 300         .release = kfd_topology_kobj_release,
 301         .sysfs_ops = &iolink_ops,
 302 };
 303
 304 static ssize_t mem_show(struct kobject *kobj, struct attribute *attr,
 305                 char *buffer)
 306 {
 307         int offs = 0;
 308         struct kfd_mem_properties *mem;
 309
 310         /* Making sure that the buffer is an empty string */
 311         buffer[0] = 0;
 312
 313         mem = container_of(attr, struct kfd_mem_properties, attr);
 314         if (mem->gpu && kfd_devcgroup_check_permission(mem->gpu))
 315                 return -EPERM;
 316         sysfs_show_32bit_prop(buffer, offs, "heap_type", mem->heap_type);
 317         sysfs_show_64bit_prop(buffer, offs, "size_in_bytes",
 318                               mem->size_in_bytes);
 319         sysfs_show_32bit_prop(buffer, offs, "flags", mem->flags);
 320         sysfs_show_32bit_prop(buffer, offs, "width", mem->width);
 321         sysfs_show_32bit_prop(buffer, offs, "mem_clk_max",
 322                               mem->mem_clk_max);
 323
 324         return offs;
 325 }
 326
 327 static const struct sysfs_ops mem_ops = {
 328         .show = mem_show,
 329 };
 330
 331 static struct kobj_type mem_type = {
 332         .release = kfd_topology_kobj_release,
 333         .sysfs_ops = &mem_ops,
 334 };
 335
 336 static ssize_t kfd_cache_show(struct kobject *kobj, struct attribute *attr,
 337                 char *buffer)
 338 {
 339         int offs = 0;
 340         uint32_t i, j;
 341         struct kfd_cache_properties *cache;
 342
 343         /* Making sure that the buffer is an empty string */
 344         buffer[0] = 0;
 345
 346         cache = container_of(attr, struct kfd_cache_properties, attr);
 347         if (cache->gpu && kfd_devcgroup_check_permission(cache->gpu))
 348                 return -EPERM;
 349         sysfs_show_32bit_prop(buffer, offs, "processor_id_low",
 350                         cache->processor_id_low);
 351         sysfs_show_32bit_prop(buffer, offs, "level", cache->cache_level);
 352         sysfs_show_32bit_prop(buffer, offs, "size", cache->cache_size);
 353         sysfs_show_32bit_prop(buffer, offs, "cache_line_size",
 354                               cache->cacheline_size);
 355         sysfs_show_32bit_prop(buffer, offs, "cache_lines_per_tag",
 356                               cache->cachelines_per_tag);
 357         sysfs_show_32bit_prop(buffer, offs, "association", cache->cache_assoc);
 358         sysfs_show_32bit_prop(buffer, offs, "latency", cache->cache_latency);
 359         sysfs_show_32bit_prop(buffer, offs, "type", cache->cache_type);
 360         offs += snprintf(buffer+offs, PAGE_SIZE-offs, "sibling_map ");
 361         for (i = 0; i < CRAT_SIBLINGMAP_SIZE; i++)
 362                 for (j = 0; j < sizeof(cache->sibling_map[0])*8; j++)
 363                         /* Check each bit */
 364                         offs += snprintf(buffer+offs, PAGE_SIZE-offs, "%d,",
 365                                          (cache->sibling_map[i] >> j) & 1);
 366
 367         /* Replace the last "," with end of line */
 368         buffer[offs-1] = '\n';
 369         return offs;
 370 }
 371
 372 static const struct sysfs_ops cache_ops = {
 373         .show = kfd_cache_show,
 374 };
 375
 376 static struct kobj_type cache_type = {
 377         .release = kfd_topology_kobj_release,
 378         .sysfs_ops = &cache_ops,
 379 };
 380
 381 /****** Sysfs of Performance Counters ******/
 382
 383 struct kfd_perf_attr {
 384         struct kobj_attribute attr;
 385         uint32_t data;
 386 };
 387
 388 static ssize_t perf_show(struct kobject *kobj, struct kobj_attribute *attrs,
 389                         char *buf)
 390 {
 391         int offs = 0;
 392         struct kfd_perf_attr *attr;
 393
 394         buf[0] = 0;
 395         attr = container_of(attrs, struct kfd_perf_attr, attr);
 396         if (!attr->data) /* invalid data for PMC */
 397                 return 0;
 398         else
 399                 return sysfs_show_32bit_val(buf, offs, attr->data);
 400 }
 401
 402 #define KFD_PERF_DESC(_name, _data)                     \
 403 {                                                       \
 404         .attr  = __ATTR(_name, 0444, perf_show, NULL),  \
 405         .data = _data,                                  \
 406 }
 407
 408 static struct kfd_perf_attr perf_attr_iommu[] = {
 409         KFD_PERF_DESC(max_concurrent, 0),
 410         KFD_PERF_DESC(num_counters, 0),
 411         KFD_PERF_DESC(counter_ids, 0),
 412 };
 413 /****************************************/
 414
 415 static ssize_t node_show(struct kobject *kobj, struct attribute *attr,
 416                 char *buffer)
 417 {
 418         int offs = 0;
 419         struct kfd_topology_device *dev;
 420         uint32_t log_max_watch_addr;
 421
 422         /* Making sure that the buffer is an empty string */
 423         buffer[0] = 0;
 424
 425         if (strcmp(attr->name, "gpu_id") == 0) {
 426                 dev = container_of(attr, struct kfd_topology_device,
 427                                 attr_gpuid);
 428                 if (dev->gpu && kfd_devcgroup_check_permission(dev->gpu))
 429                         return -EPERM;
 430                 return sysfs_show_32bit_val(buffer, offs, dev->gpu_id);
 431         }
 432
 433         if (strcmp(attr->name, "name") == 0) {
 434                 dev = container_of(attr, struct kfd_topology_device,
 435                                 attr_name);
 436
 437                 if (dev->gpu && kfd_devcgroup_check_permission(dev->gpu))
 438                         return -EPERM;
 439                 return sysfs_show_str_val(buffer, offs, dev->node_props.name);
 440         }
 441
 442         dev = container_of(attr, struct kfd_topology_device,
 443                         attr_props);
 444         if (dev->gpu && kfd_devcgroup_check_permission(dev->gpu))
 445                 return -EPERM;
 446         sysfs_show_32bit_prop(buffer, offs, "cpu_cores_count",
 447                               dev->node_props.cpu_cores_count);
 448         sysfs_show_32bit_prop(buffer, offs, "simd_count",
 449                               dev->gpu ? dev->node_props.simd_count : 0);
 450         sysfs_show_32bit_prop(buffer, offs, "mem_banks_count",
 451                               dev->node_props.mem_banks_count);
 452         sysfs_show_32bit_prop(buffer, offs, "caches_count",
 453                               dev->node_props.caches_count);
 454         sysfs_show_32bit_prop(buffer, offs, "io_links_count",
 455                               dev->node_props.io_links_count);
 456         sysfs_show_32bit_prop(buffer, offs, "cpu_core_id_base",
 457                               dev->node_props.cpu_core_id_base);
 458         sysfs_show_32bit_prop(buffer, offs, "simd_id_base",
 459                               dev->node_props.simd_id_base);
 460         sysfs_show_32bit_prop(buffer, offs, "max_waves_per_simd",
 461                               dev->node_props.max_waves_per_simd);
 462         sysfs_show_32bit_prop(buffer, offs, "lds_size_in_kb",
 463                               dev->node_props.lds_size_in_kb);
 464         sysfs_show_32bit_prop(buffer, offs, "gds_size_in_kb",
 465                               dev->node_props.gds_size_in_kb);
 466         sysfs_show_32bit_prop(buffer, offs, "num_gws",
 467                               dev->node_props.num_gws);
 468         sysfs_show_32bit_prop(buffer, offs, "wave_front_size",
 469                               dev->node_props.wave_front_size);
 470         sysfs_show_32bit_prop(buffer, offs, "array_count",
 471                               dev->node_props.array_count);
 472         sysfs_show_32bit_prop(buffer, offs, "simd_arrays_per_engine",
 473                               dev->node_props.simd_arrays_per_engine);
 474         sysfs_show_32bit_prop(buffer, offs, "cu_per_simd_array",
 475                               dev->node_props.cu_per_simd_array);
 476         sysfs_show_32bit_prop(buffer, offs, "simd_per_cu",
 477                               dev->node_props.simd_per_cu);
 478         sysfs_show_32bit_prop(buffer, offs, "max_slots_scratch_cu",
 479                               dev->node_props.max_slots_scratch_cu);
 480         sysfs_show_32bit_prop(buffer, offs, "vendor_id",
 481                               dev->node_props.vendor_id);
 482         sysfs_show_32bit_prop(buffer, offs, "device_id",
 483                               dev->node_props.device_id);
 484         sysfs_show_32bit_prop(buffer, offs, "location_id",
 485                               dev->node_props.location_id);
 486         sysfs_show_32bit_prop(buffer, offs, "domain",
 487                               dev->node_props.domain);
 488         sysfs_show_32bit_prop(buffer, offs, "drm_render_minor",
 489                               dev->node_props.drm_render_minor);
 490         sysfs_show_64bit_prop(buffer, offs, "hive_id",
 491                               dev->node_props.hive_id);
 492         sysfs_show_32bit_prop(buffer, offs, "num_sdma_engines",
 493                               dev->node_props.num_sdma_engines);
 494         sysfs_show_32bit_prop(buffer, offs, "num_sdma_xgmi_engines",
 495                               dev->node_props.num_sdma_xgmi_engines);
 496         sysfs_show_32bit_prop(buffer, offs, "num_sdma_queues_per_engine",
 497                               dev->node_props.num_sdma_queues_per_engine);
 498         sysfs_show_32bit_prop(buffer, offs, "num_cp_queues",
 499                               dev->node_props.num_cp_queues);
 500         sysfs_show_64bit_prop(buffer, offs, "unique_id",
 501                               dev->node_props.unique_id);
 502
 503         if (dev->gpu) {
 504                 log_max_watch_addr =
 505                         __ilog2_u32(dev->gpu->device_info->num_of_watch_points);
 506
 507                 if (log_max_watch_addr) {
 508                         dev->node_props.capability |=
 509                                         HSA_CAP_WATCH_POINTS_SUPPORTED;
 510
 511                         dev->node_props.capability |=
 512                                 ((log_max_watch_addr <<
 513                                         HSA_CAP_WATCH_POINTS_TOTALBITS_SHIFT) &
 514                                 HSA_CAP_WATCH_POINTS_TOTALBITS_MASK);
 515                 }
 516
 517                 if (dev->gpu->device_info->asic_family == CHIP_TONGA)
 518                         dev->node_props.capability |=
 519                                         HSA_CAP_AQL_QUEUE_DOUBLE_MAP;
 520
 521                 sysfs_show_32bit_prop(buffer, offs, "max_engine_clk_fcompute",
 522                         dev->node_props.max_engine_clk_fcompute);
 523
 524                 sysfs_show_64bit_prop(buffer, offs, "local_mem_size", 0ULL);
 525
 526                 sysfs_show_32bit_prop(buffer, offs, "fw_version",
 527                                       dev->gpu->mec_fw_version);
 528                 sysfs_show_32bit_prop(buffer, offs, "capability",
 529                                       dev->node_props.capability);
 530                 sysfs_show_32bit_prop(buffer, offs, "sdma_fw_version",
 531                                       dev->gpu->sdma_fw_version);
 532         }
 533
 534         return sysfs_show_32bit_prop(buffer, offs, "max_engine_clk_ccompute",
 535                                      cpufreq_quick_get_max(0)/1000);
 536 }
 537
 538 static const struct sysfs_ops node_ops = {
 539         .show = node_show,
 540 };
 541
 542 static struct kobj_type node_type = {
 543         .release = kfd_topology_kobj_release,
 544         .sysfs_ops = &node_ops,
 545 };
 546
 547 static void kfd_remove_sysfs_file(struct kobject *kobj, struct attribute *attr)
 548 {
 549         sysfs_remove_file(kobj, attr);
 550         kobject_del(kobj);
 551         kobject_put(kobj);
 552 }
 553
 554 static void kfd_remove_sysfs_node_entry(struct kfd_topology_device *dev)
 555 {
 556         struct kfd_iolink_properties *iolink;
 557         struct kfd_cache_properties *cache;
 558         struct kfd_mem_properties *mem;
 559         struct kfd_perf_properties *perf;
 560
 561         if (dev->kobj_iolink) {
 562                 list_for_each_entry(iolink, &dev->io_link_props, list)
 563                         if (iolink->kobj) {
 564                                 kfd_remove_sysfs_file(iolink->kobj,
 565                                                         &iolink->attr);
 566                                 iolink->kobj = NULL;
 567                         }
 568                 kobject_del(dev->kobj_iolink);
 569                 kobject_put(dev->kobj_iolink);
 570                 dev->kobj_iolink = NULL;
 571         }
 572
 573         if (dev->kobj_cache) {
 574                 list_for_each_entry(cache, &dev->cache_props, list)
 575                         if (cache->kobj) {
 576                                 kfd_remove_sysfs_file(cache->kobj,
 577                                                         &cache->attr);
 578                                 cache->kobj = NULL;
 579                         }
 580                 kobject_del(dev->kobj_cache);
 581                 kobject_put(dev->kobj_cache);
 582                 dev->kobj_cache = NULL;
 583         }
 584
 585         if (dev->kobj_mem) {
 586                 list_for_each_entry(mem, &dev->mem_props, list)
 587                         if (mem->kobj) {
 588                                 kfd_remove_sysfs_file(mem->kobj, &mem->attr);
 589                                 mem->kobj = NULL;
 590                         }
 591                 kobject_del(dev->kobj_mem);
 592                 kobject_put(dev->kobj_mem);
 593                 dev->kobj_mem = NULL;
 594         }
 595
 596         if (dev->kobj_perf) {
 597                 list_for_each_entry(perf, &dev->perf_props, list) {
 598                         kfree(perf->attr_group);
 599                         perf->attr_group = NULL;
 600                 }
 601                 kobject_del(dev->kobj_perf);
 602                 kobject_put(dev->kobj_perf);
 603                 dev->kobj_perf = NULL;
 604         }
 605
 606         if (dev->kobj_node) {
 607                 sysfs_remove_file(dev->kobj_node, &dev->attr_gpuid);
 608                 sysfs_remove_file(dev->kobj_node, &dev->attr_name);
 609                 sysfs_remove_file(dev->kobj_node, &dev->attr_props);
 610                 kobject_del(dev->kobj_node);
 611                 kobject_put(dev->kobj_node);
 612                 dev->kobj_node = NULL;
 613         }
 614 }
 615
 616 static int kfd_build_sysfs_node_entry(struct kfd_topology_device *dev,
 617                 uint32_t id)
 618 {
 619         struct kfd_iolink_properties *iolink;
 620         struct kfd_cache_properties *cache;
 621         struct kfd_mem_properties *mem;
 622         struct kfd_perf_properties *perf;
 623         int ret;
 624         uint32_t i, num_attrs;
 625         struct attribute **attrs;
 626
 627         if (WARN_ON(dev->kobj_node))
 628                 return -EEXIST;
 629
 630         /*
 631          * Creating the sysfs folders
 632          */
 633         dev->kobj_node = kfd_alloc_struct(dev->kobj_node);
 634         if (!dev->kobj_node)
 635                 return -ENOMEM;
 636
 637         ret = kobject_init_and_add(dev->kobj_node, &node_type,
 638                         sys_props.kobj_nodes, "%d", id);
 639         if (ret < 0) {
 640                 kobject_put(dev->kobj_node);
 641                 return ret;
 642         }
 643
 644         dev->kobj_mem = kobject_create_and_add("mem_banks", dev->kobj_node);
 645         if (!dev->kobj_mem)
 646                 return -ENOMEM;
 647
 648         dev->kobj_cache = kobject_create_and_add("caches", dev->kobj_node);
 649         if (!dev->kobj_cache)
 650                 return -ENOMEM;
 651
 652         dev->kobj_iolink = kobject_create_and_add("io_links", dev->kobj_node);
 653         if (!dev->kobj_iolink)
 654                 return -ENOMEM;
 655
 656         dev->kobj_perf = kobject_create_and_add("perf", dev->kobj_node);
 657         if (!dev->kobj_perf)
 658                 return -ENOMEM;
 659
 660         /*
 661          * Creating sysfs files for node properties
 662          */
 663         dev->attr_gpuid.name = "gpu_id";
 664         dev->attr_gpuid.mode = KFD_SYSFS_FILE_MODE;
 665         sysfs_attr_init(&dev->attr_gpuid);
 666         dev->attr_name.name = "name";
 667         dev->attr_name.mode = KFD_SYSFS_FILE_MODE;
 668         sysfs_attr_init(&dev->attr_name);
 669         dev->attr_props.name = "properties";
 670         dev->attr_props.mode = KFD_SYSFS_FILE_MODE;
 671         sysfs_attr_init(&dev->attr_props);
 672         ret = sysfs_create_file(dev->kobj_node, &dev->attr_gpuid);
 673         if (ret < 0)
 674                 return ret;
 675         ret = sysfs_create_file(dev->kobj_node, &dev->attr_name);
 676         if (ret < 0)
 677                 return ret;
 678         ret = sysfs_create_file(dev->kobj_node, &dev->attr_props);
 679         if (ret < 0)
 680                 return ret;
 681
 682         i = 0;
 683         list_for_each_entry(mem, &dev->mem_props, list) {
 684                 mem->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL);
 685                 if (!mem->kobj)
 686                         return -ENOMEM;
 687                 ret = kobject_init_and_add(mem->kobj, &mem_type,
 688                                 dev->kobj_mem, "%d", i);
 689                 if (ret < 0) {
 690                         kobject_put(mem->kobj);
 691                         return ret;
 692                 }
 693
 694                 mem->attr.name = "properties";
 695                 mem->attr.mode = KFD_SYSFS_FILE_MODE;
 696                 sysfs_attr_init(&mem->attr);
 697                 ret = sysfs_create_file(mem->kobj, &mem->attr);
 698                 if (ret < 0)
 699                         return ret;
 700                 i++;
 701         }
 702
 703         i = 0;
 704         list_for_each_entry(cache, &dev->cache_props, list) {
 705                 cache->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL);
 706                 if (!cache->kobj)
 707                         return -ENOMEM;
 708                 ret = kobject_init_and_add(cache->kobj, &cache_type,
 709                                 dev->kobj_cache, "%d", i);
 710                 if (ret < 0) {
 711                         kobject_put(cache->kobj);
 712                         return ret;
 713                 }
 714
 715                 cache->attr.name = "properties";
 716                 cache->attr.mode = KFD_SYSFS_FILE_MODE;
 717                 sysfs_attr_init(&cache->attr);
 718                 ret = sysfs_create_file(cache->kobj, &cache->attr);
 719                 if (ret < 0)
 720                         return ret;
 721                 i++;
 722         }
 723
 724         i = 0;
 725         list_for_each_entry(iolink, &dev->io_link_props, list) {
 726                 iolink->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL);
 727                 if (!iolink->kobj)
 728                         return -ENOMEM;
 729                 ret = kobject_init_and_add(iolink->kobj, &iolink_type,
 730                                 dev->kobj_iolink, "%d", i);
 731                 if (ret < 0) {
 732                         kobject_put(iolink->kobj);
 733                         return ret;
 734                 }
 735
 736                 iolink->attr.name = "properties";
 737                 iolink->attr.mode = KFD_SYSFS_FILE_MODE;
 738                 sysfs_attr_init(&iolink->attr);
 739                 ret = sysfs_create_file(iolink->kobj, &iolink->attr);
 740                 if (ret < 0)
 741                         return ret;
 742                 i++;
 743         }
 744
 745         /* All hardware blocks have the same number of attributes. */
 746         num_attrs = ARRAY_SIZE(perf_attr_iommu);
 747         list_for_each_entry(perf, &dev->perf_props, list) {
 748                 perf->attr_group = kzalloc(sizeof(struct kfd_perf_attr)
 749                         * num_attrs + sizeof(struct attribute_group),
 750                         GFP_KERNEL);
 751                 if (!perf->attr_group)
 752                         return -ENOMEM;
 753
 754                 attrs = (struct attribute **)(perf->attr_group + 1);
 755                 if (!strcmp(perf->block_name, "iommu")) {
 756                 /* Information of IOMMU's num_counters and counter_ids is shown
 757                  * under /sys/bus/event_source/devices/amd_iommu. We don't
 758                  * duplicate here.
 759                  */
 760                         perf_attr_iommu[0].data = perf->max_concurrent;
 761                         for (i = 0; i < num_attrs; i++)
 762                                 attrs[i] = &perf_attr_iommu[i].attr.attr;
 763                 }
 764                 perf->attr_group->name = perf->block_name;
 765                 perf->attr_group->attrs = attrs;
 766                 ret = sysfs_create_group(dev->kobj_perf, perf->attr_group);
 767                 if (ret < 0)
 768                         return ret;
 769         }
 770
 771         return 0;
 772 }
 773
 774 /* Called with write topology lock acquired */
 775 static int kfd_build_sysfs_node_tree(void)
 776 {
 777         struct kfd_topology_device *dev;
 778         int ret;
 779         uint32_t i = 0;
 780
 781         list_for_each_entry(dev, &topology_device_list, list) {
 782                 ret = kfd_build_sysfs_node_entry(dev, i);
 783                 if (ret < 0)
 784                         return ret;
 785                 i++;
 786         }
 787
 788         return 0;
 789 }
 790
 791 /* Called with write topology lock acquired */
 792 static void kfd_remove_sysfs_node_tree(void)
 793 {
 794         struct kfd_topology_device *dev;
 795
 796         list_for_each_entry(dev, &topology_device_list, list)
 797                 kfd_remove_sysfs_node_entry(dev);
 798 }
 799
 800 static int kfd_topology_update_sysfs(void)
 801 {
 802         int ret;
 803
 804         if (!sys_props.kobj_topology) {
 805                 sys_props.kobj_topology =
 806                                 kfd_alloc_struct(sys_props.kobj_topology);
 807                 if (!sys_props.kobj_topology)
 808                         return -ENOMEM;
 809
 810                 ret = kobject_init_and_add(sys_props.kobj_topology,
 811                                 &sysprops_type,  &kfd_device->kobj,
 812                                 "topology");
 813                 if (ret < 0) {
 814                         kobject_put(sys_props.kobj_topology);
 815                         return ret;
 816                 }
 817
 818                 sys_props.kobj_nodes = kobject_create_and_add("nodes",
 819                                 sys_props.kobj_topology);
 820                 if (!sys_props.kobj_nodes)
 821                         return -ENOMEM;
 822
 823                 sys_props.attr_genid.name = "generation_id";
 824                 sys_props.attr_genid.mode = KFD_SYSFS_FILE_MODE;
 825                 sysfs_attr_init(&sys_props.attr_genid);
 826                 ret = sysfs_create_file(sys_props.kobj_topology,
 827                                 &sys_props.attr_genid);
 828                 if (ret < 0)
 829                         return ret;
 830
 831                 sys_props.attr_props.name = "system_properties";
 832                 sys_props.attr_props.mode = KFD_SYSFS_FILE_MODE;
 833                 sysfs_attr_init(&sys_props.attr_props);
 834                 ret = sysfs_create_file(sys_props.kobj_topology,
 835                                 &sys_props.attr_props);
 836                 if (ret < 0)
 837                         return ret;
 838         }
 839
 840         kfd_remove_sysfs_node_tree();
 841
 842         return kfd_build_sysfs_node_tree();
 843 }
 844
 845 static void kfd_topology_release_sysfs(void)
 846 {
 847         kfd_remove_sysfs_node_tree();
 848         if (sys_props.kobj_topology) {
 849                 sysfs_remove_file(sys_props.kobj_topology,
 850                                 &sys_props.attr_genid);
 851                 sysfs_remove_file(sys_props.kobj_topology,
 852                                 &sys_props.attr_props);
 853                 if (sys_props.kobj_nodes) {
 854                         kobject_del(sys_props.kobj_nodes);
 855                         kobject_put(sys_props.kobj_nodes);
 856                         sys_props.kobj_nodes = NULL;
 857                 }
 858                 kobject_del(sys_props.kobj_topology);
 859                 kobject_put(sys_props.kobj_topology);
 860                 sys_props.kobj_topology = NULL;
 861         }
 862 }
 863
 864 /* Called with write topology_lock acquired */
 865 static void kfd_topology_update_device_list(struct list_head *temp_list,
 866                                         struct list_head *master_list)
 867 {
 868         while (!list_empty(temp_list)) {
 869                 list_move_tail(temp_list->next, master_list);
 870                 sys_props.num_devices++;
 871         }
 872 }
 873
 874 static void kfd_debug_print_topology(void)
 875 {
 876         struct kfd_topology_device *dev;
 877
 878         down_read(&topology_lock);
 879
 880         dev = list_last_entry(&topology_device_list,
 881                         struct kfd_topology_device, list);
 882         if (dev) {
 883                 if (dev->node_props.cpu_cores_count &&
 884                                 dev->node_props.simd_count) {
 885                         pr_info("Topology: Add APU node [0x%0x:0x%0x]\n",
 886                                 dev->node_props.device_id,
 887                                 dev->node_props.vendor_id);
 888                 } else if (dev->node_props.cpu_cores_count)
 889                         pr_info("Topology: Add CPU node\n");
 890                 else if (dev->node_props.simd_count)
 891                         pr_info("Topology: Add dGPU node [0x%0x:0x%0x]\n",
 892                                 dev->node_props.device_id,
 893                                 dev->node_props.vendor_id);
 894         }
 895         up_read(&topology_lock);
 896 }
 897
 898 /* Helper function for intializing platform_xx members of
 899  * kfd_system_properties. Uses OEM info from the last CPU/APU node.
 900  */
 901 static void kfd_update_system_properties(void)
 902 {
 903         struct kfd_topology_device *dev;
 904
 905         down_read(&topology_lock);
 906         dev = list_last_entry(&topology_device_list,
 907                         struct kfd_topology_device, list);
 908         if (dev) {
 909                 sys_props.platform_id =
 910                         (*((uint64_t *)dev->oem_id)) & CRAT_OEMID_64BIT_MASK;
 911                 sys_props.platform_oem = *((uint64_t *)dev->oem_table_id);
 912                 sys_props.platform_rev = dev->oem_revision;
 913         }
 914         up_read(&topology_lock);
 915 }
 916
 917 static void find_system_memory(const struct dmi_header *dm,
 918         void *private)
 919 {
 920         struct kfd_mem_properties *mem;
 921         u16 mem_width, mem_clock;
 922         struct kfd_topology_device *kdev =
 923                 (struct kfd_topology_device *)private;
 924         const u8 *dmi_data = (const u8 *)(dm + 1);
 925
 926         if (dm->type == DMI_ENTRY_MEM_DEVICE && dm->length >= 0x15) {
 927                 mem_width = (u16)(*(const u16 *)(dmi_data + 0x6));
 928                 mem_clock = (u16)(*(const u16 *)(dmi_data + 0x11));
 929                 list_for_each_entry(mem, &kdev->mem_props, list) {
 930                         if (mem_width != 0xFFFF && mem_width != 0)
 931                                 mem->width = mem_width;
 932                         if (mem_clock != 0)
 933                                 mem->mem_clk_max = mem_clock;
 934                 }
 935         }
 936 }
 937
 938 /*
 939  * Performance counters information is not part of CRAT but we would like to
 940  * put them in the sysfs under topology directory for Thunk to get the data.
 941  * This function is called before updating the sysfs.
 942  */
 943 static int kfd_add_perf_to_topology(struct kfd_topology_device *kdev)
 944 {
 945         /* These are the only counters supported so far */
 946         return kfd_iommu_add_perf_counters(kdev);
 947 }
 948
 949 /* kfd_add_non_crat_information - Add information that is not currently
 950  *      defined in CRAT but is necessary for KFD topology
 951  * @dev - topology device to which addition info is added
 952  */
 953 static void kfd_add_non_crat_information(struct kfd_topology_device *kdev)
 954 {
 955         /* Check if CPU only node. */
 956         if (!kdev->gpu) {
 957                 /* Add system memory information */
 958                 dmi_walk(find_system_memory, kdev);
 959         }
 960         /* TODO: For GPU node, rearrange code from kfd_topology_add_device */
 961 }
 962
 963 /* kfd_is_acpi_crat_invalid - CRAT from ACPI is valid only for AMD APU devices.
 964  *      Ignore CRAT for all other devices. AMD APU is identified if both CPU
 965  *      and GPU cores are present.
 966  * @device_list - topology device list created by parsing ACPI CRAT table.
 967  * @return - TRUE if invalid, FALSE is valid.
 968  */
 969 static bool kfd_is_acpi_crat_invalid(struct list_head *device_list)
 970 {
 971         struct kfd_topology_device *dev;
 972
 973         list_for_each_entry(dev, device_list, list) {
 974                 if (dev->node_props.cpu_cores_count &&
 975                         dev->node_props.simd_count)
 976                         return false;
 977         }
 978         pr_info("Ignoring ACPI CRAT on non-APU system\n");
 979         return true;
 980 }
 981
 982 int kfd_topology_init(void)
 983 {
 984         void *crat_image = NULL;
 985         size_t image_size = 0;
 986         int ret;
 987         struct list_head temp_topology_device_list;
 988         int cpu_only_node = 0;
 989         struct kfd_topology_device *kdev;
 990         int proximity_domain;
 991
 992         /* topology_device_list - Master list of all topology devices
 993          * temp_topology_device_list - temporary list created while parsing CRAT
 994          * or VCRAT. Once parsing is complete the contents of list is moved to
 995          * topology_device_list
 996          */
 997
 998         /* Initialize the head for the both the lists */
 999         INIT_LIST_HEAD(&topology_device_list);
1000         INIT_LIST_HEAD(&temp_topology_device_list);
1001         init_rwsem(&topology_lock);
1002
1003         memset(&sys_props, 0, sizeof(sys_props));
1004
1005         /* Proximity domains in ACPI CRAT tables start counting at
1006          * 0. The same should be true for virtual CRAT tables created
1007          * at this stage. GPUs added later in kfd_topology_add_device
1008          * use a counter.
1009          */
1010         proximity_domain = 0;
1011
1012         /*
1013          * Get the CRAT image from the ACPI. If ACPI doesn't have one
1014          * or if ACPI CRAT is invalid create a virtual CRAT.
1015          * NOTE: The current implementation expects all AMD APUs to have
1016          *      CRAT. If no CRAT is available, it is assumed to be a CPU
1017          */
1018         ret = kfd_create_crat_image_acpi(&crat_image, &image_size);
1019         if (!ret) {
1020                 ret = kfd_parse_crat_table(crat_image,
1021                                            &temp_topology_device_list,
1022                                            proximity_domain);
1023                 if (ret ||
1024                     kfd_is_acpi_crat_invalid(&temp_topology_device_list)) {
1025                         kfd_release_topology_device_list(
1026                                 &temp_topology_device_list);
1027                         kfd_destroy_crat_image(crat_image);
1028                         crat_image = NULL;
1029                 }
1030         }
1031
1032         if (!crat_image) {
1033                 ret = kfd_create_crat_image_virtual(&crat_image, &image_size,
1034                                                     COMPUTE_UNIT_CPU, NULL,
1035                                                     proximity_domain);
1036                 cpu_only_node = 1;
1037                 if (ret) {
1038                         pr_err("Error creating VCRAT table for CPU\n");
1039                         return ret;
1040                 }
1041
1042                 ret = kfd_parse_crat_table(crat_image,
1043                                            &temp_topology_device_list,
1044                                            proximity_domain);
1045                 if (ret) {
1046                         pr_err("Error parsing VCRAT table for CPU\n");
1047                         goto err;
1048                 }
1049         }
1050
1051         kdev = list_first_entry(&temp_topology_device_list,
1052                                 struct kfd_topology_device, list);
1053         kfd_add_perf_to_topology(kdev);
1054
1055         down_write(&topology_lock);
1056         kfd_topology_update_device_list(&temp_topology_device_list,
1057                                         &topology_device_list);
1058         atomic_set(&topology_crat_proximity_domain, sys_props.num_devices-1);
1059         ret = kfd_topology_update_sysfs();
1060         up_write(&topology_lock);
1061
1062         if (!ret) {
1063                 sys_props.generation_count++;
1064                 kfd_update_system_properties();
1065                 kfd_debug_print_topology();
1066         } else
1067                 pr_err("Failed to update topology in sysfs ret=%d\n", ret);
1068
1069         /* For nodes with GPU, this information gets added
1070          * when GPU is detected (kfd_topology_add_device).
1071          */
1072         if (cpu_only_node) {
1073                 /* Add additional information to CPU only node created above */
1074                 down_write(&topology_lock);
1075                 kdev = list_first_entry(&topology_device_list,
1076                                 struct kfd_topology_device, list);
1077                 up_write(&topology_lock);
1078                 kfd_add_non_crat_information(kdev);
1079         }
1080
1081 err:
1082         kfd_destroy_crat_image(crat_image);
1083         return ret;
1084 }
1085
1086 void kfd_topology_shutdown(void)
1087 {
1088         down_write(&topology_lock);
1089         kfd_topology_release_sysfs();
1090         kfd_release_live_view();
1091         up_write(&topology_lock);
1092 }
1093
1094 static uint32_t kfd_generate_gpu_id(struct kfd_dev *gpu)
1095 {
1096         uint32_t hashout;
1097         uint32_t buf[7];
1098         uint64_t local_mem_size;
1099         int i;
1100         struct kfd_local_mem_info local_mem_info;
1101
1102         if (!gpu)
1103                 return 0;
1104
1105         amdgpu_amdkfd_get_local_mem_info(gpu->kgd, &local_mem_info);
1106
1107         local_mem_size = local_mem_info.local_mem_size_private +
1108                         local_mem_info.local_mem_size_public;
1109
1110         buf[0] = gpu->pdev->devfn;
1111         buf[1] = gpu->pdev->subsystem_vendor |
1112                 (gpu->pdev->subsystem_device << 16);
1113         buf[2] = pci_domain_nr(gpu->pdev->bus);
1114         buf[3] = gpu->pdev->device;
1115         buf[4] = gpu->pdev->bus->number;
1116         buf[5] = lower_32_bits(local_mem_size);
1117         buf[6] = upper_32_bits(local_mem_size);
1118
1119         for (i = 0, hashout = 0; i < 7; i++)
1120                 hashout ^= hash_32(buf[i], KFD_GPU_ID_HASH_WIDTH);
1121
1122         return hashout;
1123 }
1124 /* kfd_assign_gpu - Attach @gpu to the correct kfd topology device. If
1125  *              the GPU device is not already present in the topology device
1126  *              list then return NULL. This means a new topology device has to
1127  *              be created for this GPU.
1128  */
1129 static struct kfd_topology_device *kfd_assign_gpu(struct kfd_dev *gpu)
1130 {
1131         struct kfd_topology_device *dev;
1132         struct kfd_topology_device *out_dev = NULL;
1133         struct kfd_mem_properties *mem;
1134         struct kfd_cache_properties *cache;
1135         struct kfd_iolink_properties *iolink;
1136
1137         down_write(&topology_lock);
1138         list_for_each_entry(dev, &topology_device_list, list) {
1139                 /* Discrete GPUs need their own topology device list
1140                  * entries. Don't assign them to CPU/APU nodes.
1141                  */
1142                 if (!gpu->use_iommu_v2 &&
1143                     dev->node_props.cpu_cores_count)
1144                         continue;
1145
1146                 if (!dev->gpu && (dev->node_props.simd_count > 0)) {
1147                         dev->gpu = gpu;
1148                         out_dev = dev;
1149
1150                         list_for_each_entry(mem, &dev->mem_props, list)
1151                                 mem->gpu = dev->gpu;
1152                         list_for_each_entry(cache, &dev->cache_props, list)
1153                                 cache->gpu = dev->gpu;
1154                         list_for_each_entry(iolink, &dev->io_link_props, list)
1155                                 iolink->gpu = dev->gpu;
1156                         break;
1157                 }
1158         }
1159         up_write(&topology_lock);
1160         return out_dev;
1161 }
1162
1163 static void kfd_notify_gpu_change(uint32_t gpu_id, int arrival)
1164 {
1165         /*
1166          * TODO: Generate an event for thunk about the arrival/removal
1167          * of the GPU
1168          */
1169 }
1170
1171 /* kfd_fill_mem_clk_max_info - Since CRAT doesn't have memory clock info,
1172  *              patch this after CRAT parsing.
1173  */
1174 static void kfd_fill_mem_clk_max_info(struct kfd_topology_device *dev)
1175 {
1176         struct kfd_mem_properties *mem;
1177         struct kfd_local_mem_info local_mem_info;
1178
1179         if (!dev)
1180                 return;
1181
1182         /* Currently, amdgpu driver (amdgpu_mc) deals only with GPUs with
1183          * single bank of VRAM local memory.
1184          * for dGPUs - VCRAT reports only one bank of Local Memory
1185          * for APUs - If CRAT from ACPI reports more than one bank, then
1186          *      all the banks will report the same mem_clk_max information
1187          */
1188         amdgpu_amdkfd_get_local_mem_info(dev->gpu->kgd, &local_mem_info);
1189
1190         list_for_each_entry(mem, &dev->mem_props, list)
1191                 mem->mem_clk_max = local_mem_info.mem_clk_max;
1192 }
1193
1194 static void kfd_fill_iolink_non_crat_info(struct kfd_topology_device *dev)
1195 {
1196         struct kfd_iolink_properties *link, *cpu_link;
1197         struct kfd_topology_device *cpu_dev;
1198         uint32_t cap;
1199         uint32_t cpu_flag = CRAT_IOLINK_FLAGS_ENABLED;
1200         uint32_t flag = CRAT_IOLINK_FLAGS_ENABLED;
1201
1202         if (!dev || !dev->gpu)
1203                 return;
1204
1205         pcie_capability_read_dword(dev->gpu->pdev,
1206                         PCI_EXP_DEVCAP2, &cap);
1207
1208         if (!(cap & (PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
1209                      PCI_EXP_DEVCAP2_ATOMIC_COMP64)))
1210                 cpu_flag |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT |
1211                         CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT;
1212
1213         if (!dev->gpu->pci_atomic_requested ||
1214             dev->gpu->device_info->asic_family == CHIP_HAWAII)
1215                 flag |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT |
1216                         CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT;
1217
1218         /* GPU only creates direct links so apply flags setting to all */
1219         list_for_each_entry(link, &dev->io_link_props, list) {
1220                 link->flags = flag;
1221                 cpu_dev = kfd_topology_device_by_proximity_domain(
1222                                 link->node_to);
1223                 if (cpu_dev) {
1224                         list_for_each_entry(cpu_link,
1225                                             &cpu_dev->io_link_props, list)
1226                                 if (cpu_link->node_to == link->node_from)
1227                                         cpu_link->flags = cpu_flag;
1228                 }
1229         }
1230 }
1231
1232 int kfd_topology_add_device(struct kfd_dev *gpu)
1233 {
1234         uint32_t gpu_id;
1235         struct kfd_topology_device *dev;
1236         struct kfd_cu_info cu_info;
1237         int res = 0;
1238         struct list_head temp_topology_device_list;
1239         void *crat_image = NULL;
1240         size_t image_size = 0;
1241         int proximity_domain;
1242         struct amdgpu_ras *ctx;
1243
1244         INIT_LIST_HEAD(&temp_topology_device_list);
1245
1246         gpu_id = kfd_generate_gpu_id(gpu);
1247
1248         pr_debug("Adding new GPU (ID: 0x%x) to topology\n", gpu_id);
1249
1250         proximity_domain = atomic_inc_return(&topology_crat_proximity_domain);
1251
1252         /* Check to see if this gpu device exists in the topology_device_list.
1253          * If so, assign the gpu to that device,
1254          * else create a Virtual CRAT for this gpu device and then parse that
1255          * CRAT to create a new topology device. Once created assign the gpu to
1256          * that topology device
1257          */
1258         dev = kfd_assign_gpu(gpu);
1259         if (!dev) {
1260                 res = kfd_create_crat_image_virtual(&crat_image, &image_size,
1261                                                     COMPUTE_UNIT_GPU, gpu,
1262                                                     proximity_domain);
1263                 if (res) {
1264                         pr_err("Error creating VCRAT for GPU (ID: 0x%x)\n",
1265                                gpu_id);
1266                         return res;
1267                 }
1268                 res = kfd_parse_crat_table(crat_image,
1269                                            &temp_topology_device_list,
1270                                            proximity_domain);
1271                 if (res) {
1272                         pr_err("Error parsing VCRAT for GPU (ID: 0x%x)\n",
1273                                gpu_id);
1274                         goto err;
1275                 }
1276
1277                 down_write(&topology_lock);
1278                 kfd_topology_update_device_list(&temp_topology_device_list,
1279                         &topology_device_list);
1280
1281                 /* Update the SYSFS tree, since we added another topology
1282                  * device
1283                  */
1284                 res = kfd_topology_update_sysfs();
1285                 up_write(&topology_lock);
1286
1287                 if (!res)
1288                         sys_props.generation_count++;
1289                 else
1290                         pr_err("Failed to update GPU (ID: 0x%x) to sysfs topology. res=%d\n",
1291                                                 gpu_id, res);
1292                 dev = kfd_assign_gpu(gpu);
1293                 if (WARN_ON(!dev)) {
1294                         res = -ENODEV;
1295                         goto err;
1296                 }
1297         }
1298
1299         dev->gpu_id = gpu_id;
1300         gpu->id = gpu_id;
1301
1302         /* TODO: Move the following lines to function
1303          *      kfd_add_non_crat_information
1304          */
1305
1306         /* Fill-in additional information that is not available in CRAT but
1307          * needed for the topology
1308          */
1309
1310         amdgpu_amdkfd_get_cu_info(dev->gpu->kgd, &cu_info);
1311
1312         strncpy(dev->node_props.name, gpu->device_info->asic_name,
1313                         KFD_TOPOLOGY_PUBLIC_NAME_SIZE);
1314
1315         dev->node_props.simd_arrays_per_engine =
1316                 cu_info.num_shader_arrays_per_engine;
1317
1318         dev->node_props.vendor_id = gpu->pdev->vendor;
1319         dev->node_props.device_id = gpu->pdev->device;
1320         dev->node_props.capability |=
1321                 ((amdgpu_amdkfd_get_asic_rev_id(dev->gpu->kgd) <<
1322                         HSA_CAP_ASIC_REVISION_SHIFT) &
1323                         HSA_CAP_ASIC_REVISION_MASK);
1324         dev->node_props.location_id = pci_dev_id(gpu->pdev);
1325         dev->node_props.domain = pci_domain_nr(gpu->pdev->bus);
1326         dev->node_props.max_engine_clk_fcompute =
1327                 amdgpu_amdkfd_get_max_engine_clock_in_mhz(dev->gpu->kgd);
1328         dev->node_props.max_engine_clk_ccompute =
1329                 cpufreq_quick_get_max(0) / 1000;
1330         dev->node_props.drm_render_minor =
1331                 gpu->shared_resources.drm_render_minor;
1332
1333         dev->node_props.hive_id = gpu->hive_id;
1334         dev->node_props.num_sdma_engines = gpu->device_info->num_sdma_engines;
1335         dev->node_props.num_sdma_xgmi_engines =
1336                                 gpu->device_info->num_xgmi_sdma_engines;
1337         dev->node_props.num_sdma_queues_per_engine =
1338                                 gpu->device_info->num_sdma_queues_per_engine;
1339         dev->node_props.num_gws = (dev->gpu->gws &&
1340                 dev->gpu->dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) ?
1341                 amdgpu_amdkfd_get_num_gws(dev->gpu->kgd) : 0;
1342         dev->node_props.num_cp_queues = get_cp_queues_num(dev->gpu->dqm);
1343         dev->node_props.unique_id = gpu->unique_id;
1344
1345         kfd_fill_mem_clk_max_info(dev);
1346         kfd_fill_iolink_non_crat_info(dev);
1347
1348         switch (dev->gpu->device_info->asic_family) {
1349         case CHIP_KAVERI:
1350         case CHIP_HAWAII:
1351         case CHIP_TONGA:
1352                 dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_PRE_1_0 <<
1353                         HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) &
1354                         HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK);
1355                 break;
1356         case CHIP_CARRIZO:
1357         case CHIP_FIJI:
1358         case CHIP_POLARIS10:
1359         case CHIP_POLARIS11:
1360         case CHIP_POLARIS12:
1361         case CHIP_VEGAM:
1362                 pr_debug("Adding doorbell packet type capability\n");
1363                 dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_1_0 <<
1364                         HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) &
1365                         HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK);
1366                 break;
1367         case CHIP_VEGA10:
1368         case CHIP_VEGA12:
1369         case CHIP_VEGA20:
1370         case CHIP_RAVEN:
1371         case CHIP_RENOIR:
1372         case CHIP_ARCTURUS:
1373         case CHIP_NAVI10:
1374         case CHIP_NAVI12:
1375         case CHIP_NAVI14:
1376         case CHIP_SIENNA_CICHLID:
1377         case CHIP_NAVY_FLOUNDER:
1378                 dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_2_0 <<
1379                         HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) &
1380                         HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK);
1381                 break;
1382         default:
1383                 WARN(1, "Unexpected ASIC family %u",
1384                      dev->gpu->device_info->asic_family);
1385         }
1386
1387         /*
1388         * Overwrite ATS capability according to needs_iommu_device to fix
1389         * potential missing corresponding bit in CRAT of BIOS.
1390         */
1391         if (dev->gpu->use_iommu_v2)
1392                 dev->node_props.capability |= HSA_CAP_ATS_PRESENT;
1393         else
1394                 dev->node_props.capability &= ~HSA_CAP_ATS_PRESENT;
1395
1396         /* Fix errors in CZ CRAT.
1397          * simd_count: Carrizo CRAT reports wrong simd_count, probably
1398          *              because it doesn't consider masked out CUs
1399          * max_waves_per_simd: Carrizo reports wrong max_waves_per_simd
1400          */
1401         if (dev->gpu->device_info->asic_family == CHIP_CARRIZO) {
1402                 dev->node_props.simd_count =
1403                         cu_info.simd_per_cu * cu_info.cu_active_number;
1404                 dev->node_props.max_waves_per_simd = 10;
1405         }
1406
1407         ctx = amdgpu_ras_get_context((struct amdgpu_device *)(dev->gpu->kgd));
1408         if (ctx) {
1409                 /* kfd only concerns sram ecc on GFX/SDMA and HBM ecc on UMC */
1410                 dev->node_props.capability |=
1411                         (((ctx->features & BIT(AMDGPU_RAS_BLOCK__SDMA)) != 0) ||
1412                          ((ctx->features & BIT(AMDGPU_RAS_BLOCK__GFX)) != 0)) ?
1413                         HSA_CAP_SRAM_EDCSUPPORTED : 0;
1414                 dev->node_props.capability |= ((ctx->features & BIT(AMDGPU_RAS_BLOCK__UMC)) != 0) ?
1415                         HSA_CAP_MEM_EDCSUPPORTED : 0;
1416
1417                 dev->node_props.capability |= (ctx->features != 0) ?
1418                         HSA_CAP_RASEVENTNOTIFY : 0;
1419         }
1420
1421         kfd_debug_print_topology();
1422
1423         if (!res)
1424                 kfd_notify_gpu_change(gpu_id, 1);
1425 err:
1426         kfd_destroy_crat_image(crat_image);
1427         return res;
1428 }
1429
1430 int kfd_topology_remove_device(struct kfd_dev *gpu)
1431 {
1432         struct kfd_topology_device *dev, *tmp;
1433         uint32_t gpu_id;
1434         int res = -ENODEV;
1435
1436         down_write(&topology_lock);
1437
1438         list_for_each_entry_safe(dev, tmp, &topology_device_list, list)
1439                 if (dev->gpu == gpu) {
1440                         gpu_id = dev->gpu_id;
1441                         kfd_remove_sysfs_node_entry(dev);
1442                         kfd_release_topology_device(dev);
1443                         sys_props.num_devices--;
1444                         res = 0;
1445                         if (kfd_topology_update_sysfs() < 0)
1446                                 kfd_topology_release_sysfs();
1447                         break;
1448                 }
1449
1450         up_write(&topology_lock);
1451
1452         if (!res)
1453                 kfd_notify_gpu_change(gpu_id, 0);
1454
1455         return res;
1456 }
1457
1458 /* kfd_topology_enum_kfd_devices - Enumerate through all devices in KFD
1459  *      topology. If GPU device is found @idx, then valid kfd_dev pointer is
1460  *      returned through @kdev
1461  * Return -     0: On success (@kdev will be NULL for non GPU nodes)
1462  *              -1: If end of list
1463  */
1464 int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_dev **kdev)
1465 {
1466
1467         struct kfd_topology_device *top_dev;
1468         uint8_t device_idx = 0;
1469
1470         *kdev = NULL;
1471         down_read(&topology_lock);
1472
1473         list_for_each_entry(top_dev, &topology_device_list, list) {
1474                 if (device_idx == idx) {
1475                         *kdev = top_dev->gpu;
1476                         up_read(&topology_lock);
1477                         return 0;
1478                 }
1479
1480                 device_idx++;
1481         }
1482
1483         up_read(&topology_lock);
1484
1485         return -1;
1486
1487 }
1488
1489 static int kfd_cpumask_to_apic_id(const struct cpumask *cpumask)
1490 {
1491         int first_cpu_of_numa_node;
1492
1493         if (!cpumask || cpumask == cpu_none_mask)
1494                 return -1;
1495         first_cpu_of_numa_node = cpumask_first(cpumask);
1496         if (first_cpu_of_numa_node >= nr_cpu_ids)
1497                 return -1;
1498 #ifdef CONFIG_X86_64
1499         return cpu_data(first_cpu_of_numa_node).apicid;
1500 #else
1501         return first_cpu_of_numa_node;
1502 #endif
1503 }
1504
1505 /* kfd_numa_node_to_apic_id - Returns the APIC ID of the first logical processor
1506  *      of the given NUMA node (numa_node_id)
1507  * Return -1 on failure
1508  */
1509 int kfd_numa_node_to_apic_id(int numa_node_id)
1510 {
1511         if (numa_node_id == -1) {
1512                 pr_warn("Invalid NUMA Node. Use online CPU mask\n");
1513                 return kfd_cpumask_to_apic_id(cpu_online_mask);
1514         }
1515         return kfd_cpumask_to_apic_id(cpumask_of_node(numa_node_id));
1516 }
1517
1518 void kfd_double_confirm_iommu_support(struct kfd_dev *gpu)
1519 {
1520         struct kfd_topology_device *dev;
1521
1522         gpu->use_iommu_v2 = false;
1523
1524         if (!gpu->device_info->needs_iommu_device)
1525                 return;
1526
1527         down_read(&topology_lock);
1528
1529         /* Only use IOMMUv2 if there is an APU topology node with no GPU
1530          * assigned yet. This GPU will be assigned to it.
1531          */
1532         list_for_each_entry(dev, &topology_device_list, list)
1533                 if (dev->node_props.cpu_cores_count &&
1534                     dev->node_props.simd_count &&
1535                     !dev->gpu)
1536                         gpu->use_iommu_v2 = true;
1537
1538         up_read(&topology_lock);
1539 }
1540
1541 #if defined(CONFIG_DEBUG_FS)
1542
1543 int kfd_debugfs_hqds_by_device(struct seq_file *m, void *data)
1544 {
1545         struct kfd_topology_device *dev;
1546         unsigned int i = 0;
1547         int r = 0;
1548
1549         down_read(&topology_lock);
1550
1551         list_for_each_entry(dev, &topology_device_list, list) {
1552                 if (!dev->gpu) {
1553                         i++;
1554                         continue;
1555                 }
1556
1557                 seq_printf(m, "Node %u, gpu_id %x:\n", i++, dev->gpu->id);
1558                 r = dqm_debugfs_hqds(m, dev->gpu->dqm);
1559                 if (r)
1560                         break;
1561         }
1562
1563         up_read(&topology_lock);
1564
1565         return r;
1566 }
1567
1568 int kfd_debugfs_rls_by_device(struct seq_file *m, void *data)
1569 {
1570         struct kfd_topology_device *dev;
1571         unsigned int i = 0;
1572         int r = 0;
1573
1574         down_read(&topology_lock);
1575
1576         list_for_each_entry(dev, &topology_device_list, list) {
1577                 if (!dev->gpu) {
1578                         i++;
1579                         continue;
1580                 }
1581
1582                 seq_printf(m, "Node %u, gpu_id %x:\n", i++, dev->gpu->id);
1583                 r = pm_debugfs_runlist(m, &dev->gpu->dqm->packets);
1584                 if (r)
1585                         break;
1586         }
1587
1588         up_read(&topology_lock);
1589
1590         return r;
1591 }
1592
1593 #endif