Merge branch 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6
[linux-2.6-microblaze.git] / drivers / gpu / drm / amd / amdgpu / amdgpu_xgmi.c
1 /*
2  * Copyright 2018 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  *
23  */
24 #include <linux/list.h>
25 #include "amdgpu.h"
26 #include "amdgpu_xgmi.h"
27 #include "amdgpu_smu.h"
28 #include "amdgpu_ras.h"
29 #include "df/df_3_6_offset.h"
30
31 static DEFINE_MUTEX(xgmi_mutex);
32
33 #define AMDGPU_MAX_XGMI_HIVE                    8
34 #define AMDGPU_MAX_XGMI_DEVICE_PER_HIVE         4
35
36 static struct amdgpu_hive_info xgmi_hives[AMDGPU_MAX_XGMI_HIVE];
37 static unsigned hive_count = 0;
38
39 void *amdgpu_xgmi_hive_try_lock(struct amdgpu_hive_info *hive)
40 {
41         return &hive->device_list;
42 }
43
44 /**
45  * DOC: AMDGPU XGMI Support
46  *
47  * XGMI is a high speed interconnect that joins multiple GPU cards
48  * into a homogeneous memory space that is organized by a collective
49  * hive ID and individual node IDs, both of which are 64-bit numbers.
50  *
51  * The file xgmi_device_id contains the unique per GPU device ID and
52  * is stored in the /sys/class/drm/card${cardno}/device/ directory.
53  *
54  * Inside the device directory a sub-directory 'xgmi_hive_info' is
55  * created which contains the hive ID and the list of nodes.
56  *
57  * The hive ID is stored in:
58  *   /sys/class/drm/card${cardno}/device/xgmi_hive_info/xgmi_hive_id
59  *
60  * The node information is stored in numbered directories:
61  *   /sys/class/drm/card${cardno}/device/xgmi_hive_info/node${nodeno}/xgmi_device_id
62  *
63  * Each device has their own xgmi_hive_info direction with a mirror
64  * set of node sub-directories.
65  *
66  * The XGMI memory space is built by contiguously adding the power of
67  * two padded VRAM space from each node to each other.
68  *
69  */
70
71
72 static ssize_t amdgpu_xgmi_show_hive_id(struct device *dev,
73                 struct device_attribute *attr, char *buf)
74 {
75         struct amdgpu_hive_info *hive =
76                         container_of(attr, struct amdgpu_hive_info, dev_attr);
77
78         return snprintf(buf, PAGE_SIZE, "%llu\n", hive->hive_id);
79 }
80
81 static int amdgpu_xgmi_sysfs_create(struct amdgpu_device *adev,
82                                     struct amdgpu_hive_info *hive)
83 {
84         int ret = 0;
85
86         if (WARN_ON(hive->kobj))
87                 return -EINVAL;
88
89         hive->kobj = kobject_create_and_add("xgmi_hive_info", &adev->dev->kobj);
90         if (!hive->kobj) {
91                 dev_err(adev->dev, "XGMI: Failed to allocate sysfs entry!\n");
92                 return -EINVAL;
93         }
94
95         hive->dev_attr = (struct device_attribute) {
96                 .attr = {
97                         .name = "xgmi_hive_id",
98                         .mode = S_IRUGO,
99
100                 },
101                 .show = amdgpu_xgmi_show_hive_id,
102         };
103
104         ret = sysfs_create_file(hive->kobj, &hive->dev_attr.attr);
105         if (ret) {
106                 dev_err(adev->dev, "XGMI: Failed to create device file xgmi_hive_id\n");
107                 kobject_del(hive->kobj);
108                 kobject_put(hive->kobj);
109                 hive->kobj = NULL;
110         }
111
112         return ret;
113 }
114
115 static void amdgpu_xgmi_sysfs_destroy(struct amdgpu_device *adev,
116                                     struct amdgpu_hive_info *hive)
117 {
118         sysfs_remove_file(hive->kobj, &hive->dev_attr.attr);
119         kobject_del(hive->kobj);
120         kobject_put(hive->kobj);
121         hive->kobj = NULL;
122 }
123
124 static ssize_t amdgpu_xgmi_show_device_id(struct device *dev,
125                                      struct device_attribute *attr,
126                                      char *buf)
127 {
128         struct drm_device *ddev = dev_get_drvdata(dev);
129         struct amdgpu_device *adev = ddev->dev_private;
130
131         return snprintf(buf, PAGE_SIZE, "%llu\n", adev->gmc.xgmi.node_id);
132
133 }
134
135 #define AMDGPU_XGMI_SET_FICAA(o)        ((o) | 0x456801)
136 static ssize_t amdgpu_xgmi_show_error(struct device *dev,
137                                       struct device_attribute *attr,
138                                       char *buf)
139 {
140         struct drm_device *ddev = dev_get_drvdata(dev);
141         struct amdgpu_device *adev = ddev->dev_private;
142         uint32_t ficaa_pie_ctl_in, ficaa_pie_status_in;
143         uint64_t fica_out;
144         unsigned int error_count = 0;
145
146         ficaa_pie_ctl_in = AMDGPU_XGMI_SET_FICAA(0x200);
147         ficaa_pie_status_in = AMDGPU_XGMI_SET_FICAA(0x208);
148
149         fica_out = adev->df.funcs->get_fica(adev, ficaa_pie_ctl_in);
150         if (fica_out != 0x1f)
151                 pr_err("xGMI error counters not enabled!\n");
152
153         fica_out = adev->df.funcs->get_fica(adev, ficaa_pie_status_in);
154
155         if ((fica_out & 0xffff) == 2)
156                 error_count = ((fica_out >> 62) & 0x1) + (fica_out >> 63);
157
158         adev->df.funcs->set_fica(adev, ficaa_pie_status_in, 0, 0);
159
160         return snprintf(buf, PAGE_SIZE, "%d\n", error_count);
161 }
162
163
164 static DEVICE_ATTR(xgmi_device_id, S_IRUGO, amdgpu_xgmi_show_device_id, NULL);
165 static DEVICE_ATTR(xgmi_error, S_IRUGO, amdgpu_xgmi_show_error, NULL);
166
167 static int amdgpu_xgmi_sysfs_add_dev_info(struct amdgpu_device *adev,
168                                          struct amdgpu_hive_info *hive)
169 {
170         int ret = 0;
171         char node[10] = { 0 };
172
173         /* Create xgmi device id file */
174         ret = device_create_file(adev->dev, &dev_attr_xgmi_device_id);
175         if (ret) {
176                 dev_err(adev->dev, "XGMI: Failed to create device file xgmi_device_id\n");
177                 return ret;
178         }
179
180         /* Create xgmi error file */
181         ret = device_create_file(adev->dev, &dev_attr_xgmi_error);
182         if (ret)
183                 pr_err("failed to create xgmi_error\n");
184
185
186         /* Create sysfs link to hive info folder on the first device */
187         if (adev != hive->adev) {
188                 ret = sysfs_create_link(&adev->dev->kobj, hive->kobj,
189                                         "xgmi_hive_info");
190                 if (ret) {
191                         dev_err(adev->dev, "XGMI: Failed to create link to hive info");
192                         goto remove_file;
193                 }
194         }
195
196         sprintf(node, "node%d", hive->number_devices);
197         /* Create sysfs link form the hive folder to yourself */
198         ret = sysfs_create_link(hive->kobj, &adev->dev->kobj, node);
199         if (ret) {
200                 dev_err(adev->dev, "XGMI: Failed to create link from hive info");
201                 goto remove_link;
202         }
203
204         goto success;
205
206
207 remove_link:
208         sysfs_remove_link(&adev->dev->kobj, adev->ddev->unique);
209
210 remove_file:
211         device_remove_file(adev->dev, &dev_attr_xgmi_device_id);
212
213 success:
214         return ret;
215 }
216
217 static void amdgpu_xgmi_sysfs_rem_dev_info(struct amdgpu_device *adev,
218                                           struct amdgpu_hive_info *hive)
219 {
220         device_remove_file(adev->dev, &dev_attr_xgmi_device_id);
221         sysfs_remove_link(&adev->dev->kobj, adev->ddev->unique);
222         sysfs_remove_link(hive->kobj, adev->ddev->unique);
223 }
224
225
226
227 struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev, int lock)
228 {
229         int i;
230         struct amdgpu_hive_info *tmp;
231
232         if (!adev->gmc.xgmi.hive_id)
233                 return NULL;
234
235         mutex_lock(&xgmi_mutex);
236
237         for (i = 0 ; i < hive_count; ++i) {
238                 tmp = &xgmi_hives[i];
239                 if (tmp->hive_id == adev->gmc.xgmi.hive_id) {
240                         if (lock)
241                                 mutex_lock(&tmp->hive_lock);
242                         mutex_unlock(&xgmi_mutex);
243                         return tmp;
244                 }
245         }
246         if (i >= AMDGPU_MAX_XGMI_HIVE) {
247                 mutex_unlock(&xgmi_mutex);
248                 return NULL;
249         }
250
251         /* initialize new hive if not exist */
252         tmp = &xgmi_hives[hive_count++];
253
254         if (amdgpu_xgmi_sysfs_create(adev, tmp)) {
255                 mutex_unlock(&xgmi_mutex);
256                 return NULL;
257         }
258
259         tmp->adev = adev;
260         tmp->hive_id = adev->gmc.xgmi.hive_id;
261         INIT_LIST_HEAD(&tmp->device_list);
262         mutex_init(&tmp->hive_lock);
263         mutex_init(&tmp->reset_lock);
264         task_barrier_init(&tmp->tb);
265
266         if (lock)
267                 mutex_lock(&tmp->hive_lock);
268         tmp->pstate = -1;
269         mutex_unlock(&xgmi_mutex);
270
271         return tmp;
272 }
273
274 int amdgpu_xgmi_set_pstate(struct amdgpu_device *adev, int pstate)
275 {
276         int ret = 0;
277         struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev, 0);
278         struct amdgpu_device *tmp_adev;
279         bool update_hive_pstate = true;
280         bool is_high_pstate = pstate && adev->asic_type == CHIP_VEGA20;
281
282         if (!hive)
283                 return 0;
284
285         mutex_lock(&hive->hive_lock);
286
287         if (hive->pstate == pstate) {
288                 adev->pstate = is_high_pstate ? pstate : adev->pstate;
289                 goto out;
290         }
291
292         dev_dbg(adev->dev, "Set xgmi pstate %d.\n", pstate);
293
294         ret = amdgpu_dpm_set_xgmi_pstate(adev, pstate);
295         if (ret) {
296                 dev_err(adev->dev,
297                         "XGMI: Set pstate failure on device %llx, hive %llx, ret %d",
298                         adev->gmc.xgmi.node_id,
299                         adev->gmc.xgmi.hive_id, ret);
300                 goto out;
301         }
302
303         /* Update device pstate */
304         adev->pstate = pstate;
305
306         /*
307          * Update the hive pstate only all devices of the hive
308          * are in the same pstate
309          */
310         list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
311                 if (tmp_adev->pstate != adev->pstate) {
312                         update_hive_pstate = false;
313                         break;
314                 }
315         }
316         if (update_hive_pstate || is_high_pstate)
317                 hive->pstate = pstate;
318
319 out:
320         mutex_unlock(&hive->hive_lock);
321
322         return ret;
323 }
324
325 int amdgpu_xgmi_update_topology(struct amdgpu_hive_info *hive, struct amdgpu_device *adev)
326 {
327         int ret = -EINVAL;
328
329         /* Each psp need to set the latest topology */
330         ret = psp_xgmi_set_topology_info(&adev->psp,
331                                          hive->number_devices,
332                                          &adev->psp.xgmi_context.top_info);
333         if (ret)
334                 dev_err(adev->dev,
335                         "XGMI: Set topology failure on device %llx, hive %llx, ret %d",
336                         adev->gmc.xgmi.node_id,
337                         adev->gmc.xgmi.hive_id, ret);
338
339         return ret;
340 }
341
342
343 int amdgpu_xgmi_get_hops_count(struct amdgpu_device *adev,
344                 struct amdgpu_device *peer_adev)
345 {
346         struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
347         int i;
348
349         for (i = 0 ; i < top->num_nodes; ++i)
350                 if (top->nodes[i].node_id == peer_adev->gmc.xgmi.node_id)
351                         return top->nodes[i].num_hops;
352         return  -EINVAL;
353 }
354
355 int amdgpu_xgmi_add_device(struct amdgpu_device *adev)
356 {
357         struct psp_xgmi_topology_info *top_info;
358         struct amdgpu_hive_info *hive;
359         struct amdgpu_xgmi      *entry;
360         struct amdgpu_device *tmp_adev = NULL;
361
362         int count = 0, ret = 0;
363
364         if (!adev->gmc.xgmi.supported)
365                 return 0;
366
367         if (amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_PSP)) {
368                 ret = psp_xgmi_get_hive_id(&adev->psp, &adev->gmc.xgmi.hive_id);
369                 if (ret) {
370                         dev_err(adev->dev,
371                                 "XGMI: Failed to get hive id\n");
372                         return ret;
373                 }
374
375                 ret = psp_xgmi_get_node_id(&adev->psp, &adev->gmc.xgmi.node_id);
376                 if (ret) {
377                         dev_err(adev->dev,
378                                 "XGMI: Failed to get node id\n");
379                         return ret;
380                 }
381         } else {
382                 adev->gmc.xgmi.hive_id = 16;
383                 adev->gmc.xgmi.node_id = adev->gmc.xgmi.physical_node_id + 16;
384         }
385
386         hive = amdgpu_get_xgmi_hive(adev, 1);
387         if (!hive) {
388                 ret = -EINVAL;
389                 dev_err(adev->dev,
390                         "XGMI: node 0x%llx, can not match hive 0x%llx in the hive list.\n",
391                         adev->gmc.xgmi.node_id, adev->gmc.xgmi.hive_id);
392                 goto exit;
393         }
394
395         /* Set default device pstate */
396         adev->pstate = -1;
397
398         top_info = &adev->psp.xgmi_context.top_info;
399
400         list_add_tail(&adev->gmc.xgmi.head, &hive->device_list);
401         list_for_each_entry(entry, &hive->device_list, head)
402                 top_info->nodes[count++].node_id = entry->node_id;
403         top_info->num_nodes = count;
404         hive->number_devices = count;
405
406         task_barrier_add_task(&hive->tb);
407
408         if (amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_PSP)) {
409                 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
410                         /* update node list for other device in the hive */
411                         if (tmp_adev != adev) {
412                                 top_info = &tmp_adev->psp.xgmi_context.top_info;
413                                 top_info->nodes[count - 1].node_id =
414                                         adev->gmc.xgmi.node_id;
415                                 top_info->num_nodes = count;
416                         }
417                         ret = amdgpu_xgmi_update_topology(hive, tmp_adev);
418                         if (ret)
419                                 goto exit;
420                 }
421
422                 /* get latest topology info for each device from psp */
423                 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
424                         ret = psp_xgmi_get_topology_info(&tmp_adev->psp, count,
425                                         &tmp_adev->psp.xgmi_context.top_info);
426                         if (ret) {
427                                 dev_err(tmp_adev->dev,
428                                         "XGMI: Get topology failure on device %llx, hive %llx, ret %d",
429                                         tmp_adev->gmc.xgmi.node_id,
430                                         tmp_adev->gmc.xgmi.hive_id, ret);
431                                 /* To do : continue with some node failed or disable the whole hive */
432                                 goto exit;
433                         }
434                 }
435         }
436
437         if (!ret)
438                 ret = amdgpu_xgmi_sysfs_add_dev_info(adev, hive);
439
440
441         mutex_unlock(&hive->hive_lock);
442 exit:
443         if (!ret)
444                 dev_info(adev->dev, "XGMI: Add node %d, hive 0x%llx.\n",
445                          adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id);
446         else
447                 dev_err(adev->dev, "XGMI: Failed to add node %d, hive 0x%llx ret: %d\n",
448                         adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id,
449                         ret);
450
451         return ret;
452 }
453
454 void amdgpu_xgmi_remove_device(struct amdgpu_device *adev)
455 {
456         struct amdgpu_hive_info *hive;
457
458         if (!adev->gmc.xgmi.supported)
459                 return;
460
461         hive = amdgpu_get_xgmi_hive(adev, 1);
462         if (!hive)
463                 return;
464
465         if (!(hive->number_devices--)) {
466                 amdgpu_xgmi_sysfs_destroy(adev, hive);
467                 mutex_destroy(&hive->hive_lock);
468                 mutex_destroy(&hive->reset_lock);
469         } else {
470                 task_barrier_rem_task(&hive->tb);
471                 amdgpu_xgmi_sysfs_rem_dev_info(adev, hive);
472                 mutex_unlock(&hive->hive_lock);
473         }
474 }
475
476 int amdgpu_xgmi_ras_late_init(struct amdgpu_device *adev)
477 {
478         int r;
479         struct ras_ih_if ih_info = {
480                 .cb = NULL,
481         };
482         struct ras_fs_if fs_info = {
483                 .sysfs_name = "xgmi_wafl_err_count",
484                 .debugfs_name = "xgmi_wafl_err_inject",
485         };
486
487         if (!adev->gmc.xgmi.supported ||
488             adev->gmc.xgmi.num_physical_nodes == 0)
489                 return 0;
490
491         if (!adev->gmc.xgmi.ras_if) {
492                 adev->gmc.xgmi.ras_if = kmalloc(sizeof(struct ras_common_if), GFP_KERNEL);
493                 if (!adev->gmc.xgmi.ras_if)
494                         return -ENOMEM;
495                 adev->gmc.xgmi.ras_if->block = AMDGPU_RAS_BLOCK__XGMI_WAFL;
496                 adev->gmc.xgmi.ras_if->type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
497                 adev->gmc.xgmi.ras_if->sub_block_index = 0;
498                 strcpy(adev->gmc.xgmi.ras_if->name, "xgmi_wafl");
499         }
500         ih_info.head = fs_info.head = *adev->gmc.xgmi.ras_if;
501         r = amdgpu_ras_late_init(adev, adev->gmc.xgmi.ras_if,
502                                  &fs_info, &ih_info);
503         if (r || !amdgpu_ras_is_supported(adev, adev->gmc.xgmi.ras_if->block)) {
504                 kfree(adev->gmc.xgmi.ras_if);
505                 adev->gmc.xgmi.ras_if = NULL;
506         }
507
508         return r;
509 }
510
511 void amdgpu_xgmi_ras_fini(struct amdgpu_device *adev)
512 {
513         if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__XGMI_WAFL) &&
514                         adev->gmc.xgmi.ras_if) {
515                 struct ras_common_if *ras_if = adev->gmc.xgmi.ras_if;
516                 struct ras_ih_if ih_info = {
517                         .cb = NULL,
518                 };
519
520                 amdgpu_ras_late_fini(adev, ras_if, &ih_info);
521                 kfree(ras_if);
522         }
523 }