1 // SPDX-License-Identifier: MIT
3 * Copyright © 2020 Intel Corporation
6 #include <linux/sort.h>
8 #include "selftests/i915_random.h"
10 static const unsigned int sizes[] = {
20 static struct drm_i915_gem_object *
21 create_lmem_or_internal(struct drm_i915_private *i915, size_t size)
23 struct drm_i915_gem_object *obj;
25 obj = i915_gem_object_create_lmem(i915, size, 0);
29 return i915_gem_object_create_internal(i915, size);
32 static int copy(struct intel_migrate *migrate,
33 int (*fn)(struct intel_migrate *migrate,
34 struct i915_gem_ww_ctx *ww,
35 struct drm_i915_gem_object *src,
36 struct drm_i915_gem_object *dst,
37 struct i915_request **out),
38 u32 sz, struct rnd_state *prng)
40 struct drm_i915_private *i915 = migrate->context->engine->i915;
41 struct drm_i915_gem_object *src, *dst;
42 struct i915_request *rq;
43 struct i915_gem_ww_ctx ww;
48 src = create_lmem_or_internal(i915, sz);
52 dst = i915_gem_object_create_internal(i915, sz);
56 for_i915_gem_ww(&ww, err, true) {
57 err = i915_gem_object_lock(src, &ww);
61 err = i915_gem_object_lock(dst, &ww);
65 vaddr = i915_gem_object_pin_map(src, I915_MAP_WC);
71 for (i = 0; i < sz / sizeof(u32); i++)
73 i915_gem_object_flush_map(src);
75 vaddr = i915_gem_object_pin_map(dst, I915_MAP_WC);
81 for (i = 0; i < sz / sizeof(u32); i++)
83 i915_gem_object_flush_map(dst);
85 err = fn(migrate, &ww, src, dst, &rq);
89 if (err != -EDEADLK && err != -EINTR && err != -ERESTARTSYS)
90 pr_err("%ps failed, size: %u\n", fn, sz);
92 i915_request_wait(rq, 0, HZ);
95 i915_gem_object_unpin_map(dst);
97 i915_gem_object_unpin_map(src);
103 if (i915_request_wait(rq, 0, HZ) < 0) {
104 pr_err("%ps timed out, size: %u\n", fn, sz);
107 i915_request_put(rq);
110 for (i = 0; !err && i < sz / PAGE_SIZE; i++) {
111 int x = i * 1024 + i915_prandom_u32_max_state(1024, prng);
114 pr_err("%ps failed, size: %u, offset: %zu\n",
115 fn, sz, x * sizeof(u32));
116 igt_hexdump(vaddr + i * 1024, 4096);
121 i915_gem_object_unpin_map(dst);
122 i915_gem_object_unpin_map(src);
125 i915_gem_object_put(dst);
127 i915_gem_object_put(src);
132 static int clear(struct intel_migrate *migrate,
133 int (*fn)(struct intel_migrate *migrate,
134 struct i915_gem_ww_ctx *ww,
135 struct drm_i915_gem_object *obj,
137 struct i915_request **out),
138 u32 sz, struct rnd_state *prng)
140 struct drm_i915_private *i915 = migrate->context->engine->i915;
141 struct drm_i915_gem_object *obj;
142 struct i915_request *rq;
143 struct i915_gem_ww_ctx ww;
148 obj = create_lmem_or_internal(i915, sz);
152 for_i915_gem_ww(&ww, err, true) {
153 err = i915_gem_object_lock(obj, &ww);
157 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC);
159 err = PTR_ERR(vaddr);
163 for (i = 0; i < sz / sizeof(u32); i++)
165 i915_gem_object_flush_map(obj);
167 err = fn(migrate, &ww, obj, sz, &rq);
171 if (err != -EDEADLK && err != -EINTR && err != -ERESTARTSYS)
172 pr_err("%ps failed, size: %u\n", fn, sz);
174 i915_request_wait(rq, 0, HZ);
175 i915_request_put(rq);
177 i915_gem_object_unpin_map(obj);
183 if (i915_request_wait(rq, 0, HZ) < 0) {
184 pr_err("%ps timed out, size: %u\n", fn, sz);
187 i915_request_put(rq);
190 for (i = 0; !err && i < sz / PAGE_SIZE; i++) {
191 int x = i * 1024 + i915_prandom_u32_max_state(1024, prng);
193 if (vaddr[x] != sz) {
194 pr_err("%ps failed, size: %u, offset: %zu\n",
195 fn, sz, x * sizeof(u32));
196 igt_hexdump(vaddr + i * 1024, 4096);
201 i915_gem_object_unpin_map(obj);
203 i915_gem_object_put(obj);
208 static int __migrate_copy(struct intel_migrate *migrate,
209 struct i915_gem_ww_ctx *ww,
210 struct drm_i915_gem_object *src,
211 struct drm_i915_gem_object *dst,
212 struct i915_request **out)
214 return intel_migrate_copy(migrate, ww, NULL,
215 src->mm.pages->sgl, src->cache_level,
216 i915_gem_object_is_lmem(src),
217 dst->mm.pages->sgl, dst->cache_level,
218 i915_gem_object_is_lmem(dst),
222 static int __global_copy(struct intel_migrate *migrate,
223 struct i915_gem_ww_ctx *ww,
224 struct drm_i915_gem_object *src,
225 struct drm_i915_gem_object *dst,
226 struct i915_request **out)
228 return intel_context_migrate_copy(migrate->context, NULL,
229 src->mm.pages->sgl, src->cache_level,
230 i915_gem_object_is_lmem(src),
231 dst->mm.pages->sgl, dst->cache_level,
232 i915_gem_object_is_lmem(dst),
237 migrate_copy(struct intel_migrate *migrate, u32 sz, struct rnd_state *prng)
239 return copy(migrate, __migrate_copy, sz, prng);
243 global_copy(struct intel_migrate *migrate, u32 sz, struct rnd_state *prng)
245 return copy(migrate, __global_copy, sz, prng);
248 static int __migrate_clear(struct intel_migrate *migrate,
249 struct i915_gem_ww_ctx *ww,
250 struct drm_i915_gem_object *obj,
252 struct i915_request **out)
254 return intel_migrate_clear(migrate, ww, NULL,
257 i915_gem_object_is_lmem(obj),
261 static int __global_clear(struct intel_migrate *migrate,
262 struct i915_gem_ww_ctx *ww,
263 struct drm_i915_gem_object *obj,
265 struct i915_request **out)
267 return intel_context_migrate_clear(migrate->context, NULL,
270 i915_gem_object_is_lmem(obj),
275 migrate_clear(struct intel_migrate *migrate, u32 sz, struct rnd_state *prng)
277 return clear(migrate, __migrate_clear, sz, prng);
281 global_clear(struct intel_migrate *migrate, u32 sz, struct rnd_state *prng)
283 return clear(migrate, __global_clear, sz, prng);
286 static int live_migrate_copy(void *arg)
288 struct intel_migrate *migrate = arg;
289 struct drm_i915_private *i915 = migrate->context->engine->i915;
290 I915_RND_STATE(prng);
293 for (i = 0; i < ARRAY_SIZE(sizes); i++) {
296 err = migrate_copy(migrate, sizes[i], &prng);
298 err = global_copy(migrate, sizes[i], &prng);
299 i915_gem_drain_freed_objects(i915);
307 static int live_migrate_clear(void *arg)
309 struct intel_migrate *migrate = arg;
310 struct drm_i915_private *i915 = migrate->context->engine->i915;
311 I915_RND_STATE(prng);
314 for (i = 0; i < ARRAY_SIZE(sizes); i++) {
317 err = migrate_clear(migrate, sizes[i], &prng);
319 err = global_clear(migrate, sizes[i], &prng);
321 i915_gem_drain_freed_objects(i915);
329 struct threaded_migrate {
330 struct intel_migrate *migrate;
331 struct task_struct *tsk;
332 struct rnd_state prng;
335 static int threaded_migrate(struct intel_migrate *migrate,
336 int (*fn)(void *arg),
339 const unsigned int n_cpus = num_online_cpus() + 1;
340 struct threaded_migrate *thread;
341 I915_RND_STATE(prng);
345 thread = kcalloc(n_cpus, sizeof(*thread), GFP_KERNEL);
349 for (i = 0; i < n_cpus; ++i) {
350 struct task_struct *tsk;
352 thread[i].migrate = migrate;
354 I915_RND_STATE_INITIALIZER(prandom_u32_state(&prng));
356 tsk = kthread_run(fn, &thread[i], "igt-%d", i);
362 get_task_struct(tsk);
366 msleep(10); /* start all threads before we kthread_stop() */
368 for (i = 0; i < n_cpus; ++i) {
369 struct task_struct *tsk = thread[i].tsk;
372 if (IS_ERR_OR_NULL(tsk))
375 status = kthread_stop(tsk);
379 put_task_struct(tsk);
386 static int __thread_migrate_copy(void *arg)
388 struct threaded_migrate *tm = arg;
390 return migrate_copy(tm->migrate, 2 * CHUNK_SZ, &tm->prng);
393 static int thread_migrate_copy(void *arg)
395 return threaded_migrate(arg, __thread_migrate_copy, 0);
398 static int __thread_global_copy(void *arg)
400 struct threaded_migrate *tm = arg;
402 return global_copy(tm->migrate, 2 * CHUNK_SZ, &tm->prng);
405 static int thread_global_copy(void *arg)
407 return threaded_migrate(arg, __thread_global_copy, 0);
410 static int __thread_migrate_clear(void *arg)
412 struct threaded_migrate *tm = arg;
414 return migrate_clear(tm->migrate, 2 * CHUNK_SZ, &tm->prng);
417 static int __thread_global_clear(void *arg)
419 struct threaded_migrate *tm = arg;
421 return global_clear(tm->migrate, 2 * CHUNK_SZ, &tm->prng);
424 static int thread_migrate_clear(void *arg)
426 return threaded_migrate(arg, __thread_migrate_clear, 0);
429 static int thread_global_clear(void *arg)
431 return threaded_migrate(arg, __thread_global_clear, 0);
434 int intel_migrate_live_selftests(struct drm_i915_private *i915)
436 static const struct i915_subtest tests[] = {
437 SUBTEST(live_migrate_copy),
438 SUBTEST(live_migrate_clear),
439 SUBTEST(thread_migrate_copy),
440 SUBTEST(thread_migrate_clear),
441 SUBTEST(thread_global_copy),
442 SUBTEST(thread_global_clear),
444 struct intel_gt *gt = &i915->gt;
446 if (!gt->migrate.context)
449 return i915_subtests(tests, >->migrate);
452 static struct drm_i915_gem_object *
453 create_init_lmem_internal(struct intel_gt *gt, size_t sz, bool try_lmem)
455 struct drm_i915_gem_object *obj = NULL;
459 obj = i915_gem_object_create_lmem(gt->i915, sz, 0);
461 if (IS_ERR_OR_NULL(obj)) {
462 obj = i915_gem_object_create_internal(gt->i915, sz);
467 i915_gem_object_trylock(obj);
468 err = i915_gem_object_pin_pages(obj);
470 i915_gem_object_unlock(obj);
471 i915_gem_object_put(obj);
478 static int wrap_ktime_compare(const void *A, const void *B)
480 const ktime_t *a = A, *b = B;
482 return ktime_compare(*a, *b);
485 static int __perf_clear_blt(struct intel_context *ce,
486 struct scatterlist *sg,
487 enum i915_cache_level cache_level,
495 for (pass = 0; pass < ARRAY_SIZE(t); pass++) {
496 struct i915_request *rq;
501 err = intel_context_migrate_clear(ce, NULL, sg, cache_level,
504 if (i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT) < 0)
506 i915_request_put(rq);
512 t[pass] = ktime_sub(t1, t0);
517 sort(t, ARRAY_SIZE(t), sizeof(*t), wrap_ktime_compare, NULL);
518 pr_info("%s: %zd KiB fill: %lld MiB/s\n",
519 ce->engine->name, sz >> 10,
520 div64_u64(mul_u32_u32(4 * sz,
522 t[1] + 2 * t[2] + t[3]) >> 20);
526 static int perf_clear_blt(void *arg)
528 struct intel_gt *gt = arg;
529 static const unsigned long sizes[] = {
537 for (i = 0; i < ARRAY_SIZE(sizes); i++) {
538 struct drm_i915_gem_object *dst;
541 dst = create_init_lmem_internal(gt, sizes[i], true);
545 err = __perf_clear_blt(gt->migrate.context,
548 i915_gem_object_is_lmem(dst),
551 i915_gem_object_unlock(dst);
552 i915_gem_object_put(dst);
560 static int __perf_copy_blt(struct intel_context *ce,
561 struct scatterlist *src,
562 enum i915_cache_level src_cache_level,
564 struct scatterlist *dst,
565 enum i915_cache_level dst_cache_level,
573 for (pass = 0; pass < ARRAY_SIZE(t); pass++) {
574 struct i915_request *rq;
579 err = intel_context_migrate_copy(ce, NULL,
580 src, src_cache_level,
582 dst, dst_cache_level,
586 if (i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT) < 0)
588 i915_request_put(rq);
594 t[pass] = ktime_sub(t1, t0);
599 sort(t, ARRAY_SIZE(t), sizeof(*t), wrap_ktime_compare, NULL);
600 pr_info("%s: %zd KiB copy: %lld MiB/s\n",
601 ce->engine->name, sz >> 10,
602 div64_u64(mul_u32_u32(4 * sz,
604 t[1] + 2 * t[2] + t[3]) >> 20);
608 static int perf_copy_blt(void *arg)
610 struct intel_gt *gt = arg;
611 static const unsigned long sizes[] = {
619 for (i = 0; i < ARRAY_SIZE(sizes); i++) {
620 struct drm_i915_gem_object *src, *dst;
623 src = create_init_lmem_internal(gt, sizes[i], true);
627 dst = create_init_lmem_internal(gt, sizes[i], false);
633 err = __perf_copy_blt(gt->migrate.context,
636 i915_gem_object_is_lmem(src),
639 i915_gem_object_is_lmem(dst),
642 i915_gem_object_unlock(dst);
643 i915_gem_object_put(dst);
645 i915_gem_object_unlock(src);
646 i915_gem_object_put(src);
654 int intel_migrate_perf_selftests(struct drm_i915_private *i915)
656 static const struct i915_subtest tests[] = {
657 SUBTEST(perf_clear_blt),
658 SUBTEST(perf_copy_blt),
660 struct intel_gt *gt = &i915->gt;
662 if (intel_gt_is_wedged(gt))
665 if (!gt->migrate.context)
668 return intel_gt_live_subtests(tests, gt);