drivers/dma/dmatest.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * DMA Engine test module
   4  *
   5  * Copyright (C) 2007 Atmel Corporation
   6  * Copyright (C) 2013 Intel Corporation
   7  */
   8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   9
  10 #include <linux/delay.h>
  11 #include <linux/dma-mapping.h>
  12 #include <linux/dmaengine.h>
  13 #include <linux/freezer.h>
  14 #include <linux/init.h>
  15 #include <linux/kthread.h>
  16 #include <linux/sched/task.h>
  17 #include <linux/module.h>
  18 #include <linux/moduleparam.h>
  19 #include <linux/random.h>
  20 #include <linux/slab.h>
  21 #include <linux/wait.h>
  22
  23 static unsigned int test_buf_size = 16384;
  24 module_param(test_buf_size, uint, S_IRUGO | S_IWUSR);
  25 MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer");
  26
  27 static char test_device[32];
  28 module_param_string(device, test_device, sizeof(test_device),
  29                 S_IRUGO | S_IWUSR);
  30 MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)");
  31
  32 static unsigned int threads_per_chan = 1;
  33 module_param(threads_per_chan, uint, S_IRUGO | S_IWUSR);
  34 MODULE_PARM_DESC(threads_per_chan,
  35                 "Number of threads to start per channel (default: 1)");
  36
  37 static unsigned int max_channels;
  38 module_param(max_channels, uint, S_IRUGO | S_IWUSR);
  39 MODULE_PARM_DESC(max_channels,
  40                 "Maximum number of channels to use (default: all)");
  41
  42 static unsigned int iterations;
  43 module_param(iterations, uint, S_IRUGO | S_IWUSR);
  44 MODULE_PARM_DESC(iterations,
  45                 "Iterations before stopping test (default: infinite)");
  46
  47 static unsigned int dmatest;
  48 module_param(dmatest, uint, S_IRUGO | S_IWUSR);
  49 MODULE_PARM_DESC(dmatest,
  50                 "dmatest 0-memcpy 1-memset (default: 0)");
  51
  52 static unsigned int xor_sources = 3;
  53 module_param(xor_sources, uint, S_IRUGO | S_IWUSR);
  54 MODULE_PARM_DESC(xor_sources,
  55                 "Number of xor source buffers (default: 3)");
  56
  57 static unsigned int pq_sources = 3;
  58 module_param(pq_sources, uint, S_IRUGO | S_IWUSR);
  59 MODULE_PARM_DESC(pq_sources,
  60                 "Number of p+q source buffers (default: 3)");
  61
  62 static int timeout = 3000;
  63 module_param(timeout, uint, S_IRUGO | S_IWUSR);
  64 MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), "
  65                  "Pass 0xFFFFFFFF (4294967295) for maximum timeout");
  66
  67 static bool noverify;
  68 module_param(noverify, bool, S_IRUGO | S_IWUSR);
  69 MODULE_PARM_DESC(noverify, "Disable data verification (default: verify)");
  70
  71 static bool norandom;
  72 module_param(norandom, bool, 0644);
  73 MODULE_PARM_DESC(norandom, "Disable random offset setup (default: random)");
  74
  75 static bool polled;
  76 module_param(polled, bool, S_IRUGO | S_IWUSR);
  77 MODULE_PARM_DESC(polled, "Use polling for completion instead of interrupts");
  78
  79 static bool verbose;
  80 module_param(verbose, bool, S_IRUGO | S_IWUSR);
  81 MODULE_PARM_DESC(verbose, "Enable \"success\" result messages (default: off)");
  82
  83 static int alignment = -1;
  84 module_param(alignment, int, 0644);
  85 MODULE_PARM_DESC(alignment, "Custom data address alignment taken as 2^(alignment) (default: not used (-1))");
  86
  87 static unsigned int transfer_size;
  88 module_param(transfer_size, uint, 0644);
  89 MODULE_PARM_DESC(transfer_size, "Optional custom transfer size in bytes (default: not used (0))");
  90
  91 /**
  92  * struct dmatest_params - test parameters.
  93  * @buf_size:           size of the memcpy test buffer
  94  * @channel:            bus ID of the channel to test
  95  * @device:             bus ID of the DMA Engine to test
  96  * @threads_per_chan:   number of threads to start per channel
  97  * @max_channels:       maximum number of channels to use
  98  * @iterations:         iterations before stopping test
  99  * @xor_sources:        number of xor source buffers
 100  * @pq_sources:         number of p+q source buffers
 101  * @timeout:            transfer timeout in msec, 0 - 0xFFFFFFFF (4294967295)
 102  */
 103 struct dmatest_params {
 104         unsigned int    buf_size;
 105         char            channel[20];
 106         char            device[32];
 107         unsigned int    threads_per_chan;
 108         unsigned int    max_channels;
 109         unsigned int    iterations;
 110         unsigned int    xor_sources;
 111         unsigned int    pq_sources;
 112         unsigned int    timeout;
 113         bool            noverify;
 114         bool            norandom;
 115         int             alignment;
 116         unsigned int    transfer_size;
 117         bool            polled;
 118 };
 119
 120 /**
 121  * struct dmatest_info - test information.
 122  * @params:             test parameters
 123  * @lock:               access protection to the fields of this structure
 124  */
 125 static struct dmatest_info {
 126         /* Test parameters */
 127         struct dmatest_params   params;
 128
 129         /* Internal state */
 130         struct list_head        channels;
 131         unsigned int            nr_channels;
 132         struct mutex            lock;
 133         bool                    did_init;
 134 } test_info = {
 135         .channels = LIST_HEAD_INIT(test_info.channels),
 136         .lock = __MUTEX_INITIALIZER(test_info.lock),
 137 };
 138
 139 static int dmatest_run_set(const char *val, const struct kernel_param *kp);
 140 static int dmatest_run_get(char *val, const struct kernel_param *kp);
 141 static const struct kernel_param_ops run_ops = {
 142         .set = dmatest_run_set,
 143         .get = dmatest_run_get,
 144 };
 145 static bool dmatest_run;
 146 module_param_cb(run, &run_ops, &dmatest_run, S_IRUGO | S_IWUSR);
 147 MODULE_PARM_DESC(run, "Run the test (default: false)");
 148
 149 static int dmatest_chan_set(const char *val, const struct kernel_param *kp);
 150 static int dmatest_chan_get(char *val, const struct kernel_param *kp);
 151 static const struct kernel_param_ops multi_chan_ops = {
 152         .set = dmatest_chan_set,
 153         .get = dmatest_chan_get,
 154 };
 155
 156 static char test_channel[20];
 157 static struct kparam_string newchan_kps = {
 158         .string = test_channel,
 159         .maxlen = 20,
 160 };
 161 module_param_cb(channel, &multi_chan_ops, &newchan_kps, 0644);
 162 MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)");
 163
 164 static int dmatest_test_list_get(char *val, const struct kernel_param *kp);
 165 static const struct kernel_param_ops test_list_ops = {
 166         .get = dmatest_test_list_get,
 167 };
 168 module_param_cb(test_list, &test_list_ops, NULL, 0444);
 169 MODULE_PARM_DESC(test_list, "Print current test list");
 170
 171 /* Maximum amount of mismatched bytes in buffer to print */
 172 #define MAX_ERROR_COUNT         32
 173
 174 /*
 175  * Initialization patterns. All bytes in the source buffer has bit 7
 176  * set, all bytes in the destination buffer has bit 7 cleared.
 177  *
 178  * Bit 6 is set for all bytes which are to be copied by the DMA
 179  * engine. Bit 5 is set for all bytes which are to be overwritten by
 180  * the DMA engine.
 181  *
 182  * The remaining bits are the inverse of a counter which increments by
 183  * one for each byte address.
 184  */
 185 #define PATTERN_SRC             0x80
 186 #define PATTERN_DST             0x00
 187 #define PATTERN_COPY            0x40
 188 #define PATTERN_OVERWRITE       0x20
 189 #define PATTERN_COUNT_MASK      0x1f
 190 #define PATTERN_MEMSET_IDX      0x01
 191
 192 /* Fixed point arithmetic ops */
 193 #define FIXPT_SHIFT             8
 194 #define FIXPNT_MASK             0xFF
 195 #define FIXPT_TO_INT(a) ((a) >> FIXPT_SHIFT)
 196 #define INT_TO_FIXPT(a) ((a) << FIXPT_SHIFT)
 197 #define FIXPT_GET_FRAC(a)       ((((a) & FIXPNT_MASK) * 100) >> FIXPT_SHIFT)
 198
 199 /* poor man's completion - we want to use wait_event_freezable() on it */
 200 struct dmatest_done {
 201         bool                    done;
 202         wait_queue_head_t       *wait;
 203 };
 204
 205 struct dmatest_data {
 206         u8              **raw;
 207         u8              **aligned;
 208         unsigned int    cnt;
 209         unsigned int    off;
 210 };
 211
 212 struct dmatest_thread {
 213         struct list_head        node;
 214         struct dmatest_info     *info;
 215         struct task_struct      *task;
 216         struct dma_chan         *chan;
 217         struct dmatest_data     src;
 218         struct dmatest_data     dst;
 219         enum dma_transaction_type type;
 220         wait_queue_head_t done_wait;
 221         struct dmatest_done test_done;
 222         bool                    done;
 223         bool                    pending;
 224 };
 225
 226 struct dmatest_chan {
 227         struct list_head        node;
 228         struct dma_chan         *chan;
 229         struct list_head        threads;
 230 };
 231
 232 static DECLARE_WAIT_QUEUE_HEAD(thread_wait);
 233 static bool wait;
 234
 235 static bool is_threaded_test_run(struct dmatest_info *info)
 236 {
 237         struct dmatest_chan *dtc;
 238
 239         list_for_each_entry(dtc, &info->channels, node) {
 240                 struct dmatest_thread *thread;
 241
 242                 list_for_each_entry(thread, &dtc->threads, node) {
 243                         if (!thread->done && !thread->pending)
 244                                 return true;
 245                 }
 246         }
 247
 248         return false;
 249 }
 250
 251 static bool is_threaded_test_pending(struct dmatest_info *info)
 252 {
 253         struct dmatest_chan *dtc;
 254
 255         list_for_each_entry(dtc, &info->channels, node) {
 256                 struct dmatest_thread *thread;
 257
 258                 list_for_each_entry(thread, &dtc->threads, node) {
 259                         if (thread->pending)
 260                                 return true;
 261                 }
 262         }
 263
 264         return false;
 265 }
 266
 267 static int dmatest_wait_get(char *val, const struct kernel_param *kp)
 268 {
 269         struct dmatest_info *info = &test_info;
 270         struct dmatest_params *params = &info->params;
 271
 272         if (params->iterations)
 273                 wait_event(thread_wait, !is_threaded_test_run(info));
 274         wait = true;
 275         return param_get_bool(val, kp);
 276 }
 277
 278 static const struct kernel_param_ops wait_ops = {
 279         .get = dmatest_wait_get,
 280         .set = param_set_bool,
 281 };
 282 module_param_cb(wait, &wait_ops, &wait, S_IRUGO);
 283 MODULE_PARM_DESC(wait, "Wait for tests to complete (default: false)");
 284
 285 static bool dmatest_match_channel(struct dmatest_params *params,
 286                 struct dma_chan *chan)
 287 {
 288         if (params->channel[0] == '\0')
 289                 return true;
 290         return strcmp(dma_chan_name(chan), params->channel) == 0;
 291 }
 292
 293 static bool dmatest_match_device(struct dmatest_params *params,
 294                 struct dma_device *device)
 295 {
 296         if (params->device[0] == '\0')
 297                 return true;
 298         return strcmp(dev_name(device->dev), params->device) == 0;
 299 }
 300
 301 static unsigned long dmatest_random(void)
 302 {
 303         unsigned long buf;
 304
 305         prandom_bytes(&buf, sizeof(buf));
 306         return buf;
 307 }
 308
 309 static inline u8 gen_inv_idx(u8 index, bool is_memset)
 310 {
 311         u8 val = is_memset ? PATTERN_MEMSET_IDX : index;
 312
 313         return ~val & PATTERN_COUNT_MASK;
 314 }
 315
 316 static inline u8 gen_src_value(u8 index, bool is_memset)
 317 {
 318         return PATTERN_SRC | gen_inv_idx(index, is_memset);
 319 }
 320
 321 static inline u8 gen_dst_value(u8 index, bool is_memset)
 322 {
 323         return PATTERN_DST | gen_inv_idx(index, is_memset);
 324 }
 325
 326 static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len,
 327                 unsigned int buf_size, bool is_memset)
 328 {
 329         unsigned int i;
 330         u8 *buf;
 331
 332         for (; (buf = *bufs); bufs++) {
 333                 for (i = 0; i < start; i++)
 334                         buf[i] = gen_src_value(i, is_memset);
 335                 for ( ; i < start + len; i++)
 336                         buf[i] = gen_src_value(i, is_memset) | PATTERN_COPY;
 337                 for ( ; i < buf_size; i++)
 338                         buf[i] = gen_src_value(i, is_memset);
 339                 buf++;
 340         }
 341 }
 342
 343 static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len,
 344                 unsigned int buf_size, bool is_memset)
 345 {
 346         unsigned int i;
 347         u8 *buf;
 348
 349         for (; (buf = *bufs); bufs++) {
 350                 for (i = 0; i < start; i++)
 351                         buf[i] = gen_dst_value(i, is_memset);
 352                 for ( ; i < start + len; i++)
 353                         buf[i] = gen_dst_value(i, is_memset) |
 354                                                 PATTERN_OVERWRITE;
 355                 for ( ; i < buf_size; i++)
 356                         buf[i] = gen_dst_value(i, is_memset);
 357         }
 358 }
 359
 360 static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index,
 361                 unsigned int counter, bool is_srcbuf, bool is_memset)
 362 {
 363         u8              diff = actual ^ pattern;
 364         u8              expected = pattern | gen_inv_idx(counter, is_memset);
 365         const char      *thread_name = current->comm;
 366
 367         if (is_srcbuf)
 368                 pr_warn("%s: srcbuf[0x%x] overwritten! Expected %02x, got %02x\n",
 369                         thread_name, index, expected, actual);
 370         else if ((pattern & PATTERN_COPY)
 371                         && (diff & (PATTERN_COPY | PATTERN_OVERWRITE)))
 372                 pr_warn("%s: dstbuf[0x%x] not copied! Expected %02x, got %02x\n",
 373                         thread_name, index, expected, actual);
 374         else if (diff & PATTERN_SRC)
 375                 pr_warn("%s: dstbuf[0x%x] was copied! Expected %02x, got %02x\n",
 376                         thread_name, index, expected, actual);
 377         else
 378                 pr_warn("%s: dstbuf[0x%x] mismatch! Expected %02x, got %02x\n",
 379                         thread_name, index, expected, actual);
 380 }
 381
 382 static unsigned int dmatest_verify(u8 **bufs, unsigned int start,
 383                 unsigned int end, unsigned int counter, u8 pattern,
 384                 bool is_srcbuf, bool is_memset)
 385 {
 386         unsigned int i;
 387         unsigned int error_count = 0;
 388         u8 actual;
 389         u8 expected;
 390         u8 *buf;
 391         unsigned int counter_orig = counter;
 392
 393         for (; (buf = *bufs); bufs++) {
 394                 counter = counter_orig;
 395                 for (i = start; i < end; i++) {
 396                         actual = buf[i];
 397                         expected = pattern | gen_inv_idx(counter, is_memset);
 398                         if (actual != expected) {
 399                                 if (error_count < MAX_ERROR_COUNT)
 400                                         dmatest_mismatch(actual, pattern, i,
 401                                                          counter, is_srcbuf,
 402                                                          is_memset);
 403                                 error_count++;
 404                         }
 405                         counter++;
 406                 }
 407         }
 408
 409         if (error_count > MAX_ERROR_COUNT)
 410                 pr_warn("%s: %u errors suppressed\n",
 411                         current->comm, error_count - MAX_ERROR_COUNT);
 412
 413         return error_count;
 414 }
 415
 416
 417 static void dmatest_callback(void *arg)
 418 {
 419         struct dmatest_done *done = arg;
 420         struct dmatest_thread *thread =
 421                 container_of(done, struct dmatest_thread, test_done);
 422         if (!thread->done) {
 423                 done->done = true;
 424                 wake_up_all(done->wait);
 425         } else {
 426                 /*
 427                  * If thread->done, it means that this callback occurred
 428                  * after the parent thread has cleaned up. This can
 429                  * happen in the case that driver doesn't implement
 430                  * the terminate_all() functionality and a dma operation
 431                  * did not occur within the timeout period
 432                  */
 433                 WARN(1, "dmatest: Kernel memory may be corrupted!!\n");
 434         }
 435 }
 436
 437 static unsigned int min_odd(unsigned int x, unsigned int y)
 438 {
 439         unsigned int val = min(x, y);
 440
 441         return val % 2 ? val : val - 1;
 442 }
 443
 444 static void result(const char *err, unsigned int n, unsigned int src_off,
 445                    unsigned int dst_off, unsigned int len, unsigned long data)
 446 {
 447         pr_info("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
 448                 current->comm, n, err, src_off, dst_off, len, data);
 449 }
 450
 451 static void dbg_result(const char *err, unsigned int n, unsigned int src_off,
 452                        unsigned int dst_off, unsigned int len,
 453                        unsigned long data)
 454 {
 455         pr_debug("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
 456                  current->comm, n, err, src_off, dst_off, len, data);
 457 }
 458
 459 #define verbose_result(err, n, src_off, dst_off, len, data) ({  \
 460         if (verbose)                                            \
 461                 result(err, n, src_off, dst_off, len, data);    \
 462         else                                                    \
 463                 dbg_result(err, n, src_off, dst_off, len, data);\
 464 })
 465
 466 static unsigned long long dmatest_persec(s64 runtime, unsigned int val)
 467 {
 468         unsigned long long per_sec = 1000000;
 469
 470         if (runtime <= 0)
 471                 return 0;
 472
 473         /* drop precision until runtime is 32-bits */
 474         while (runtime > UINT_MAX) {
 475                 runtime >>= 1;
 476                 per_sec <<= 1;
 477         }
 478
 479         per_sec *= val;
 480         per_sec = INT_TO_FIXPT(per_sec);
 481         do_div(per_sec, runtime);
 482
 483         return per_sec;
 484 }
 485
 486 static unsigned long long dmatest_KBs(s64 runtime, unsigned long long len)
 487 {
 488         return FIXPT_TO_INT(dmatest_persec(runtime, len >> 10));
 489 }
 490
 491 static void __dmatest_free_test_data(struct dmatest_data *d, unsigned int cnt)
 492 {
 493         unsigned int i;
 494
 495         for (i = 0; i < cnt; i++)
 496                 kfree(d->raw[i]);
 497
 498         kfree(d->aligned);
 499         kfree(d->raw);
 500 }
 501
 502 static void dmatest_free_test_data(struct dmatest_data *d)
 503 {
 504         __dmatest_free_test_data(d, d->cnt);
 505 }
 506
 507 static int dmatest_alloc_test_data(struct dmatest_data *d,
 508                 unsigned int buf_size, u8 align)
 509 {
 510         unsigned int i = 0;
 511
 512         d->raw = kcalloc(d->cnt + 1, sizeof(u8 *), GFP_KERNEL);
 513         if (!d->raw)
 514                 return -ENOMEM;
 515
 516         d->aligned = kcalloc(d->cnt + 1, sizeof(u8 *), GFP_KERNEL);
 517         if (!d->aligned)
 518                 goto err;
 519
 520         for (i = 0; i < d->cnt; i++) {
 521                 d->raw[i] = kmalloc(buf_size + align, GFP_KERNEL);
 522                 if (!d->raw[i])
 523                         goto err;
 524
 525                 /* align to alignment restriction */
 526                 if (align)
 527                         d->aligned[i] = PTR_ALIGN(d->raw[i], align);
 528                 else
 529                         d->aligned[i] = d->raw[i];
 530         }
 531
 532         return 0;
 533 err:
 534         __dmatest_free_test_data(d, i);
 535         return -ENOMEM;
 536 }
 537
 538 /*
 539  * This function repeatedly tests DMA transfers of various lengths and
 540  * offsets for a given operation type until it is told to exit by
 541  * kthread_stop(). There may be multiple threads running this function
 542  * in parallel for a single channel, and there may be multiple channels
 543  * being tested in parallel.
 544  *
 545  * Before each test, the source and destination buffer is initialized
 546  * with a known pattern. This pattern is different depending on
 547  * whether it's in an area which is supposed to be copied or
 548  * overwritten, and different in the source and destination buffers.
 549  * So if the DMA engine doesn't copy exactly what we tell it to copy,
 550  * we'll notice.
 551  */
 552 static int dmatest_func(void *data)
 553 {
 554         struct dmatest_thread   *thread = data;
 555         struct dmatest_done     *done = &thread->test_done;
 556         struct dmatest_info     *info;
 557         struct dmatest_params   *params;
 558         struct dma_chan         *chan;
 559         struct dma_device       *dev;
 560         unsigned int            error_count;
 561         unsigned int            failed_tests = 0;
 562         unsigned int            total_tests = 0;
 563         dma_cookie_t            cookie;
 564         enum dma_status         status;
 565         enum dma_ctrl_flags     flags;
 566         u8                      *pq_coefs = NULL;
 567         int                     ret;
 568         unsigned int            buf_size;
 569         struct dmatest_data     *src;
 570         struct dmatest_data     *dst;
 571         int                     i;
 572         ktime_t                 ktime, start, diff;
 573         ktime_t                 filltime = 0;
 574         ktime_t                 comparetime = 0;
 575         s64                     runtime = 0;
 576         unsigned long long      total_len = 0;
 577         unsigned long long      iops = 0;
 578         u8                      align = 0;
 579         bool                    is_memset = false;
 580         dma_addr_t              *srcs;
 581         dma_addr_t              *dma_pq;
 582
 583         set_freezable();
 584
 585         ret = -ENOMEM;
 586
 587         smp_rmb();
 588         thread->pending = false;
 589         info = thread->info;
 590         params = &info->params;
 591         chan = thread->chan;
 592         dev = chan->device;
 593         src = &thread->src;
 594         dst = &thread->dst;
 595         if (thread->type == DMA_MEMCPY) {
 596                 align = params->alignment < 0 ? dev->copy_align :
 597                                                 params->alignment;
 598                 src->cnt = dst->cnt = 1;
 599         } else if (thread->type == DMA_MEMSET) {
 600                 align = params->alignment < 0 ? dev->fill_align :
 601                                                 params->alignment;
 602                 src->cnt = dst->cnt = 1;
 603                 is_memset = true;
 604         } else if (thread->type == DMA_XOR) {
 605                 /* force odd to ensure dst = src */
 606                 src->cnt = min_odd(params->xor_sources | 1, dev->max_xor);
 607                 dst->cnt = 1;
 608                 align = params->alignment < 0 ? dev->xor_align :
 609                                                 params->alignment;
 610         } else if (thread->type == DMA_PQ) {
 611                 /* force odd to ensure dst = src */
 612                 src->cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0));
 613                 dst->cnt = 2;
 614                 align = params->alignment < 0 ? dev->pq_align :
 615                                                 params->alignment;
 616
 617                 pq_coefs = kmalloc(params->pq_sources + 1, GFP_KERNEL);
 618                 if (!pq_coefs)
 619                         goto err_thread_type;
 620
 621                 for (i = 0; i < src->cnt; i++)
 622                         pq_coefs[i] = 1;
 623         } else
 624                 goto err_thread_type;
 625
 626         /* Check if buffer count fits into map count variable (u8) */
 627         if ((src->cnt + dst->cnt) >= 255) {
 628                 pr_err("too many buffers (%d of 255 supported)\n",
 629                        src->cnt + dst->cnt);
 630                 goto err_free_coefs;
 631         }
 632
 633         buf_size = params->buf_size;
 634         if (1 << align > buf_size) {
 635                 pr_err("%u-byte buffer too small for %d-byte alignment\n",
 636                        buf_size, 1 << align);
 637                 goto err_free_coefs;
 638         }
 639
 640         if (dmatest_alloc_test_data(src, buf_size, align) < 0)
 641                 goto err_free_coefs;
 642
 643         if (dmatest_alloc_test_data(dst, buf_size, align) < 0)
 644                 goto err_src;
 645
 646         set_user_nice(current, 10);
 647
 648         srcs = kcalloc(src->cnt, sizeof(dma_addr_t), GFP_KERNEL);
 649         if (!srcs)
 650                 goto err_dst;
 651
 652         dma_pq = kcalloc(dst->cnt, sizeof(dma_addr_t), GFP_KERNEL);
 653         if (!dma_pq)
 654                 goto err_srcs_array;
 655
 656         /*
 657          * src and dst buffers are freed by ourselves below
 658          */
 659         if (params->polled)
 660                 flags = DMA_CTRL_ACK;
 661         else
 662                 flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
 663
 664         ktime = ktime_get();
 665         while (!(kthread_should_stop() ||
 666                (params->iterations && total_tests >= params->iterations))) {
 667                 struct dma_async_tx_descriptor *tx = NULL;
 668                 struct dmaengine_unmap_data *um;
 669                 dma_addr_t *dsts;
 670                 unsigned int len;
 671
 672                 total_tests++;
 673
 674                 if (params->transfer_size) {
 675                         if (params->transfer_size >= buf_size) {
 676                                 pr_err("%u-byte transfer size must be lower than %u-buffer size\n",
 677                                        params->transfer_size, buf_size);
 678                                 break;
 679                         }
 680                         len = params->transfer_size;
 681                 } else if (params->norandom) {
 682                         len = buf_size;
 683                 } else {
 684                         len = dmatest_random() % buf_size + 1;
 685                 }
 686
 687                 /* Do not alter transfer size explicitly defined by user */
 688                 if (!params->transfer_size) {
 689                         len = (len >> align) << align;
 690                         if (!len)
 691                                 len = 1 << align;
 692                 }
 693                 total_len += len;
 694
 695                 if (params->norandom) {
 696                         src->off = 0;
 697                         dst->off = 0;
 698                 } else {
 699                         src->off = dmatest_random() % (buf_size - len + 1);
 700                         dst->off = dmatest_random() % (buf_size - len + 1);
 701
 702                         src->off = (src->off >> align) << align;
 703                         dst->off = (dst->off >> align) << align;
 704                 }
 705
 706                 if (!params->noverify) {
 707                         start = ktime_get();
 708                         dmatest_init_srcs(src->aligned, src->off, len,
 709                                           buf_size, is_memset);
 710                         dmatest_init_dsts(dst->aligned, dst->off, len,
 711                                           buf_size, is_memset);
 712
 713                         diff = ktime_sub(ktime_get(), start);
 714                         filltime = ktime_add(filltime, diff);
 715                 }
 716
 717                 um = dmaengine_get_unmap_data(dev->dev, src->cnt + dst->cnt,
 718                                               GFP_KERNEL);
 719                 if (!um) {
 720                         failed_tests++;
 721                         result("unmap data NULL", total_tests,
 722                                src->off, dst->off, len, ret);
 723                         continue;
 724                 }
 725
 726                 um->len = buf_size;
 727                 for (i = 0; i < src->cnt; i++) {
 728                         void *buf = src->aligned[i];
 729                         struct page *pg = virt_to_page(buf);
 730                         unsigned long pg_off = offset_in_page(buf);
 731
 732                         um->addr[i] = dma_map_page(dev->dev, pg, pg_off,
 733                                                    um->len, DMA_TO_DEVICE);
 734                         srcs[i] = um->addr[i] + src->off;
 735                         ret = dma_mapping_error(dev->dev, um->addr[i]);
 736                         if (ret) {
 737                                 result("src mapping error", total_tests,
 738                                        src->off, dst->off, len, ret);
 739                                 goto error_unmap_continue;
 740                         }
 741                         um->to_cnt++;
 742                 }
 743                 /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */
 744                 dsts = &um->addr[src->cnt];
 745                 for (i = 0; i < dst->cnt; i++) {
 746                         void *buf = dst->aligned[i];
 747                         struct page *pg = virt_to_page(buf);
 748                         unsigned long pg_off = offset_in_page(buf);
 749
 750                         dsts[i] = dma_map_page(dev->dev, pg, pg_off, um->len,
 751                                                DMA_BIDIRECTIONAL);
 752                         ret = dma_mapping_error(dev->dev, dsts[i]);
 753                         if (ret) {
 754                                 result("dst mapping error", total_tests,
 755                                        src->off, dst->off, len, ret);
 756                                 goto error_unmap_continue;
 757                         }
 758                         um->bidi_cnt++;
 759                 }
 760
 761                 if (thread->type == DMA_MEMCPY)
 762                         tx = dev->device_prep_dma_memcpy(chan,
 763                                                          dsts[0] + dst->off,
 764                                                          srcs[0], len, flags);
 765                 else if (thread->type == DMA_MEMSET)
 766                         tx = dev->device_prep_dma_memset(chan,
 767                                                 dsts[0] + dst->off,
 768                                                 *(src->aligned[0] + src->off),
 769                                                 len, flags);
 770                 else if (thread->type == DMA_XOR)
 771                         tx = dev->device_prep_dma_xor(chan,
 772                                                       dsts[0] + dst->off,
 773                                                       srcs, src->cnt,
 774                                                       len, flags);
 775                 else if (thread->type == DMA_PQ) {
 776                         for (i = 0; i < dst->cnt; i++)
 777                                 dma_pq[i] = dsts[i] + dst->off;
 778                         tx = dev->device_prep_dma_pq(chan, dma_pq, srcs,
 779                                                      src->cnt, pq_coefs,
 780                                                      len, flags);
 781                 }
 782
 783                 if (!tx) {
 784                         result("prep error", total_tests, src->off,
 785                                dst->off, len, ret);
 786                         msleep(100);
 787                         goto error_unmap_continue;
 788                 }
 789
 790                 done->done = false;
 791                 if (!params->polled) {
 792                         tx->callback = dmatest_callback;
 793                         tx->callback_param = done;
 794                 }
 795                 cookie = tx->tx_submit(tx);
 796
 797                 if (dma_submit_error(cookie)) {
 798                         result("submit error", total_tests, src->off,
 799                                dst->off, len, ret);
 800                         msleep(100);
 801                         goto error_unmap_continue;
 802                 }
 803
 804                 if (params->polled) {
 805                         status = dma_sync_wait(chan, cookie);
 806                         dmaengine_terminate_sync(chan);
 807                         if (status == DMA_COMPLETE)
 808                                 done->done = true;
 809                 } else {
 810                         dma_async_issue_pending(chan);
 811
 812                         wait_event_freezable_timeout(thread->done_wait,
 813                                         done->done,
 814                                         msecs_to_jiffies(params->timeout));
 815
 816                         status = dma_async_is_tx_complete(chan, cookie, NULL,
 817                                                           NULL);
 818                 }
 819
 820                 if (!done->done) {
 821                         result("test timed out", total_tests, src->off, dst->off,
 822                                len, 0);
 823                         goto error_unmap_continue;
 824                 } else if (status != DMA_COMPLETE) {
 825                         result(status == DMA_ERROR ?
 826                                "completion error status" :
 827                                "completion busy status", total_tests, src->off,
 828                                dst->off, len, ret);
 829                         goto error_unmap_continue;
 830                 }
 831
 832                 dmaengine_unmap_put(um);
 833
 834                 if (params->noverify) {
 835                         verbose_result("test passed", total_tests, src->off,
 836                                        dst->off, len, 0);
 837                         continue;
 838                 }
 839
 840                 start = ktime_get();
 841                 pr_debug("%s: verifying source buffer...\n", current->comm);
 842                 error_count = dmatest_verify(src->aligned, 0, src->off,
 843                                 0, PATTERN_SRC, true, is_memset);
 844                 error_count += dmatest_verify(src->aligned, src->off,
 845                                 src->off + len, src->off,
 846                                 PATTERN_SRC | PATTERN_COPY, true, is_memset);
 847                 error_count += dmatest_verify(src->aligned, src->off + len,
 848                                 buf_size, src->off + len,
 849                                 PATTERN_SRC, true, is_memset);
 850
 851                 pr_debug("%s: verifying dest buffer...\n", current->comm);
 852                 error_count += dmatest_verify(dst->aligned, 0, dst->off,
 853                                 0, PATTERN_DST, false, is_memset);
 854
 855                 error_count += dmatest_verify(dst->aligned, dst->off,
 856                                 dst->off + len, src->off,
 857                                 PATTERN_SRC | PATTERN_COPY, false, is_memset);
 858
 859                 error_count += dmatest_verify(dst->aligned, dst->off + len,
 860                                 buf_size, dst->off + len,
 861                                 PATTERN_DST, false, is_memset);
 862
 863                 diff = ktime_sub(ktime_get(), start);
 864                 comparetime = ktime_add(comparetime, diff);
 865
 866                 if (error_count) {
 867                         result("data error", total_tests, src->off, dst->off,
 868                                len, error_count);
 869                         failed_tests++;
 870                 } else {
 871                         verbose_result("test passed", total_tests, src->off,
 872                                        dst->off, len, 0);
 873                 }
 874
 875                 continue;
 876
 877 error_unmap_continue:
 878                 dmaengine_unmap_put(um);
 879                 failed_tests++;
 880         }
 881         ktime = ktime_sub(ktime_get(), ktime);
 882         ktime = ktime_sub(ktime, comparetime);
 883         ktime = ktime_sub(ktime, filltime);
 884         runtime = ktime_to_us(ktime);
 885
 886         ret = 0;
 887         kfree(dma_pq);
 888 err_srcs_array:
 889         kfree(srcs);
 890 err_dst:
 891         dmatest_free_test_data(dst);
 892 err_src:
 893         dmatest_free_test_data(src);
 894 err_free_coefs:
 895         kfree(pq_coefs);
 896 err_thread_type:
 897         iops = dmatest_persec(runtime, total_tests);
 898         pr_info("%s: summary %u tests, %u failures %llu.%02llu iops %llu KB/s (%d)\n",
 899                 current->comm, total_tests, failed_tests,
 900                 FIXPT_TO_INT(iops), FIXPT_GET_FRAC(iops),
 901                 dmatest_KBs(runtime, total_len), ret);
 902
 903         /* terminate all transfers on specified channels */
 904         if (ret || failed_tests)
 905                 dmaengine_terminate_sync(chan);
 906
 907         thread->done = true;
 908         wake_up(&thread_wait);
 909
 910         return ret;
 911 }
 912
 913 static void dmatest_cleanup_channel(struct dmatest_chan *dtc)
 914 {
 915         struct dmatest_thread   *thread;
 916         struct dmatest_thread   *_thread;
 917         int                     ret;
 918
 919         list_for_each_entry_safe(thread, _thread, &dtc->threads, node) {
 920                 ret = kthread_stop(thread->task);
 921                 pr_debug("thread %s exited with status %d\n",
 922                          thread->task->comm, ret);
 923                 list_del(&thread->node);
 924                 put_task_struct(thread->task);
 925                 kfree(thread);
 926         }
 927
 928         /* terminate all transfers on specified channels */
 929         dmaengine_terminate_sync(dtc->chan);
 930
 931         kfree(dtc);
 932 }
 933
 934 static int dmatest_add_threads(struct dmatest_info *info,
 935                 struct dmatest_chan *dtc, enum dma_transaction_type type)
 936 {
 937         struct dmatest_params *params = &info->params;
 938         struct dmatest_thread *thread;
 939         struct dma_chan *chan = dtc->chan;
 940         char *op;
 941         unsigned int i;
 942
 943         if (type == DMA_MEMCPY)
 944                 op = "copy";
 945         else if (type == DMA_MEMSET)
 946                 op = "set";
 947         else if (type == DMA_XOR)
 948                 op = "xor";
 949         else if (type == DMA_PQ)
 950                 op = "pq";
 951         else
 952                 return -EINVAL;
 953
 954         for (i = 0; i < params->threads_per_chan; i++) {
 955                 thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL);
 956                 if (!thread) {
 957                         pr_warn("No memory for %s-%s%u\n",
 958                                 dma_chan_name(chan), op, i);
 959                         break;
 960                 }
 961                 thread->info = info;
 962                 thread->chan = dtc->chan;
 963                 thread->type = type;
 964                 thread->test_done.wait = &thread->done_wait;
 965                 init_waitqueue_head(&thread->done_wait);
 966                 smp_wmb();
 967                 thread->task = kthread_create(dmatest_func, thread, "%s-%s%u",
 968                                 dma_chan_name(chan), op, i);
 969                 if (IS_ERR(thread->task)) {
 970                         pr_warn("Failed to create thread %s-%s%u\n",
 971                                 dma_chan_name(chan), op, i);
 972                         kfree(thread);
 973                         break;
 974                 }
 975
 976                 /* srcbuf and dstbuf are allocated by the thread itself */
 977                 get_task_struct(thread->task);
 978                 list_add_tail(&thread->node, &dtc->threads);
 979                 thread->pending = true;
 980         }
 981
 982         return i;
 983 }
 984
 985 static int dmatest_add_channel(struct dmatest_info *info,
 986                 struct dma_chan *chan)
 987 {
 988         struct dmatest_chan     *dtc;
 989         struct dma_device       *dma_dev = chan->device;
 990         unsigned int            thread_count = 0;
 991         int cnt;
 992
 993         dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL);
 994         if (!dtc) {
 995                 pr_warn("No memory for %s\n", dma_chan_name(chan));
 996                 return -ENOMEM;
 997         }
 998
 999         dtc->chan = chan;
1000         INIT_LIST_HEAD(&dtc->threads);
1001
1002         if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
1003                 if (dmatest == 0) {
1004                         cnt = dmatest_add_threads(info, dtc, DMA_MEMCPY);
1005                         thread_count += cnt > 0 ? cnt : 0;
1006                 }
1007         }
1008
1009         if (dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) {
1010                 if (dmatest == 1) {
1011                         cnt = dmatest_add_threads(info, dtc, DMA_MEMSET);
1012                         thread_count += cnt > 0 ? cnt : 0;
1013                 }
1014         }
1015
1016         if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1017                 cnt = dmatest_add_threads(info, dtc, DMA_XOR);
1018                 thread_count += cnt > 0 ? cnt : 0;
1019         }
1020         if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
1021                 cnt = dmatest_add_threads(info, dtc, DMA_PQ);
1022                 thread_count += cnt > 0 ? cnt : 0;
1023         }
1024
1025         pr_info("Added %u threads using %s\n",
1026                 thread_count, dma_chan_name(chan));
1027
1028         list_add_tail(&dtc->node, &info->channels);
1029         info->nr_channels++;
1030
1031         return 0;
1032 }
1033
1034 static bool filter(struct dma_chan *chan, void *param)
1035 {
1036         struct dmatest_params *params = param;
1037
1038         if (!dmatest_match_channel(params, chan) ||
1039             !dmatest_match_device(params, chan->device))
1040                 return false;
1041         else
1042                 return true;
1043 }
1044
1045 static void request_channels(struct dmatest_info *info,
1046                              enum dma_transaction_type type)
1047 {
1048         dma_cap_mask_t mask;
1049
1050         dma_cap_zero(mask);
1051         dma_cap_set(type, mask);
1052         for (;;) {
1053                 struct dmatest_params *params = &info->params;
1054                 struct dma_chan *chan;
1055
1056                 chan = dma_request_channel(mask, filter, params);
1057                 if (chan) {
1058                         if (dmatest_add_channel(info, chan)) {
1059                                 dma_release_channel(chan);
1060                                 break; /* add_channel failed, punt */
1061                         }
1062                 } else
1063                         break; /* no more channels available */
1064                 if (params->max_channels &&
1065                     info->nr_channels >= params->max_channels)
1066                         break; /* we have all we need */
1067         }
1068 }
1069
1070 static void add_threaded_test(struct dmatest_info *info)
1071 {
1072         struct dmatest_params *params = &info->params;
1073
1074         /* Copy test parameters */
1075         params->buf_size = test_buf_size;
1076         strlcpy(params->channel, strim(test_channel), sizeof(params->channel));
1077         strlcpy(params->device, strim(test_device), sizeof(params->device));
1078         params->threads_per_chan = threads_per_chan;
1079         params->max_channels = max_channels;
1080         params->iterations = iterations;
1081         params->xor_sources = xor_sources;
1082         params->pq_sources = pq_sources;
1083         params->timeout = timeout;
1084         params->noverify = noverify;
1085         params->norandom = norandom;
1086         params->alignment = alignment;
1087         params->transfer_size = transfer_size;
1088         params->polled = polled;
1089
1090         request_channels(info, DMA_MEMCPY);
1091         request_channels(info, DMA_MEMSET);
1092         request_channels(info, DMA_XOR);
1093         request_channels(info, DMA_PQ);
1094 }
1095
1096 static void run_pending_tests(struct dmatest_info *info)
1097 {
1098         struct dmatest_chan *dtc;
1099         unsigned int thread_count = 0;
1100
1101         list_for_each_entry(dtc, &info->channels, node) {
1102                 struct dmatest_thread *thread;
1103
1104                 thread_count = 0;
1105                 list_for_each_entry(thread, &dtc->threads, node) {
1106                         wake_up_process(thread->task);
1107                         thread_count++;
1108                 }
1109                 pr_info("Started %u threads using %s\n",
1110                         thread_count, dma_chan_name(dtc->chan));
1111         }
1112 }
1113
1114 static void stop_threaded_test(struct dmatest_info *info)
1115 {
1116         struct dmatest_chan *dtc, *_dtc;
1117         struct dma_chan *chan;
1118
1119         list_for_each_entry_safe(dtc, _dtc, &info->channels, node) {
1120                 list_del(&dtc->node);
1121                 chan = dtc->chan;
1122                 dmatest_cleanup_channel(dtc);
1123                 pr_debug("dropped channel %s\n", dma_chan_name(chan));
1124                 dma_release_channel(chan);
1125         }
1126
1127         info->nr_channels = 0;
1128 }
1129
1130 static void start_threaded_tests(struct dmatest_info *info)
1131 {
1132         /* we might be called early to set run=, defer running until all
1133          * parameters have been evaluated
1134          */
1135         if (!info->did_init)
1136                 return;
1137
1138         run_pending_tests(info);
1139 }
1140
1141 static int dmatest_run_get(char *val, const struct kernel_param *kp)
1142 {
1143         struct dmatest_info *info = &test_info;
1144
1145         mutex_lock(&info->lock);
1146         if (is_threaded_test_run(info)) {
1147                 dmatest_run = true;
1148         } else {
1149                 if (!is_threaded_test_pending(info))
1150                         stop_threaded_test(info);
1151                 dmatest_run = false;
1152         }
1153         mutex_unlock(&info->lock);
1154
1155         return param_get_bool(val, kp);
1156 }
1157
1158 static int dmatest_run_set(const char *val, const struct kernel_param *kp)
1159 {
1160         struct dmatest_info *info = &test_info;
1161         int ret;
1162
1163         mutex_lock(&info->lock);
1164         ret = param_set_bool(val, kp);
1165         if (ret) {
1166                 mutex_unlock(&info->lock);
1167                 return ret;
1168         } else if (dmatest_run) {
1169                 if (!is_threaded_test_pending(info)) {
1170                         pr_info("No channels configured, continue with any\n");
1171                         add_threaded_test(info);
1172                 }
1173                 start_threaded_tests(info);
1174         } else {
1175                 stop_threaded_test(info);
1176         }
1177
1178         mutex_unlock(&info->lock);
1179
1180         return ret;
1181 }
1182
1183 static int dmatest_chan_set(const char *val, const struct kernel_param *kp)
1184 {
1185         struct dmatest_info *info = &test_info;
1186         struct dmatest_chan *dtc;
1187         char chan_reset_val[20];
1188         int ret = 0;
1189
1190         mutex_lock(&info->lock);
1191         ret = param_set_copystring(val, kp);
1192         if (ret) {
1193                 mutex_unlock(&info->lock);
1194                 return ret;
1195         }
1196         /*Clear any previously run threads */
1197         if (!is_threaded_test_run(info) && !is_threaded_test_pending(info))
1198                 stop_threaded_test(info);
1199         /* Reject channels that are already registered */
1200         if (is_threaded_test_pending(info)) {
1201                 list_for_each_entry(dtc, &info->channels, node) {
1202                         if (strcmp(dma_chan_name(dtc->chan),
1203                                    strim(test_channel)) == 0) {
1204                                 dtc = list_last_entry(&info->channels,
1205                                                       struct dmatest_chan,
1206                                                       node);
1207                                 strlcpy(chan_reset_val,
1208                                         dma_chan_name(dtc->chan),
1209                                         sizeof(chan_reset_val));
1210                                 ret = -EBUSY;
1211                                 goto add_chan_err;
1212                         }
1213                 }
1214         }
1215
1216         add_threaded_test(info);
1217
1218         /* Check if channel was added successfully */
1219         dtc = list_last_entry(&info->channels, struct dmatest_chan, node);
1220
1221         if (dtc->chan) {
1222                 /*
1223                  * if new channel was not successfully added, revert the
1224                  * "test_channel" string to the name of the last successfully
1225                  * added channel. exception for when users issues empty string
1226                  * to channel parameter.
1227                  */
1228                 if ((strcmp(dma_chan_name(dtc->chan), strim(test_channel)) != 0)
1229                     && (strcmp("", strim(test_channel)) != 0)) {
1230                         ret = -EINVAL;
1231                         strlcpy(chan_reset_val, dma_chan_name(dtc->chan),
1232                                 sizeof(chan_reset_val));
1233                         goto add_chan_err;
1234                 }
1235
1236         } else {
1237                 /* Clear test_channel if no channels were added successfully */
1238                 strlcpy(chan_reset_val, "", sizeof(chan_reset_val));
1239                 ret = -EBUSY;
1240                 goto add_chan_err;
1241         }
1242
1243         mutex_unlock(&info->lock);
1244
1245         return ret;
1246
1247 add_chan_err:
1248         param_set_copystring(chan_reset_val, kp);
1249         mutex_unlock(&info->lock);
1250
1251         return ret;
1252 }
1253
1254 static int dmatest_chan_get(char *val, const struct kernel_param *kp)
1255 {
1256         struct dmatest_info *info = &test_info;
1257
1258         mutex_lock(&info->lock);
1259         if (!is_threaded_test_run(info) && !is_threaded_test_pending(info)) {
1260                 stop_threaded_test(info);
1261                 strlcpy(test_channel, "", sizeof(test_channel));
1262         }
1263         mutex_unlock(&info->lock);
1264
1265         return param_get_string(val, kp);
1266 }
1267
1268 static int dmatest_test_list_get(char *val, const struct kernel_param *kp)
1269 {
1270         struct dmatest_info *info = &test_info;
1271         struct dmatest_chan *dtc;
1272         unsigned int thread_count = 0;
1273
1274         list_for_each_entry(dtc, &info->channels, node) {
1275                 struct dmatest_thread *thread;
1276
1277                 thread_count = 0;
1278                 list_for_each_entry(thread, &dtc->threads, node) {
1279                         thread_count++;
1280                 }
1281                 pr_info("%u threads using %s\n",
1282                         thread_count, dma_chan_name(dtc->chan));
1283         }
1284
1285         return 0;
1286 }
1287
1288 static int __init dmatest_init(void)
1289 {
1290         struct dmatest_info *info = &test_info;
1291         struct dmatest_params *params = &info->params;
1292
1293         if (dmatest_run) {
1294                 mutex_lock(&info->lock);
1295                 add_threaded_test(info);
1296                 run_pending_tests(info);
1297                 mutex_unlock(&info->lock);
1298         }
1299
1300         if (params->iterations && wait)
1301                 wait_event(thread_wait, !is_threaded_test_run(info));
1302
1303         /* module parameters are stable, inittime tests are started,
1304          * let userspace take over 'run' control
1305          */
1306         info->did_init = true;
1307
1308         return 0;
1309 }
1310 /* when compiled-in wait for drivers to load first */
1311 late_initcall(dmatest_init);
1312
1313 static void __exit dmatest_exit(void)
1314 {
1315         struct dmatest_info *info = &test_info;
1316
1317         mutex_lock(&info->lock);
1318         stop_threaded_test(info);
1319         mutex_unlock(&info->lock);
1320 }
1321 module_exit(dmatest_exit);
1322
1323 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
1324 MODULE_LICENSE("GPL v2");