drivers/media/v4l2-core/v4l2-mem2mem.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Memory-to-memory device framework for Video for Linux 2 and videobuf.
   4  *
   5  * Helper functions for devices that use videobuf buffers for both their
   6  * source and destination.
   7  *
   8  * Copyright (c) 2009-2010 Samsung Electronics Co., Ltd.
   9  * Pawel Osciak, <pawel@osciak.com>
  10  * Marek Szyprowski, <m.szyprowski@samsung.com>
  11  */
  12 #include <linux/module.h>
  13 #include <linux/sched.h>
  14 #include <linux/slab.h>
  15
  16 #include <media/media-device.h>
  17 #include <media/videobuf2-v4l2.h>
  18 #include <media/v4l2-mem2mem.h>
  19 #include <media/v4l2-dev.h>
  20 #include <media/v4l2-device.h>
  21 #include <media/v4l2-fh.h>
  22 #include <media/v4l2-event.h>
  23
  24 MODULE_DESCRIPTION("Mem to mem device framework for videobuf");
  25 MODULE_AUTHOR("Pawel Osciak, <pawel@osciak.com>");
  26 MODULE_LICENSE("GPL");
  27
  28 static bool debug;
  29 module_param(debug, bool, 0644);
  30
  31 #define dprintk(fmt, arg...)                                            \
  32         do {                                                            \
  33                 if (debug)                                              \
  34                         printk(KERN_DEBUG "%s: " fmt, __func__, ## arg);\
  35         } while (0)
  36
  37
  38 /* Instance is already queued on the job_queue */
  39 #define TRANS_QUEUED            (1 << 0)
  40 /* Instance is currently running in hardware */
  41 #define TRANS_RUNNING           (1 << 1)
  42 /* Instance is currently aborting */
  43 #define TRANS_ABORT             (1 << 2)
  44
  45
  46 /* The job queue is not running new jobs */
  47 #define QUEUE_PAUSED            (1 << 0)
  48
  49
  50 /* Offset base for buffers on the destination queue - used to distinguish
  51  * between source and destination buffers when mmapping - they receive the same
  52  * offsets but for different queues */
  53 #define DST_QUEUE_OFF_BASE      (1 << 30)
  54
  55 enum v4l2_m2m_entity_type {
  56         MEM2MEM_ENT_TYPE_SOURCE,
  57         MEM2MEM_ENT_TYPE_SINK,
  58         MEM2MEM_ENT_TYPE_PROC
  59 };
  60
  61 static const char * const m2m_entity_name[] = {
  62         "source",
  63         "sink",
  64         "proc"
  65 };
  66
  67 /**
  68  * struct v4l2_m2m_dev - per-device context
  69  * @source:             &struct media_entity pointer with the source entity
  70  *                      Used only when the M2M device is registered via
  71  *                      v4l2_m2m_unregister_media_controller().
  72  * @source_pad:         &struct media_pad with the source pad.
  73  *                      Used only when the M2M device is registered via
  74  *                      v4l2_m2m_unregister_media_controller().
  75  * @sink:               &struct media_entity pointer with the sink entity
  76  *                      Used only when the M2M device is registered via
  77  *                      v4l2_m2m_unregister_media_controller().
  78  * @sink_pad:           &struct media_pad with the sink pad.
  79  *                      Used only when the M2M device is registered via
  80  *                      v4l2_m2m_unregister_media_controller().
  81  * @proc:               &struct media_entity pointer with the M2M device itself.
  82  * @proc_pads:          &struct media_pad with the @proc pads.
  83  *                      Used only when the M2M device is registered via
  84  *                      v4l2_m2m_unregister_media_controller().
  85  * @intf_devnode:       &struct media_intf devnode pointer with the interface
  86  *                      with controls the M2M device.
  87  * @curr_ctx:           currently running instance
  88  * @job_queue:          instances queued to run
  89  * @job_spinlock:       protects job_queue
  90  * @job_work:           worker to run queued jobs.
  91  * @job_queue_flags:    flags of the queue status, %QUEUE_PAUSED.
  92  * @m2m_ops:            driver callbacks
  93  */
  94 struct v4l2_m2m_dev {
  95         struct v4l2_m2m_ctx     *curr_ctx;
  96 #ifdef CONFIG_MEDIA_CONTROLLER
  97         struct media_entity     *source;
  98         struct media_pad        source_pad;
  99         struct media_entity     sink;
 100         struct media_pad        sink_pad;
 101         struct media_entity     proc;
 102         struct media_pad        proc_pads[2];
 103         struct media_intf_devnode *intf_devnode;
 104 #endif
 105
 106         struct list_head        job_queue;
 107         spinlock_t              job_spinlock;
 108         struct work_struct      job_work;
 109         unsigned long           job_queue_flags;
 110
 111         const struct v4l2_m2m_ops *m2m_ops;
 112 };
 113
 114 static struct v4l2_m2m_queue_ctx *get_queue_ctx(struct v4l2_m2m_ctx *m2m_ctx,
 115                                                 enum v4l2_buf_type type)
 116 {
 117         if (V4L2_TYPE_IS_OUTPUT(type))
 118                 return &m2m_ctx->out_q_ctx;
 119         else
 120                 return &m2m_ctx->cap_q_ctx;
 121 }
 122
 123 struct vb2_queue *v4l2_m2m_get_vq(struct v4l2_m2m_ctx *m2m_ctx,
 124                                        enum v4l2_buf_type type)
 125 {
 126         struct v4l2_m2m_queue_ctx *q_ctx;
 127
 128         q_ctx = get_queue_ctx(m2m_ctx, type);
 129         if (!q_ctx)
 130                 return NULL;
 131
 132         return &q_ctx->q;
 133 }
 134 EXPORT_SYMBOL(v4l2_m2m_get_vq);
 135
 136 struct vb2_v4l2_buffer *v4l2_m2m_next_buf(struct v4l2_m2m_queue_ctx *q_ctx)
 137 {
 138         struct v4l2_m2m_buffer *b;
 139         unsigned long flags;
 140
 141         spin_lock_irqsave(&q_ctx->rdy_spinlock, flags);
 142
 143         if (list_empty(&q_ctx->rdy_queue)) {
 144                 spin_unlock_irqrestore(&q_ctx->rdy_spinlock, flags);
 145                 return NULL;
 146         }
 147
 148         b = list_first_entry(&q_ctx->rdy_queue, struct v4l2_m2m_buffer, list);
 149         spin_unlock_irqrestore(&q_ctx->rdy_spinlock, flags);
 150         return &b->vb;
 151 }
 152 EXPORT_SYMBOL_GPL(v4l2_m2m_next_buf);
 153
 154 struct vb2_v4l2_buffer *v4l2_m2m_last_buf(struct v4l2_m2m_queue_ctx *q_ctx)
 155 {
 156         struct v4l2_m2m_buffer *b;
 157         unsigned long flags;
 158
 159         spin_lock_irqsave(&q_ctx->rdy_spinlock, flags);
 160
 161         if (list_empty(&q_ctx->rdy_queue)) {
 162                 spin_unlock_irqrestore(&q_ctx->rdy_spinlock, flags);
 163                 return NULL;
 164         }
 165
 166         b = list_last_entry(&q_ctx->rdy_queue, struct v4l2_m2m_buffer, list);
 167         spin_unlock_irqrestore(&q_ctx->rdy_spinlock, flags);
 168         return &b->vb;
 169 }
 170 EXPORT_SYMBOL_GPL(v4l2_m2m_last_buf);
 171
 172 struct vb2_v4l2_buffer *v4l2_m2m_buf_remove(struct v4l2_m2m_queue_ctx *q_ctx)
 173 {
 174         struct v4l2_m2m_buffer *b;
 175         unsigned long flags;
 176
 177         spin_lock_irqsave(&q_ctx->rdy_spinlock, flags);
 178         if (list_empty(&q_ctx->rdy_queue)) {
 179                 spin_unlock_irqrestore(&q_ctx->rdy_spinlock, flags);
 180                 return NULL;
 181         }
 182         b = list_first_entry(&q_ctx->rdy_queue, struct v4l2_m2m_buffer, list);
 183         list_del(&b->list);
 184         q_ctx->num_rdy--;
 185         spin_unlock_irqrestore(&q_ctx->rdy_spinlock, flags);
 186
 187         return &b->vb;
 188 }
 189 EXPORT_SYMBOL_GPL(v4l2_m2m_buf_remove);
 190
 191 void v4l2_m2m_buf_remove_by_buf(struct v4l2_m2m_queue_ctx *q_ctx,
 192                                 struct vb2_v4l2_buffer *vbuf)
 193 {
 194         struct v4l2_m2m_buffer *b;
 195         unsigned long flags;
 196
 197         spin_lock_irqsave(&q_ctx->rdy_spinlock, flags);
 198         b = container_of(vbuf, struct v4l2_m2m_buffer, vb);
 199         list_del(&b->list);
 200         q_ctx->num_rdy--;
 201         spin_unlock_irqrestore(&q_ctx->rdy_spinlock, flags);
 202 }
 203 EXPORT_SYMBOL_GPL(v4l2_m2m_buf_remove_by_buf);
 204
 205 struct vb2_v4l2_buffer *
 206 v4l2_m2m_buf_remove_by_idx(struct v4l2_m2m_queue_ctx *q_ctx, unsigned int idx)
 207
 208 {
 209         struct v4l2_m2m_buffer *b, *tmp;
 210         struct vb2_v4l2_buffer *ret = NULL;
 211         unsigned long flags;
 212
 213         spin_lock_irqsave(&q_ctx->rdy_spinlock, flags);
 214         list_for_each_entry_safe(b, tmp, &q_ctx->rdy_queue, list) {
 215                 if (b->vb.vb2_buf.index == idx) {
 216                         list_del(&b->list);
 217                         q_ctx->num_rdy--;
 218                         ret = &b->vb;
 219                         break;
 220                 }
 221         }
 222         spin_unlock_irqrestore(&q_ctx->rdy_spinlock, flags);
 223
 224         return ret;
 225 }
 226 EXPORT_SYMBOL_GPL(v4l2_m2m_buf_remove_by_idx);
 227
 228 /*
 229  * Scheduling handlers
 230  */
 231
 232 void *v4l2_m2m_get_curr_priv(struct v4l2_m2m_dev *m2m_dev)
 233 {
 234         unsigned long flags;
 235         void *ret = NULL;
 236
 237         spin_lock_irqsave(&m2m_dev->job_spinlock, flags);
 238         if (m2m_dev->curr_ctx)
 239                 ret = m2m_dev->curr_ctx->priv;
 240         spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
 241
 242         return ret;
 243 }
 244 EXPORT_SYMBOL(v4l2_m2m_get_curr_priv);
 245
 246 /**
 247  * v4l2_m2m_try_run() - select next job to perform and run it if possible
 248  * @m2m_dev: per-device context
 249  *
 250  * Get next transaction (if present) from the waiting jobs list and run it.
 251  *
 252  * Note that this function can run on a given v4l2_m2m_ctx context,
 253  * but call .device_run for another context.
 254  */
 255 static void v4l2_m2m_try_run(struct v4l2_m2m_dev *m2m_dev)
 256 {
 257         unsigned long flags;
 258
 259         spin_lock_irqsave(&m2m_dev->job_spinlock, flags);
 260         if (NULL != m2m_dev->curr_ctx) {
 261                 spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
 262                 dprintk("Another instance is running, won't run now\n");
 263                 return;
 264         }
 265
 266         if (list_empty(&m2m_dev->job_queue)) {
 267                 spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
 268                 dprintk("No job pending\n");
 269                 return;
 270         }
 271
 272         if (m2m_dev->job_queue_flags & QUEUE_PAUSED) {
 273                 spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
 274                 dprintk("Running new jobs is paused\n");
 275                 return;
 276         }
 277
 278         m2m_dev->curr_ctx = list_first_entry(&m2m_dev->job_queue,
 279                                    struct v4l2_m2m_ctx, queue);
 280         m2m_dev->curr_ctx->job_flags |= TRANS_RUNNING;
 281         spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
 282
 283         dprintk("Running job on m2m_ctx: %p\n", m2m_dev->curr_ctx);
 284         m2m_dev->m2m_ops->device_run(m2m_dev->curr_ctx->priv);
 285 }
 286
 287 /*
 288  * __v4l2_m2m_try_queue() - queue a job
 289  * @m2m_dev: m2m device
 290  * @m2m_ctx: m2m context
 291  *
 292  * Check if this context is ready to queue a job.
 293  *
 294  * This function can run in interrupt context.
 295  */
 296 static void __v4l2_m2m_try_queue(struct v4l2_m2m_dev *m2m_dev,
 297                                  struct v4l2_m2m_ctx *m2m_ctx)
 298 {
 299         unsigned long flags_job;
 300         struct vb2_v4l2_buffer *dst, *src;
 301
 302         dprintk("Trying to schedule a job for m2m_ctx: %p\n", m2m_ctx);
 303
 304         if (!m2m_ctx->out_q_ctx.q.streaming
 305             || !m2m_ctx->cap_q_ctx.q.streaming) {
 306                 dprintk("Streaming needs to be on for both queues\n");
 307                 return;
 308         }
 309
 310         spin_lock_irqsave(&m2m_dev->job_spinlock, flags_job);
 311
 312         /* If the context is aborted then don't schedule it */
 313         if (m2m_ctx->job_flags & TRANS_ABORT) {
 314                 dprintk("Aborted context\n");
 315                 goto job_unlock;
 316         }
 317
 318         if (m2m_ctx->job_flags & TRANS_QUEUED) {
 319                 dprintk("On job queue already\n");
 320                 goto job_unlock;
 321         }
 322
 323         src = v4l2_m2m_next_src_buf(m2m_ctx);
 324         dst = v4l2_m2m_next_dst_buf(m2m_ctx);
 325         if (!src && !m2m_ctx->out_q_ctx.buffered) {
 326                 dprintk("No input buffers available\n");
 327                 goto job_unlock;
 328         }
 329         if (!dst && !m2m_ctx->cap_q_ctx.buffered) {
 330                 dprintk("No output buffers available\n");
 331                 goto job_unlock;
 332         }
 333
 334         m2m_ctx->new_frame = true;
 335
 336         if (src && dst && dst->is_held &&
 337             dst->vb2_buf.copied_timestamp &&
 338             dst->vb2_buf.timestamp != src->vb2_buf.timestamp) {
 339                 dst->is_held = false;
 340                 v4l2_m2m_dst_buf_remove(m2m_ctx);
 341                 v4l2_m2m_buf_done(dst, VB2_BUF_STATE_DONE);
 342                 dst = v4l2_m2m_next_dst_buf(m2m_ctx);
 343
 344                 if (!dst && !m2m_ctx->cap_q_ctx.buffered) {
 345                         dprintk("No output buffers available after returning held buffer\n");
 346                         goto job_unlock;
 347                 }
 348         }
 349
 350         if (src && dst && (m2m_ctx->out_q_ctx.q.subsystem_flags &
 351                            VB2_V4L2_FL_SUPPORTS_M2M_HOLD_CAPTURE_BUF))
 352                 m2m_ctx->new_frame = !dst->vb2_buf.copied_timestamp ||
 353                         dst->vb2_buf.timestamp != src->vb2_buf.timestamp;
 354
 355         if (m2m_ctx->has_stopped) {
 356                 dprintk("Device has stopped\n");
 357                 goto job_unlock;
 358         }
 359
 360         if (m2m_dev->m2m_ops->job_ready
 361                 && (!m2m_dev->m2m_ops->job_ready(m2m_ctx->priv))) {
 362                 dprintk("Driver not ready\n");
 363                 goto job_unlock;
 364         }
 365
 366         list_add_tail(&m2m_ctx->queue, &m2m_dev->job_queue);
 367         m2m_ctx->job_flags |= TRANS_QUEUED;
 368
 369 job_unlock:
 370         spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags_job);
 371 }
 372
 373 /**
 374  * v4l2_m2m_try_schedule() - schedule and possibly run a job for any context
 375  * @m2m_ctx: m2m context
 376  *
 377  * Check if this context is ready to queue a job. If suitable,
 378  * run the next queued job on the mem2mem device.
 379  *
 380  * This function shouldn't run in interrupt context.
 381  *
 382  * Note that v4l2_m2m_try_schedule() can schedule one job for this context,
 383  * and then run another job for another context.
 384  */
 385 void v4l2_m2m_try_schedule(struct v4l2_m2m_ctx *m2m_ctx)
 386 {
 387         struct v4l2_m2m_dev *m2m_dev = m2m_ctx->m2m_dev;
 388
 389         __v4l2_m2m_try_queue(m2m_dev, m2m_ctx);
 390         v4l2_m2m_try_run(m2m_dev);
 391 }
 392 EXPORT_SYMBOL_GPL(v4l2_m2m_try_schedule);
 393
 394 /**
 395  * v4l2_m2m_device_run_work() - run pending jobs for the context
 396  * @work: Work structure used for scheduling the execution of this function.
 397  */
 398 static void v4l2_m2m_device_run_work(struct work_struct *work)
 399 {
 400         struct v4l2_m2m_dev *m2m_dev =
 401                 container_of(work, struct v4l2_m2m_dev, job_work);
 402
 403         v4l2_m2m_try_run(m2m_dev);
 404 }
 405
 406 /**
 407  * v4l2_m2m_cancel_job() - cancel pending jobs for the context
 408  * @m2m_ctx: m2m context with jobs to be canceled
 409  *
 410  * In case of streamoff or release called on any context,
 411  * 1] If the context is currently running, then abort job will be called
 412  * 2] If the context is queued, then the context will be removed from
 413  *    the job_queue
 414  */
 415 static void v4l2_m2m_cancel_job(struct v4l2_m2m_ctx *m2m_ctx)
 416 {
 417         struct v4l2_m2m_dev *m2m_dev;
 418         unsigned long flags;
 419
 420         m2m_dev = m2m_ctx->m2m_dev;
 421         spin_lock_irqsave(&m2m_dev->job_spinlock, flags);
 422
 423         m2m_ctx->job_flags |= TRANS_ABORT;
 424         if (m2m_ctx->job_flags & TRANS_RUNNING) {
 425                 spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
 426                 if (m2m_dev->m2m_ops->job_abort)
 427                         m2m_dev->m2m_ops->job_abort(m2m_ctx->priv);
 428                 dprintk("m2m_ctx %p running, will wait to complete\n", m2m_ctx);
 429                 wait_event(m2m_ctx->finished,
 430                                 !(m2m_ctx->job_flags & TRANS_RUNNING));
 431         } else if (m2m_ctx->job_flags & TRANS_QUEUED) {
 432                 list_del(&m2m_ctx->queue);
 433                 m2m_ctx->job_flags &= ~(TRANS_QUEUED | TRANS_RUNNING);
 434                 spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
 435                 dprintk("m2m_ctx: %p had been on queue and was removed\n",
 436                         m2m_ctx);
 437         } else {
 438                 /* Do nothing, was not on queue/running */
 439                 spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
 440         }
 441 }
 442
 443 /*
 444  * Schedule the next job, called from v4l2_m2m_job_finish() or
 445  * v4l2_m2m_buf_done_and_job_finish().
 446  */
 447 static void v4l2_m2m_schedule_next_job(struct v4l2_m2m_dev *m2m_dev,
 448                                        struct v4l2_m2m_ctx *m2m_ctx)
 449 {
 450         /*
 451          * This instance might have more buffers ready, but since we do not
 452          * allow more than one job on the job_queue per instance, each has
 453          * to be scheduled separately after the previous one finishes.
 454          */
 455         __v4l2_m2m_try_queue(m2m_dev, m2m_ctx);
 456
 457         /*
 458          * We might be running in atomic context,
 459          * but the job must be run in non-atomic context.
 460          */
 461         schedule_work(&m2m_dev->job_work);
 462 }
 463
 464 /*
 465  * Assumes job_spinlock is held, called from v4l2_m2m_job_finish() or
 466  * v4l2_m2m_buf_done_and_job_finish().
 467  */
 468 static bool _v4l2_m2m_job_finish(struct v4l2_m2m_dev *m2m_dev,
 469                                  struct v4l2_m2m_ctx *m2m_ctx)
 470 {
 471         if (!m2m_dev->curr_ctx || m2m_dev->curr_ctx != m2m_ctx) {
 472                 dprintk("Called by an instance not currently running\n");
 473                 return false;
 474         }
 475
 476         list_del(&m2m_dev->curr_ctx->queue);
 477         m2m_dev->curr_ctx->job_flags &= ~(TRANS_QUEUED | TRANS_RUNNING);
 478         wake_up(&m2m_dev->curr_ctx->finished);
 479         m2m_dev->curr_ctx = NULL;
 480         return true;
 481 }
 482
 483 void v4l2_m2m_job_finish(struct v4l2_m2m_dev *m2m_dev,
 484                          struct v4l2_m2m_ctx *m2m_ctx)
 485 {
 486         unsigned long flags;
 487         bool schedule_next;
 488
 489         /*
 490          * This function should not be used for drivers that support
 491          * holding capture buffers. Those should use
 492          * v4l2_m2m_buf_done_and_job_finish() instead.
 493          */
 494         WARN_ON(m2m_ctx->out_q_ctx.q.subsystem_flags &
 495                 VB2_V4L2_FL_SUPPORTS_M2M_HOLD_CAPTURE_BUF);
 496         spin_lock_irqsave(&m2m_dev->job_spinlock, flags);
 497         schedule_next = _v4l2_m2m_job_finish(m2m_dev, m2m_ctx);
 498         spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
 499
 500         if (schedule_next)
 501                 v4l2_m2m_schedule_next_job(m2m_dev, m2m_ctx);
 502 }
 503 EXPORT_SYMBOL(v4l2_m2m_job_finish);
 504
 505 void v4l2_m2m_buf_done_and_job_finish(struct v4l2_m2m_dev *m2m_dev,
 506                                       struct v4l2_m2m_ctx *m2m_ctx,
 507                                       enum vb2_buffer_state state)
 508 {
 509         struct vb2_v4l2_buffer *src_buf, *dst_buf;
 510         bool schedule_next = false;
 511         unsigned long flags;
 512
 513         spin_lock_irqsave(&m2m_dev->job_spinlock, flags);
 514         src_buf = v4l2_m2m_src_buf_remove(m2m_ctx);
 515         dst_buf = v4l2_m2m_next_dst_buf(m2m_ctx);
 516
 517         if (WARN_ON(!src_buf || !dst_buf))
 518                 goto unlock;
 519         dst_buf->is_held = src_buf->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF;
 520         if (!dst_buf->is_held) {
 521                 v4l2_m2m_dst_buf_remove(m2m_ctx);
 522                 v4l2_m2m_buf_done(dst_buf, state);
 523         }
 524         /*
 525          * If the request API is being used, returning the OUTPUT
 526          * (src) buffer will wake-up any process waiting on the
 527          * request file descriptor.
 528          *
 529          * Therefore, return the CAPTURE (dst) buffer first,
 530          * to avoid signalling the request file descriptor
 531          * before the CAPTURE buffer is done.
 532          */
 533         v4l2_m2m_buf_done(src_buf, state);
 534         schedule_next = _v4l2_m2m_job_finish(m2m_dev, m2m_ctx);
 535 unlock:
 536         spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
 537
 538         if (schedule_next)
 539                 v4l2_m2m_schedule_next_job(m2m_dev, m2m_ctx);
 540 }
 541 EXPORT_SYMBOL(v4l2_m2m_buf_done_and_job_finish);
 542
 543 void v4l2_m2m_suspend(struct v4l2_m2m_dev *m2m_dev)
 544 {
 545         unsigned long flags;
 546         struct v4l2_m2m_ctx *curr_ctx;
 547
 548         spin_lock_irqsave(&m2m_dev->job_spinlock, flags);
 549         m2m_dev->job_queue_flags |= QUEUE_PAUSED;
 550         curr_ctx = m2m_dev->curr_ctx;
 551         spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
 552
 553         if (curr_ctx)
 554                 wait_event(curr_ctx->finished,
 555                            !(curr_ctx->job_flags & TRANS_RUNNING));
 556 }
 557 EXPORT_SYMBOL(v4l2_m2m_suspend);
 558
 559 void v4l2_m2m_resume(struct v4l2_m2m_dev *m2m_dev)
 560 {
 561         unsigned long flags;
 562
 563         spin_lock_irqsave(&m2m_dev->job_spinlock, flags);
 564         m2m_dev->job_queue_flags &= ~QUEUE_PAUSED;
 565         spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
 566
 567         v4l2_m2m_try_run(m2m_dev);
 568 }
 569 EXPORT_SYMBOL(v4l2_m2m_resume);
 570
 571 int v4l2_m2m_reqbufs(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 572                      struct v4l2_requestbuffers *reqbufs)
 573 {
 574         struct vb2_queue *vq;
 575         int ret;
 576
 577         vq = v4l2_m2m_get_vq(m2m_ctx, reqbufs->type);
 578         ret = vb2_reqbufs(vq, reqbufs);
 579         /* If count == 0, then the owner has released all buffers and he
 580            is no longer owner of the queue. Otherwise we have an owner. */
 581         if (ret == 0)
 582                 vq->owner = reqbufs->count ? file->private_data : NULL;
 583
 584         return ret;
 585 }
 586 EXPORT_SYMBOL_GPL(v4l2_m2m_reqbufs);
 587
 588 int v4l2_m2m_querybuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 589                       struct v4l2_buffer *buf)
 590 {
 591         struct vb2_queue *vq;
 592         int ret = 0;
 593         unsigned int i;
 594
 595         vq = v4l2_m2m_get_vq(m2m_ctx, buf->type);
 596         ret = vb2_querybuf(vq, buf);
 597
 598         /* Adjust MMAP memory offsets for the CAPTURE queue */
 599         if (buf->memory == V4L2_MEMORY_MMAP && V4L2_TYPE_IS_CAPTURE(vq->type)) {
 600                 if (V4L2_TYPE_IS_MULTIPLANAR(vq->type)) {
 601                         for (i = 0; i < buf->length; ++i)
 602                                 buf->m.planes[i].m.mem_offset
 603                                         += DST_QUEUE_OFF_BASE;
 604                 } else {
 605                         buf->m.offset += DST_QUEUE_OFF_BASE;
 606                 }
 607         }
 608
 609         return ret;
 610 }
 611 EXPORT_SYMBOL_GPL(v4l2_m2m_querybuf);
 612
 613 /*
 614  * This will add the LAST flag and mark the buffer management
 615  * state as stopped.
 616  * This is called when the last capture buffer must be flagged as LAST
 617  * in draining mode from the encoder/decoder driver buf_queue() callback
 618  * or from v4l2_update_last_buf_state() when a capture buffer is available.
 619  */
 620 void v4l2_m2m_last_buffer_done(struct v4l2_m2m_ctx *m2m_ctx,
 621                                struct vb2_v4l2_buffer *vbuf)
 622 {
 623         vbuf->flags |= V4L2_BUF_FLAG_LAST;
 624         vb2_buffer_done(&vbuf->vb2_buf, VB2_BUF_STATE_DONE);
 625
 626         v4l2_m2m_mark_stopped(m2m_ctx);
 627 }
 628 EXPORT_SYMBOL_GPL(v4l2_m2m_last_buffer_done);
 629
 630 /* When stop command is issued, update buffer management state */
 631 static int v4l2_update_last_buf_state(struct v4l2_m2m_ctx *m2m_ctx)
 632 {
 633         struct vb2_v4l2_buffer *next_dst_buf;
 634
 635         if (m2m_ctx->is_draining)
 636                 return -EBUSY;
 637
 638         if (m2m_ctx->has_stopped)
 639                 return 0;
 640
 641         m2m_ctx->last_src_buf = v4l2_m2m_last_src_buf(m2m_ctx);
 642         m2m_ctx->is_draining = true;
 643
 644         /*
 645          * The processing of the last output buffer queued before
 646          * the STOP command is expected to mark the buffer management
 647          * state as stopped with v4l2_m2m_mark_stopped().
 648          */
 649         if (m2m_ctx->last_src_buf)
 650                 return 0;
 651
 652         /*
 653          * In case the output queue is empty, try to mark the last capture
 654          * buffer as LAST.
 655          */
 656         next_dst_buf = v4l2_m2m_dst_buf_remove(m2m_ctx);
 657         if (!next_dst_buf) {
 658                 /*
 659                  * Wait for the next queued one in encoder/decoder driver
 660                  * buf_queue() callback using the v4l2_m2m_dst_buf_is_last()
 661                  * helper or in v4l2_m2m_qbuf() if encoder/decoder is not yet
 662                  * streaming.
 663                  */
 664                 m2m_ctx->next_buf_last = true;
 665                 return 0;
 666         }
 667
 668         v4l2_m2m_last_buffer_done(m2m_ctx, next_dst_buf);
 669
 670         return 0;
 671 }
 672
 673 /*
 674  * Updates the encoding/decoding buffer management state, should
 675  * be called from encoder/decoder drivers start_streaming()
 676  */
 677 void v4l2_m2m_update_start_streaming_state(struct v4l2_m2m_ctx *m2m_ctx,
 678                                            struct vb2_queue *q)
 679 {
 680         /* If start streaming again, untag the last output buffer */
 681         if (V4L2_TYPE_IS_OUTPUT(q->type))
 682                 m2m_ctx->last_src_buf = NULL;
 683 }
 684 EXPORT_SYMBOL_GPL(v4l2_m2m_update_start_streaming_state);
 685
 686 /*
 687  * Updates the encoding/decoding buffer management state, should
 688  * be called from encoder/decoder driver stop_streaming()
 689  */
 690 void v4l2_m2m_update_stop_streaming_state(struct v4l2_m2m_ctx *m2m_ctx,
 691                                           struct vb2_queue *q)
 692 {
 693         if (V4L2_TYPE_IS_OUTPUT(q->type)) {
 694                 /*
 695                  * If in draining state, either mark next dst buffer as
 696                  * done or flag next one to be marked as done either
 697                  * in encoder/decoder driver buf_queue() callback using
 698                  * the v4l2_m2m_dst_buf_is_last() helper or in v4l2_m2m_qbuf()
 699                  * if encoder/decoder is not yet streaming
 700                  */
 701                 if (m2m_ctx->is_draining) {
 702                         struct vb2_v4l2_buffer *next_dst_buf;
 703
 704                         m2m_ctx->last_src_buf = NULL;
 705                         next_dst_buf = v4l2_m2m_dst_buf_remove(m2m_ctx);
 706                         if (!next_dst_buf)
 707                                 m2m_ctx->next_buf_last = true;
 708                         else
 709                                 v4l2_m2m_last_buffer_done(m2m_ctx,
 710                                                           next_dst_buf);
 711                 }
 712         } else {
 713                 v4l2_m2m_clear_state(m2m_ctx);
 714         }
 715 }
 716 EXPORT_SYMBOL_GPL(v4l2_m2m_update_stop_streaming_state);
 717
 718 static void v4l2_m2m_force_last_buf_done(struct v4l2_m2m_ctx *m2m_ctx,
 719                                          struct vb2_queue *q)
 720 {
 721         struct vb2_buffer *vb;
 722         struct vb2_v4l2_buffer *vbuf;
 723         unsigned int i;
 724
 725         if (WARN_ON(q->is_output))
 726                 return;
 727         if (list_empty(&q->queued_list))
 728                 return;
 729
 730         vb = list_first_entry(&q->queued_list, struct vb2_buffer, queued_entry);
 731         for (i = 0; i < vb->num_planes; i++)
 732                 vb2_set_plane_payload(vb, i, 0);
 733
 734         /*
 735          * Since the buffer hasn't been queued to the ready queue,
 736          * mark is active and owned before marking it LAST and DONE
 737          */
 738         vb->state = VB2_BUF_STATE_ACTIVE;
 739         atomic_inc(&q->owned_by_drv_count);
 740
 741         vbuf = to_vb2_v4l2_buffer(vb);
 742         vbuf->field = V4L2_FIELD_NONE;
 743
 744         v4l2_m2m_last_buffer_done(m2m_ctx, vbuf);
 745 }
 746
 747 int v4l2_m2m_qbuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 748                   struct v4l2_buffer *buf)
 749 {
 750         struct video_device *vdev = video_devdata(file);
 751         struct vb2_queue *vq;
 752         int ret;
 753
 754         vq = v4l2_m2m_get_vq(m2m_ctx, buf->type);
 755         if (V4L2_TYPE_IS_CAPTURE(vq->type) &&
 756             (buf->flags & V4L2_BUF_FLAG_REQUEST_FD)) {
 757                 dprintk("%s: requests cannot be used with capture buffers\n",
 758                         __func__);
 759                 return -EPERM;
 760         }
 761
 762         ret = vb2_qbuf(vq, vdev->v4l2_dev->mdev, buf);
 763         if (ret)
 764                 return ret;
 765
 766         /*
 767          * If the capture queue is streaming, but streaming hasn't started
 768          * on the device, but was asked to stop, mark the previously queued
 769          * buffer as DONE with LAST flag since it won't be queued on the
 770          * device.
 771          */
 772         if (V4L2_TYPE_IS_CAPTURE(vq->type) &&
 773             vb2_is_streaming(vq) && !vb2_start_streaming_called(vq) &&
 774            (v4l2_m2m_has_stopped(m2m_ctx) || v4l2_m2m_dst_buf_is_last(m2m_ctx)))
 775                 v4l2_m2m_force_last_buf_done(m2m_ctx, vq);
 776         else if (!(buf->flags & V4L2_BUF_FLAG_IN_REQUEST))
 777                 v4l2_m2m_try_schedule(m2m_ctx);
 778
 779         return 0;
 780 }
 781 EXPORT_SYMBOL_GPL(v4l2_m2m_qbuf);
 782
 783 int v4l2_m2m_dqbuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 784                    struct v4l2_buffer *buf)
 785 {
 786         struct vb2_queue *vq;
 787
 788         vq = v4l2_m2m_get_vq(m2m_ctx, buf->type);
 789         return vb2_dqbuf(vq, buf, file->f_flags & O_NONBLOCK);
 790 }
 791 EXPORT_SYMBOL_GPL(v4l2_m2m_dqbuf);
 792
 793 int v4l2_m2m_prepare_buf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 794                          struct v4l2_buffer *buf)
 795 {
 796         struct video_device *vdev = video_devdata(file);
 797         struct vb2_queue *vq;
 798
 799         vq = v4l2_m2m_get_vq(m2m_ctx, buf->type);
 800         return vb2_prepare_buf(vq, vdev->v4l2_dev->mdev, buf);
 801 }
 802 EXPORT_SYMBOL_GPL(v4l2_m2m_prepare_buf);
 803
 804 int v4l2_m2m_create_bufs(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 805                          struct v4l2_create_buffers *create)
 806 {
 807         struct vb2_queue *vq;
 808
 809         vq = v4l2_m2m_get_vq(m2m_ctx, create->format.type);
 810         return vb2_create_bufs(vq, create);
 811 }
 812 EXPORT_SYMBOL_GPL(v4l2_m2m_create_bufs);
 813
 814 int v4l2_m2m_expbuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 815                   struct v4l2_exportbuffer *eb)
 816 {
 817         struct vb2_queue *vq;
 818
 819         vq = v4l2_m2m_get_vq(m2m_ctx, eb->type);
 820         return vb2_expbuf(vq, eb);
 821 }
 822 EXPORT_SYMBOL_GPL(v4l2_m2m_expbuf);
 823
 824 int v4l2_m2m_streamon(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 825                       enum v4l2_buf_type type)
 826 {
 827         struct vb2_queue *vq;
 828         int ret;
 829
 830         vq = v4l2_m2m_get_vq(m2m_ctx, type);
 831         ret = vb2_streamon(vq, type);
 832         if (!ret)
 833                 v4l2_m2m_try_schedule(m2m_ctx);
 834
 835         return ret;
 836 }
 837 EXPORT_SYMBOL_GPL(v4l2_m2m_streamon);
 838
 839 int v4l2_m2m_streamoff(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 840                        enum v4l2_buf_type type)
 841 {
 842         struct v4l2_m2m_dev *m2m_dev;
 843         struct v4l2_m2m_queue_ctx *q_ctx;
 844         unsigned long flags_job, flags;
 845         int ret;
 846
 847         /* wait until the current context is dequeued from job_queue */
 848         v4l2_m2m_cancel_job(m2m_ctx);
 849
 850         q_ctx = get_queue_ctx(m2m_ctx, type);
 851         ret = vb2_streamoff(&q_ctx->q, type);
 852         if (ret)
 853                 return ret;
 854
 855         m2m_dev = m2m_ctx->m2m_dev;
 856         spin_lock_irqsave(&m2m_dev->job_spinlock, flags_job);
 857         /* We should not be scheduled anymore, since we're dropping a queue. */
 858         if (m2m_ctx->job_flags & TRANS_QUEUED)
 859                 list_del(&m2m_ctx->queue);
 860         m2m_ctx->job_flags = 0;
 861
 862         spin_lock_irqsave(&q_ctx->rdy_spinlock, flags);
 863         /* Drop queue, since streamoff returns device to the same state as after
 864          * calling reqbufs. */
 865         INIT_LIST_HEAD(&q_ctx->rdy_queue);
 866         q_ctx->num_rdy = 0;
 867         spin_unlock_irqrestore(&q_ctx->rdy_spinlock, flags);
 868
 869         if (m2m_dev->curr_ctx == m2m_ctx) {
 870                 m2m_dev->curr_ctx = NULL;
 871                 wake_up(&m2m_ctx->finished);
 872         }
 873         spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags_job);
 874
 875         return 0;
 876 }
 877 EXPORT_SYMBOL_GPL(v4l2_m2m_streamoff);
 878
 879 static __poll_t v4l2_m2m_poll_for_data(struct file *file,
 880                                        struct v4l2_m2m_ctx *m2m_ctx,
 881                                        struct poll_table_struct *wait)
 882 {
 883         struct vb2_queue *src_q, *dst_q;
 884         __poll_t rc = 0;
 885         unsigned long flags;
 886
 887         src_q = v4l2_m2m_get_src_vq(m2m_ctx);
 888         dst_q = v4l2_m2m_get_dst_vq(m2m_ctx);
 889
 890         /*
 891          * There has to be at least one buffer queued on each queued_list, which
 892          * means either in driver already or waiting for driver to claim it
 893          * and start processing.
 894          */
 895         if ((!src_q->streaming || src_q->error ||
 896              list_empty(&src_q->queued_list)) &&
 897             (!dst_q->streaming || dst_q->error ||
 898              list_empty(&dst_q->queued_list)))
 899                 return EPOLLERR;
 900
 901         spin_lock_irqsave(&src_q->done_lock, flags);
 902         if (!list_empty(&src_q->done_list))
 903                 rc |= EPOLLOUT | EPOLLWRNORM;
 904         spin_unlock_irqrestore(&src_q->done_lock, flags);
 905
 906         spin_lock_irqsave(&dst_q->done_lock, flags);
 907         /*
 908          * If the last buffer was dequeued from the capture queue, signal
 909          * userspace. DQBUF(CAPTURE) will return -EPIPE.
 910          */
 911         if (!list_empty(&dst_q->done_list) || dst_q->last_buffer_dequeued)
 912                 rc |= EPOLLIN | EPOLLRDNORM;
 913         spin_unlock_irqrestore(&dst_q->done_lock, flags);
 914
 915         return rc;
 916 }
 917
 918 __poll_t v4l2_m2m_poll(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 919                        struct poll_table_struct *wait)
 920 {
 921         struct video_device *vfd = video_devdata(file);
 922         struct vb2_queue *src_q = v4l2_m2m_get_src_vq(m2m_ctx);
 923         struct vb2_queue *dst_q = v4l2_m2m_get_dst_vq(m2m_ctx);
 924         __poll_t req_events = poll_requested_events(wait);
 925         __poll_t rc = 0;
 926
 927         /*
 928          * poll_wait() MUST be called on the first invocation on all the
 929          * potential queues of interest, even if we are not interested in their
 930          * events during this first call. Failure to do so will result in
 931          * queue's events to be ignored because the poll_table won't be capable
 932          * of adding new wait queues thereafter.
 933          */
 934         poll_wait(file, &src_q->done_wq, wait);
 935         poll_wait(file, &dst_q->done_wq, wait);
 936
 937         if (req_events & (EPOLLOUT | EPOLLWRNORM | EPOLLIN | EPOLLRDNORM))
 938                 rc = v4l2_m2m_poll_for_data(file, m2m_ctx, wait);
 939
 940         if (test_bit(V4L2_FL_USES_V4L2_FH, &vfd->flags)) {
 941                 struct v4l2_fh *fh = file->private_data;
 942
 943                 poll_wait(file, &fh->wait, wait);
 944                 if (v4l2_event_pending(fh))
 945                         rc |= EPOLLPRI;
 946         }
 947
 948         return rc;
 949 }
 950 EXPORT_SYMBOL_GPL(v4l2_m2m_poll);
 951
 952 int v4l2_m2m_mmap(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 953                          struct vm_area_struct *vma)
 954 {
 955         unsigned long offset = vma->vm_pgoff << PAGE_SHIFT;
 956         struct vb2_queue *vq;
 957
 958         if (offset < DST_QUEUE_OFF_BASE) {
 959                 vq = v4l2_m2m_get_src_vq(m2m_ctx);
 960         } else {
 961                 vq = v4l2_m2m_get_dst_vq(m2m_ctx);
 962                 vma->vm_pgoff -= (DST_QUEUE_OFF_BASE >> PAGE_SHIFT);
 963         }
 964
 965         return vb2_mmap(vq, vma);
 966 }
 967 EXPORT_SYMBOL(v4l2_m2m_mmap);
 968
 969 #if defined(CONFIG_MEDIA_CONTROLLER)
 970 void v4l2_m2m_unregister_media_controller(struct v4l2_m2m_dev *m2m_dev)
 971 {
 972         media_remove_intf_links(&m2m_dev->intf_devnode->intf);
 973         media_devnode_remove(m2m_dev->intf_devnode);
 974
 975         media_entity_remove_links(m2m_dev->source);
 976         media_entity_remove_links(&m2m_dev->sink);
 977         media_entity_remove_links(&m2m_dev->proc);
 978         media_device_unregister_entity(m2m_dev->source);
 979         media_device_unregister_entity(&m2m_dev->sink);
 980         media_device_unregister_entity(&m2m_dev->proc);
 981         kfree(m2m_dev->source->name);
 982         kfree(m2m_dev->sink.name);
 983         kfree(m2m_dev->proc.name);
 984 }
 985 EXPORT_SYMBOL_GPL(v4l2_m2m_unregister_media_controller);
 986
 987 static int v4l2_m2m_register_entity(struct media_device *mdev,
 988         struct v4l2_m2m_dev *m2m_dev, enum v4l2_m2m_entity_type type,
 989         struct video_device *vdev, int function)
 990 {
 991         struct media_entity *entity;
 992         struct media_pad *pads;
 993         char *name;
 994         unsigned int len;
 995         int num_pads;
 996         int ret;
 997
 998         switch (type) {
 999         case MEM2MEM_ENT_TYPE_SOURCE:
1000                 entity = m2m_dev->source;
1001                 pads = &m2m_dev->source_pad;
1002                 pads[0].flags = MEDIA_PAD_FL_SOURCE;
1003                 num_pads = 1;
1004                 break;
1005         case MEM2MEM_ENT_TYPE_SINK:
1006                 entity = &m2m_dev->sink;
1007                 pads = &m2m_dev->sink_pad;
1008                 pads[0].flags = MEDIA_PAD_FL_SINK;
1009                 num_pads = 1;
1010                 break;
1011         case MEM2MEM_ENT_TYPE_PROC:
1012                 entity = &m2m_dev->proc;
1013                 pads = m2m_dev->proc_pads;
1014                 pads[0].flags = MEDIA_PAD_FL_SINK;
1015                 pads[1].flags = MEDIA_PAD_FL_SOURCE;
1016                 num_pads = 2;
1017                 break;
1018         default:
1019                 return -EINVAL;
1020         }
1021
1022         entity->obj_type = MEDIA_ENTITY_TYPE_BASE;
1023         if (type != MEM2MEM_ENT_TYPE_PROC) {
1024                 entity->info.dev.major = VIDEO_MAJOR;
1025                 entity->info.dev.minor = vdev->minor;
1026         }
1027         len = strlen(vdev->name) + 2 + strlen(m2m_entity_name[type]);
1028         name = kmalloc(len, GFP_KERNEL);
1029         if (!name)
1030                 return -ENOMEM;
1031         snprintf(name, len, "%s-%s", vdev->name, m2m_entity_name[type]);
1032         entity->name = name;
1033         entity->function = function;
1034
1035         ret = media_entity_pads_init(entity, num_pads, pads);
1036         if (ret)
1037                 return ret;
1038         ret = media_device_register_entity(mdev, entity);
1039         if (ret)
1040                 return ret;
1041
1042         return 0;
1043 }
1044
1045 int v4l2_m2m_register_media_controller(struct v4l2_m2m_dev *m2m_dev,
1046                 struct video_device *vdev, int function)
1047 {
1048         struct media_device *mdev = vdev->v4l2_dev->mdev;
1049         struct media_link *link;
1050         int ret;
1051
1052         if (!mdev)
1053                 return 0;
1054
1055         /* A memory-to-memory device consists in two
1056          * DMA engine and one video processing entities.
1057          * The DMA engine entities are linked to a V4L interface
1058          */
1059
1060         /* Create the three entities with their pads */
1061         m2m_dev->source = &vdev->entity;
1062         ret = v4l2_m2m_register_entity(mdev, m2m_dev,
1063                         MEM2MEM_ENT_TYPE_SOURCE, vdev, MEDIA_ENT_F_IO_V4L);
1064         if (ret)
1065                 return ret;
1066         ret = v4l2_m2m_register_entity(mdev, m2m_dev,
1067                         MEM2MEM_ENT_TYPE_PROC, vdev, function);
1068         if (ret)
1069                 goto err_rel_entity0;
1070         ret = v4l2_m2m_register_entity(mdev, m2m_dev,
1071                         MEM2MEM_ENT_TYPE_SINK, vdev, MEDIA_ENT_F_IO_V4L);
1072         if (ret)
1073                 goto err_rel_entity1;
1074
1075         /* Connect the three entities */
1076         ret = media_create_pad_link(m2m_dev->source, 0, &m2m_dev->proc, 0,
1077                         MEDIA_LNK_FL_IMMUTABLE | MEDIA_LNK_FL_ENABLED);
1078         if (ret)
1079                 goto err_rel_entity2;
1080
1081         ret = media_create_pad_link(&m2m_dev->proc, 1, &m2m_dev->sink, 0,
1082                         MEDIA_LNK_FL_IMMUTABLE | MEDIA_LNK_FL_ENABLED);
1083         if (ret)
1084                 goto err_rm_links0;
1085
1086         /* Create video interface */
1087         m2m_dev->intf_devnode = media_devnode_create(mdev,
1088                         MEDIA_INTF_T_V4L_VIDEO, 0,
1089                         VIDEO_MAJOR, vdev->minor);
1090         if (!m2m_dev->intf_devnode) {
1091                 ret = -ENOMEM;
1092                 goto err_rm_links1;
1093         }
1094
1095         /* Connect the two DMA engines to the interface */
1096         link = media_create_intf_link(m2m_dev->source,
1097                         &m2m_dev->intf_devnode->intf,
1098                         MEDIA_LNK_FL_IMMUTABLE | MEDIA_LNK_FL_ENABLED);
1099         if (!link) {
1100                 ret = -ENOMEM;
1101                 goto err_rm_devnode;
1102         }
1103
1104         link = media_create_intf_link(&m2m_dev->sink,
1105                         &m2m_dev->intf_devnode->intf,
1106                         MEDIA_LNK_FL_IMMUTABLE | MEDIA_LNK_FL_ENABLED);
1107         if (!link) {
1108                 ret = -ENOMEM;
1109                 goto err_rm_intf_link;
1110         }
1111         return 0;
1112
1113 err_rm_intf_link:
1114         media_remove_intf_links(&m2m_dev->intf_devnode->intf);
1115 err_rm_devnode:
1116         media_devnode_remove(m2m_dev->intf_devnode);
1117 err_rm_links1:
1118         media_entity_remove_links(&m2m_dev->sink);
1119 err_rm_links0:
1120         media_entity_remove_links(&m2m_dev->proc);
1121         media_entity_remove_links(m2m_dev->source);
1122 err_rel_entity2:
1123         media_device_unregister_entity(&m2m_dev->proc);
1124         kfree(m2m_dev->proc.name);
1125 err_rel_entity1:
1126         media_device_unregister_entity(&m2m_dev->sink);
1127         kfree(m2m_dev->sink.name);
1128 err_rel_entity0:
1129         media_device_unregister_entity(m2m_dev->source);
1130         kfree(m2m_dev->source->name);
1131         return ret;
1132         return 0;
1133 }
1134 EXPORT_SYMBOL_GPL(v4l2_m2m_register_media_controller);
1135 #endif
1136
1137 struct v4l2_m2m_dev *v4l2_m2m_init(const struct v4l2_m2m_ops *m2m_ops)
1138 {
1139         struct v4l2_m2m_dev *m2m_dev;
1140
1141         if (!m2m_ops || WARN_ON(!m2m_ops->device_run))
1142                 return ERR_PTR(-EINVAL);
1143
1144         m2m_dev = kzalloc(sizeof *m2m_dev, GFP_KERNEL);
1145         if (!m2m_dev)
1146                 return ERR_PTR(-ENOMEM);
1147
1148         m2m_dev->curr_ctx = NULL;
1149         m2m_dev->m2m_ops = m2m_ops;
1150         INIT_LIST_HEAD(&m2m_dev->job_queue);
1151         spin_lock_init(&m2m_dev->job_spinlock);
1152         INIT_WORK(&m2m_dev->job_work, v4l2_m2m_device_run_work);
1153
1154         return m2m_dev;
1155 }
1156 EXPORT_SYMBOL_GPL(v4l2_m2m_init);
1157
1158 void v4l2_m2m_release(struct v4l2_m2m_dev *m2m_dev)
1159 {
1160         kfree(m2m_dev);
1161 }
1162 EXPORT_SYMBOL_GPL(v4l2_m2m_release);
1163
1164 struct v4l2_m2m_ctx *v4l2_m2m_ctx_init(struct v4l2_m2m_dev *m2m_dev,
1165                 void *drv_priv,
1166                 int (*queue_init)(void *priv, struct vb2_queue *src_vq, struct vb2_queue *dst_vq))
1167 {
1168         struct v4l2_m2m_ctx *m2m_ctx;
1169         struct v4l2_m2m_queue_ctx *out_q_ctx, *cap_q_ctx;
1170         int ret;
1171
1172         m2m_ctx = kzalloc(sizeof *m2m_ctx, GFP_KERNEL);
1173         if (!m2m_ctx)
1174                 return ERR_PTR(-ENOMEM);
1175
1176         m2m_ctx->priv = drv_priv;
1177         m2m_ctx->m2m_dev = m2m_dev;
1178         init_waitqueue_head(&m2m_ctx->finished);
1179
1180         out_q_ctx = &m2m_ctx->out_q_ctx;
1181         cap_q_ctx = &m2m_ctx->cap_q_ctx;
1182
1183         INIT_LIST_HEAD(&out_q_ctx->rdy_queue);
1184         INIT_LIST_HEAD(&cap_q_ctx->rdy_queue);
1185         spin_lock_init(&out_q_ctx->rdy_spinlock);
1186         spin_lock_init(&cap_q_ctx->rdy_spinlock);
1187
1188         INIT_LIST_HEAD(&m2m_ctx->queue);
1189
1190         ret = queue_init(drv_priv, &out_q_ctx->q, &cap_q_ctx->q);
1191
1192         if (ret)
1193                 goto err;
1194         /*
1195          * Both queues should use same the mutex to lock the m2m context.
1196          * This lock is used in some v4l2_m2m_* helpers.
1197          */
1198         if (WARN_ON(out_q_ctx->q.lock != cap_q_ctx->q.lock)) {
1199                 ret = -EINVAL;
1200                 goto err;
1201         }
1202         m2m_ctx->q_lock = out_q_ctx->q.lock;
1203
1204         return m2m_ctx;
1205 err:
1206         kfree(m2m_ctx);
1207         return ERR_PTR(ret);
1208 }
1209 EXPORT_SYMBOL_GPL(v4l2_m2m_ctx_init);
1210
1211 void v4l2_m2m_ctx_release(struct v4l2_m2m_ctx *m2m_ctx)
1212 {
1213         /* wait until the current context is dequeued from job_queue */
1214         v4l2_m2m_cancel_job(m2m_ctx);
1215
1216         vb2_queue_release(&m2m_ctx->cap_q_ctx.q);
1217         vb2_queue_release(&m2m_ctx->out_q_ctx.q);
1218
1219         kfree(m2m_ctx);
1220 }
1221 EXPORT_SYMBOL_GPL(v4l2_m2m_ctx_release);
1222
1223 void v4l2_m2m_buf_queue(struct v4l2_m2m_ctx *m2m_ctx,
1224                 struct vb2_v4l2_buffer *vbuf)
1225 {
1226         struct v4l2_m2m_buffer *b = container_of(vbuf,
1227                                 struct v4l2_m2m_buffer, vb);
1228         struct v4l2_m2m_queue_ctx *q_ctx;
1229         unsigned long flags;
1230
1231         q_ctx = get_queue_ctx(m2m_ctx, vbuf->vb2_buf.vb2_queue->type);
1232         if (!q_ctx)
1233                 return;
1234
1235         spin_lock_irqsave(&q_ctx->rdy_spinlock, flags);
1236         list_add_tail(&b->list, &q_ctx->rdy_queue);
1237         q_ctx->num_rdy++;
1238         spin_unlock_irqrestore(&q_ctx->rdy_spinlock, flags);
1239 }
1240 EXPORT_SYMBOL_GPL(v4l2_m2m_buf_queue);
1241
1242 void v4l2_m2m_buf_copy_metadata(const struct vb2_v4l2_buffer *out_vb,
1243                                 struct vb2_v4l2_buffer *cap_vb,
1244                                 bool copy_frame_flags)
1245 {
1246         u32 mask = V4L2_BUF_FLAG_TIMECODE | V4L2_BUF_FLAG_TSTAMP_SRC_MASK;
1247
1248         if (copy_frame_flags)
1249                 mask |= V4L2_BUF_FLAG_KEYFRAME | V4L2_BUF_FLAG_PFRAME |
1250                         V4L2_BUF_FLAG_BFRAME;
1251
1252         cap_vb->vb2_buf.timestamp = out_vb->vb2_buf.timestamp;
1253
1254         if (out_vb->flags & V4L2_BUF_FLAG_TIMECODE)
1255                 cap_vb->timecode = out_vb->timecode;
1256         cap_vb->field = out_vb->field;
1257         cap_vb->flags &= ~mask;
1258         cap_vb->flags |= out_vb->flags & mask;
1259         cap_vb->vb2_buf.copied_timestamp = 1;
1260 }
1261 EXPORT_SYMBOL_GPL(v4l2_m2m_buf_copy_metadata);
1262
1263 void v4l2_m2m_request_queue(struct media_request *req)
1264 {
1265         struct media_request_object *obj, *obj_safe;
1266         struct v4l2_m2m_ctx *m2m_ctx = NULL;
1267
1268         /*
1269          * Queue all objects. Note that buffer objects are at the end of the
1270          * objects list, after all other object types. Once buffer objects
1271          * are queued, the driver might delete them immediately (if the driver
1272          * processes the buffer at once), so we have to use
1273          * list_for_each_entry_safe() to handle the case where the object we
1274          * queue is deleted.
1275          */
1276         list_for_each_entry_safe(obj, obj_safe, &req->objects, list) {
1277                 struct v4l2_m2m_ctx *m2m_ctx_obj;
1278                 struct vb2_buffer *vb;
1279
1280                 if (!obj->ops->queue)
1281                         continue;
1282
1283                 if (vb2_request_object_is_buffer(obj)) {
1284                         /* Sanity checks */
1285                         vb = container_of(obj, struct vb2_buffer, req_obj);
1286                         WARN_ON(!V4L2_TYPE_IS_OUTPUT(vb->vb2_queue->type));
1287                         m2m_ctx_obj = container_of(vb->vb2_queue,
1288                                                    struct v4l2_m2m_ctx,
1289                                                    out_q_ctx.q);
1290                         WARN_ON(m2m_ctx && m2m_ctx_obj != m2m_ctx);
1291                         m2m_ctx = m2m_ctx_obj;
1292                 }
1293
1294                 /*
1295                  * The buffer we queue here can in theory be immediately
1296                  * unbound, hence the use of list_for_each_entry_safe()
1297                  * above and why we call the queue op last.
1298                  */
1299                 obj->ops->queue(obj);
1300         }
1301
1302         WARN_ON(!m2m_ctx);
1303
1304         if (m2m_ctx)
1305                 v4l2_m2m_try_schedule(m2m_ctx);
1306 }
1307 EXPORT_SYMBOL_GPL(v4l2_m2m_request_queue);
1308
1309 /* Videobuf2 ioctl helpers */
1310
1311 int v4l2_m2m_ioctl_reqbufs(struct file *file, void *priv,
1312                                 struct v4l2_requestbuffers *rb)
1313 {
1314         struct v4l2_fh *fh = file->private_data;
1315
1316         return v4l2_m2m_reqbufs(file, fh->m2m_ctx, rb);
1317 }
1318 EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_reqbufs);
1319
1320 int v4l2_m2m_ioctl_create_bufs(struct file *file, void *priv,
1321                                 struct v4l2_create_buffers *create)
1322 {
1323         struct v4l2_fh *fh = file->private_data;
1324
1325         return v4l2_m2m_create_bufs(file, fh->m2m_ctx, create);
1326 }
1327 EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_create_bufs);
1328
1329 int v4l2_m2m_ioctl_querybuf(struct file *file, void *priv,
1330                                 struct v4l2_buffer *buf)
1331 {
1332         struct v4l2_fh *fh = file->private_data;
1333
1334         return v4l2_m2m_querybuf(file, fh->m2m_ctx, buf);
1335 }
1336 EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_querybuf);
1337
1338 int v4l2_m2m_ioctl_qbuf(struct file *file, void *priv,
1339                                 struct v4l2_buffer *buf)
1340 {
1341         struct v4l2_fh *fh = file->private_data;
1342
1343         return v4l2_m2m_qbuf(file, fh->m2m_ctx, buf);
1344 }
1345 EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_qbuf);
1346
1347 int v4l2_m2m_ioctl_dqbuf(struct file *file, void *priv,
1348                                 struct v4l2_buffer *buf)
1349 {
1350         struct v4l2_fh *fh = file->private_data;
1351
1352         return v4l2_m2m_dqbuf(file, fh->m2m_ctx, buf);
1353 }
1354 EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_dqbuf);
1355
1356 int v4l2_m2m_ioctl_prepare_buf(struct file *file, void *priv,
1357                                struct v4l2_buffer *buf)
1358 {
1359         struct v4l2_fh *fh = file->private_data;
1360
1361         return v4l2_m2m_prepare_buf(file, fh->m2m_ctx, buf);
1362 }
1363 EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_prepare_buf);
1364
1365 int v4l2_m2m_ioctl_expbuf(struct file *file, void *priv,
1366                                 struct v4l2_exportbuffer *eb)
1367 {
1368         struct v4l2_fh *fh = file->private_data;
1369
1370         return v4l2_m2m_expbuf(file, fh->m2m_ctx, eb);
1371 }
1372 EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_expbuf);
1373
1374 int v4l2_m2m_ioctl_streamon(struct file *file, void *priv,
1375                                 enum v4l2_buf_type type)
1376 {
1377         struct v4l2_fh *fh = file->private_data;
1378
1379         return v4l2_m2m_streamon(file, fh->m2m_ctx, type);
1380 }
1381 EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_streamon);
1382
1383 int v4l2_m2m_ioctl_streamoff(struct file *file, void *priv,
1384                                 enum v4l2_buf_type type)
1385 {
1386         struct v4l2_fh *fh = file->private_data;
1387
1388         return v4l2_m2m_streamoff(file, fh->m2m_ctx, type);
1389 }
1390 EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_streamoff);
1391
1392 int v4l2_m2m_ioctl_try_encoder_cmd(struct file *file, void *fh,
1393                                    struct v4l2_encoder_cmd *ec)
1394 {
1395         if (ec->cmd != V4L2_ENC_CMD_STOP && ec->cmd != V4L2_ENC_CMD_START)
1396                 return -EINVAL;
1397
1398         ec->flags = 0;
1399         return 0;
1400 }
1401 EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_try_encoder_cmd);
1402
1403 int v4l2_m2m_ioctl_try_decoder_cmd(struct file *file, void *fh,
1404                                    struct v4l2_decoder_cmd *dc)
1405 {
1406         if (dc->cmd != V4L2_DEC_CMD_STOP && dc->cmd != V4L2_DEC_CMD_START)
1407                 return -EINVAL;
1408
1409         dc->flags = 0;
1410
1411         if (dc->cmd == V4L2_DEC_CMD_STOP) {
1412                 dc->stop.pts = 0;
1413         } else if (dc->cmd == V4L2_DEC_CMD_START) {
1414                 dc->start.speed = 0;
1415                 dc->start.format = V4L2_DEC_START_FMT_NONE;
1416         }
1417         return 0;
1418 }
1419 EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_try_decoder_cmd);
1420
1421 /*
1422  * Updates the encoding state on ENC_CMD_STOP/ENC_CMD_START
1423  * Should be called from the encoder driver encoder_cmd() callback
1424  */
1425 int v4l2_m2m_encoder_cmd(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
1426                          struct v4l2_encoder_cmd *ec)
1427 {
1428         if (ec->cmd != V4L2_ENC_CMD_STOP && ec->cmd != V4L2_ENC_CMD_START)
1429                 return -EINVAL;
1430
1431         if (ec->cmd == V4L2_ENC_CMD_STOP)
1432                 return v4l2_update_last_buf_state(m2m_ctx);
1433
1434         if (m2m_ctx->is_draining)
1435                 return -EBUSY;
1436
1437         if (m2m_ctx->has_stopped)
1438                 m2m_ctx->has_stopped = false;
1439
1440         return 0;
1441 }
1442 EXPORT_SYMBOL_GPL(v4l2_m2m_encoder_cmd);
1443
1444 /*
1445  * Updates the decoding state on DEC_CMD_STOP/DEC_CMD_START
1446  * Should be called from the decoder driver decoder_cmd() callback
1447  */
1448 int v4l2_m2m_decoder_cmd(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
1449                          struct v4l2_decoder_cmd *dc)
1450 {
1451         if (dc->cmd != V4L2_DEC_CMD_STOP && dc->cmd != V4L2_DEC_CMD_START)
1452                 return -EINVAL;
1453
1454         if (dc->cmd == V4L2_DEC_CMD_STOP)
1455                 return v4l2_update_last_buf_state(m2m_ctx);
1456
1457         if (m2m_ctx->is_draining)
1458                 return -EBUSY;
1459
1460         if (m2m_ctx->has_stopped)
1461                 m2m_ctx->has_stopped = false;
1462
1463         return 0;
1464 }
1465 EXPORT_SYMBOL_GPL(v4l2_m2m_decoder_cmd);
1466
1467 int v4l2_m2m_ioctl_encoder_cmd(struct file *file, void *priv,
1468                                struct v4l2_encoder_cmd *ec)
1469 {
1470         struct v4l2_fh *fh = file->private_data;
1471
1472         return v4l2_m2m_encoder_cmd(file, fh->m2m_ctx, ec);
1473 }
1474 EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_encoder_cmd);
1475
1476 int v4l2_m2m_ioctl_decoder_cmd(struct file *file, void *priv,
1477                                struct v4l2_decoder_cmd *dc)
1478 {
1479         struct v4l2_fh *fh = file->private_data;
1480
1481         return v4l2_m2m_decoder_cmd(file, fh->m2m_ctx, dc);
1482 }
1483 EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_decoder_cmd);
1484
1485 int v4l2_m2m_ioctl_stateless_try_decoder_cmd(struct file *file, void *fh,
1486                                              struct v4l2_decoder_cmd *dc)
1487 {
1488         if (dc->cmd != V4L2_DEC_CMD_FLUSH)
1489                 return -EINVAL;
1490
1491         dc->flags = 0;
1492
1493         return 0;
1494 }
1495 EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_stateless_try_decoder_cmd);
1496
1497 int v4l2_m2m_ioctl_stateless_decoder_cmd(struct file *file, void *priv,
1498                                          struct v4l2_decoder_cmd *dc)
1499 {
1500         struct v4l2_fh *fh = file->private_data;
1501         struct vb2_v4l2_buffer *out_vb, *cap_vb;
1502         struct v4l2_m2m_dev *m2m_dev = fh->m2m_ctx->m2m_dev;
1503         unsigned long flags;
1504         int ret;
1505
1506         ret = v4l2_m2m_ioctl_stateless_try_decoder_cmd(file, priv, dc);
1507         if (ret < 0)
1508                 return ret;
1509
1510         spin_lock_irqsave(&m2m_dev->job_spinlock, flags);
1511         out_vb = v4l2_m2m_last_src_buf(fh->m2m_ctx);
1512         cap_vb = v4l2_m2m_last_dst_buf(fh->m2m_ctx);
1513
1514         /*
1515          * If there is an out buffer pending, then clear any HOLD flag.
1516          *
1517          * By clearing this flag we ensure that when this output
1518          * buffer is processed any held capture buffer will be released.
1519          */
1520         if (out_vb) {
1521                 out_vb->flags &= ~V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF;
1522         } else if (cap_vb && cap_vb->is_held) {
1523                 /*
1524                  * If there were no output buffers, but there is a
1525                  * capture buffer that is held, then release that
1526                  * buffer.
1527                  */
1528                 cap_vb->is_held = false;
1529                 v4l2_m2m_dst_buf_remove(fh->m2m_ctx);
1530                 v4l2_m2m_buf_done(cap_vb, VB2_BUF_STATE_DONE);
1531         }
1532         spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
1533
1534         return 0;
1535 }
1536 EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_stateless_decoder_cmd);
1537
1538 /*
1539  * v4l2_file_operations helpers. It is assumed here same lock is used
1540  * for the output and the capture buffer queue.
1541  */
1542
1543 int v4l2_m2m_fop_mmap(struct file *file, struct vm_area_struct *vma)
1544 {
1545         struct v4l2_fh *fh = file->private_data;
1546
1547         return v4l2_m2m_mmap(file, fh->m2m_ctx, vma);
1548 }
1549 EXPORT_SYMBOL_GPL(v4l2_m2m_fop_mmap);
1550
1551 __poll_t v4l2_m2m_fop_poll(struct file *file, poll_table *wait)
1552 {
1553         struct v4l2_fh *fh = file->private_data;
1554         struct v4l2_m2m_ctx *m2m_ctx = fh->m2m_ctx;
1555         __poll_t ret;
1556
1557         if (m2m_ctx->q_lock)
1558                 mutex_lock(m2m_ctx->q_lock);
1559
1560         ret = v4l2_m2m_poll(file, m2m_ctx, wait);
1561
1562         if (m2m_ctx->q_lock)
1563                 mutex_unlock(m2m_ctx->q_lock);
1564
1565         return ret;
1566 }
1567 EXPORT_SYMBOL_GPL(v4l2_m2m_fop_poll);
1568