1 // SPDX-License-Identifier: GPL-2.0
3 * Rockchip VPU codec driver
5 * Copyright (C) 2018 Collabora, Ltd.
6 * Copyright 2018 Google LLC.
7 * Tomasz Figa <tfiga@chromium.org>
9 * Based on s5p-mfc driver by Samsung Electronics Co., Ltd.
10 * Copyright (C) 2011 Samsung Electronics Co., Ltd.
13 #include <linux/clk.h>
14 #include <linux/module.h>
16 #include <linux/platform_device.h>
18 #include <linux/pm_runtime.h>
19 #include <linux/slab.h>
20 #include <linux/videodev2.h>
21 #include <linux/workqueue.h>
22 #include <media/v4l2-event.h>
23 #include <media/v4l2-mem2mem.h>
24 #include <media/videobuf2-core.h>
25 #include <media/videobuf2-vmalloc.h>
27 #include "rockchip_vpu_common.h"
28 #include "rockchip_vpu.h"
29 #include "rockchip_vpu_hw.h"
31 #define DRIVER_NAME "rockchip-vpu"
33 int rockchip_vpu_debug;
34 module_param_named(debug, rockchip_vpu_debug, int, 0644);
35 MODULE_PARM_DESC(debug,
36 "Debug level - higher value produces more verbose messages");
38 static void rockchip_vpu_job_finish(struct rockchip_vpu_dev *vpu,
39 struct rockchip_vpu_ctx *ctx,
40 unsigned int bytesused,
41 enum vb2_buffer_state result)
43 struct vb2_v4l2_buffer *src, *dst;
46 pm_runtime_mark_last_busy(vpu->dev);
47 pm_runtime_put_autosuspend(vpu->dev);
48 clk_bulk_disable(vpu->variant->num_clocks, vpu->clocks);
50 src = v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx);
51 dst = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
58 src->sequence = ctx->sequence_out++;
59 dst->sequence = ctx->sequence_cap++;
61 dst->field = src->field;
62 if (src->flags & V4L2_BUF_FLAG_TIMECODE)
63 dst->timecode = src->timecode;
64 dst->vb2_buf.timestamp = src->vb2_buf.timestamp;
65 dst->flags &= ~(V4L2_BUF_FLAG_TSTAMP_SRC_MASK |
66 V4L2_BUF_FLAG_TIMECODE);
67 dst->flags |= src->flags & (V4L2_BUF_FLAG_TSTAMP_SRC_MASK |
68 V4L2_BUF_FLAG_TIMECODE);
70 avail_size = vb2_plane_size(&dst->vb2_buf, 0) -
71 ctx->vpu_dst_fmt->header_size;
72 if (bytesused <= avail_size) {
73 if (ctx->bounce_buf) {
74 memcpy(vb2_plane_vaddr(&dst->vb2_buf, 0) +
75 ctx->vpu_dst_fmt->header_size,
76 ctx->bounce_buf, bytesused);
78 dst->vb2_buf.planes[0].bytesused =
79 ctx->vpu_dst_fmt->header_size + bytesused;
81 result = VB2_BUF_STATE_ERROR;
84 v4l2_m2m_buf_done(src, result);
85 v4l2_m2m_buf_done(dst, result);
87 v4l2_m2m_job_finish(vpu->m2m_dev, ctx->fh.m2m_ctx);
90 void rockchip_vpu_irq_done(struct rockchip_vpu_dev *vpu,
91 unsigned int bytesused,
92 enum vb2_buffer_state result)
94 struct rockchip_vpu_ctx *ctx =
95 v4l2_m2m_get_curr_priv(vpu->m2m_dev);
98 * If cancel_delayed_work returns false
99 * the timeout expired. The watchdog is running,
100 * and will take care of finishing the job.
102 if (cancel_delayed_work(&vpu->watchdog_work))
103 rockchip_vpu_job_finish(vpu, ctx, bytesused, result);
106 void rockchip_vpu_watchdog(struct work_struct *work)
108 struct rockchip_vpu_dev *vpu;
109 struct rockchip_vpu_ctx *ctx;
111 vpu = container_of(to_delayed_work(work),
112 struct rockchip_vpu_dev, watchdog_work);
113 ctx = v4l2_m2m_get_curr_priv(vpu->m2m_dev);
115 vpu_err("frame processing timed out!\n");
116 ctx->codec_ops->reset(ctx);
117 rockchip_vpu_job_finish(vpu, ctx, 0, VB2_BUF_STATE_ERROR);
121 static void device_run(void *priv)
123 struct rockchip_vpu_ctx *ctx = priv;
126 ret = clk_bulk_enable(ctx->dev->variant->num_clocks, ctx->dev->clocks);
129 ret = pm_runtime_get_sync(ctx->dev->dev);
133 ctx->codec_ops->run(ctx);
137 rockchip_vpu_job_finish(ctx->dev, ctx, 0, VB2_BUF_STATE_ERROR);
140 static struct v4l2_m2m_ops vpu_m2m_ops = {
141 .device_run = device_run,
145 enc_queue_init(void *priv, struct vb2_queue *src_vq, struct vb2_queue *dst_vq)
147 struct rockchip_vpu_ctx *ctx = priv;
150 src_vq->type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
151 src_vq->io_modes = VB2_MMAP | VB2_DMABUF;
152 src_vq->drv_priv = ctx;
153 src_vq->ops = &rockchip_vpu_enc_queue_ops;
154 src_vq->mem_ops = &vb2_dma_contig_memops;
157 * Driver does mostly sequential access, so sacrifice TLB efficiency
158 * for faster allocation. Also, no CPU access on the source queue,
159 * so no kernel mapping needed.
161 src_vq->dma_attrs = DMA_ATTR_ALLOC_SINGLE_PAGES |
162 DMA_ATTR_NO_KERNEL_MAPPING;
163 src_vq->buf_struct_size = sizeof(struct v4l2_m2m_buffer);
164 src_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY;
165 src_vq->lock = &ctx->dev->vpu_mutex;
166 src_vq->dev = ctx->dev->v4l2_dev.dev;
168 ret = vb2_queue_init(src_vq);
173 * The CAPTURE queue doesn't need dma memory,
174 * as the CPU needs to create the JPEG frames,
175 * from the hardware-produced JPEG payload.
177 * For the DMA destination buffer, we use
180 dst_vq->type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
181 dst_vq->io_modes = VB2_MMAP | VB2_DMABUF;
182 dst_vq->drv_priv = ctx;
183 dst_vq->ops = &rockchip_vpu_enc_queue_ops;
184 dst_vq->mem_ops = &vb2_vmalloc_memops;
185 dst_vq->buf_struct_size = sizeof(struct v4l2_m2m_buffer);
186 dst_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY;
187 dst_vq->lock = &ctx->dev->vpu_mutex;
188 dst_vq->dev = ctx->dev->v4l2_dev.dev;
190 return vb2_queue_init(dst_vq);
193 static int rockchip_vpu_s_ctrl(struct v4l2_ctrl *ctrl)
195 struct rockchip_vpu_ctx *ctx;
197 ctx = container_of(ctrl->handler,
198 struct rockchip_vpu_ctx, ctrl_handler);
200 vpu_debug(1, "s_ctrl: id = %d, val = %d\n", ctrl->id, ctrl->val);
203 case V4L2_CID_JPEG_COMPRESSION_QUALITY:
204 ctx->jpeg_quality = ctrl->val;
213 static const struct v4l2_ctrl_ops rockchip_vpu_ctrl_ops = {
214 .s_ctrl = rockchip_vpu_s_ctrl,
217 static int rockchip_vpu_ctrls_setup(struct rockchip_vpu_dev *vpu,
218 struct rockchip_vpu_ctx *ctx)
220 v4l2_ctrl_handler_init(&ctx->ctrl_handler, 1);
221 if (vpu->variant->codec & RK_VPU_CODEC_JPEG) {
222 v4l2_ctrl_new_std(&ctx->ctrl_handler, &rockchip_vpu_ctrl_ops,
223 V4L2_CID_JPEG_COMPRESSION_QUALITY,
225 if (ctx->ctrl_handler.error) {
226 vpu_err("Adding JPEG control failed %d\n",
227 ctx->ctrl_handler.error);
228 v4l2_ctrl_handler_free(&ctx->ctrl_handler);
229 return ctx->ctrl_handler.error;
233 return v4l2_ctrl_handler_setup(&ctx->ctrl_handler);
237 * V4L2 file operations.
240 static int rockchip_vpu_open(struct file *filp)
242 struct rockchip_vpu_dev *vpu = video_drvdata(filp);
243 struct video_device *vdev = video_devdata(filp);
244 struct rockchip_vpu_ctx *ctx;
248 * We do not need any extra locking here, because we operate only
249 * on local data here, except reading few fields from dev, which
250 * do not change through device's lifetime (which is guaranteed by
251 * reference on module from open()) and V4L2 internal objects (such
252 * as vdev and ctx->fh), which have proper locking done in respective
253 * helper functions used here.
256 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
261 if (vdev == vpu->vfd_enc)
262 ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(vpu->m2m_dev, ctx,
265 ctx->fh.m2m_ctx = ERR_PTR(-ENODEV);
266 if (IS_ERR(ctx->fh.m2m_ctx)) {
267 ret = PTR_ERR(ctx->fh.m2m_ctx);
272 v4l2_fh_init(&ctx->fh, vdev);
273 filp->private_data = &ctx->fh;
274 v4l2_fh_add(&ctx->fh);
276 if (vdev == vpu->vfd_enc) {
277 rockchip_vpu_enc_reset_dst_fmt(vpu, ctx);
278 rockchip_vpu_enc_reset_src_fmt(vpu, ctx);
281 ret = rockchip_vpu_ctrls_setup(vpu, ctx);
283 vpu_err("Failed to set up controls\n");
286 ctx->fh.ctrl_handler = &ctx->ctrl_handler;
291 v4l2_fh_del(&ctx->fh);
292 v4l2_fh_exit(&ctx->fh);
297 static int rockchip_vpu_release(struct file *filp)
299 struct rockchip_vpu_ctx *ctx =
300 container_of(filp->private_data, struct rockchip_vpu_ctx, fh);
303 * No need for extra locking because this was the last reference
306 v4l2_m2m_ctx_release(ctx->fh.m2m_ctx);
307 v4l2_fh_del(&ctx->fh);
308 v4l2_fh_exit(&ctx->fh);
309 v4l2_ctrl_handler_free(&ctx->ctrl_handler);
315 static const struct v4l2_file_operations rockchip_vpu_fops = {
316 .owner = THIS_MODULE,
317 .open = rockchip_vpu_open,
318 .release = rockchip_vpu_release,
319 .poll = v4l2_m2m_fop_poll,
320 .unlocked_ioctl = video_ioctl2,
321 .mmap = v4l2_m2m_fop_mmap,
324 static const struct of_device_id of_rockchip_vpu_match[] = {
325 { .compatible = "rockchip,rk3399-vpu", .data = &rk3399_vpu_variant, },
326 { .compatible = "rockchip,rk3288-vpu", .data = &rk3288_vpu_variant, },
329 MODULE_DEVICE_TABLE(of, of_rockchip_vpu_match);
331 static int rockchip_vpu_video_device_register(struct rockchip_vpu_dev *vpu)
333 const struct of_device_id *match;
334 struct video_device *vfd;
337 match = of_match_node(of_rockchip_vpu_match, vpu->dev->of_node);
338 vfd = video_device_alloc();
340 v4l2_err(&vpu->v4l2_dev, "Failed to allocate video device\n");
344 vfd->fops = &rockchip_vpu_fops;
345 vfd->release = video_device_release;
346 vfd->lock = &vpu->vpu_mutex;
347 vfd->v4l2_dev = &vpu->v4l2_dev;
348 vfd->vfl_dir = VFL_DIR_M2M;
349 vfd->device_caps = V4L2_CAP_STREAMING | V4L2_CAP_VIDEO_M2M_MPLANE;
350 vfd->ioctl_ops = &rockchip_vpu_enc_ioctl_ops;
351 snprintf(vfd->name, sizeof(vfd->name), "%s-enc", match->compatible);
353 video_set_drvdata(vfd, vpu);
355 ret = video_register_device(vfd, VFL_TYPE_GRABBER, -1);
357 v4l2_err(&vpu->v4l2_dev, "Failed to register video device\n");
360 v4l2_info(&vpu->v4l2_dev, "registered as /dev/video%d\n", vfd->num);
362 function = MEDIA_ENT_F_PROC_VIDEO_ENCODER;
363 ret = v4l2_m2m_register_media_controller(vpu->m2m_dev, vfd, function);
365 v4l2_err(&vpu->v4l2_dev, "Failed to init mem2mem media controller\n");
366 goto err_unreg_video;
371 video_unregister_device(vfd);
373 video_device_release(vfd);
377 static int rockchip_vpu_probe(struct platform_device *pdev)
379 const struct of_device_id *match;
380 struct rockchip_vpu_dev *vpu;
381 struct resource *res;
384 vpu = devm_kzalloc(&pdev->dev, sizeof(*vpu), GFP_KERNEL);
388 vpu->dev = &pdev->dev;
390 mutex_init(&vpu->vpu_mutex);
391 spin_lock_init(&vpu->irqlock);
393 match = of_match_node(of_rockchip_vpu_match, pdev->dev.of_node);
394 vpu->variant = match->data;
396 INIT_DELAYED_WORK(&vpu->watchdog_work, rockchip_vpu_watchdog);
398 for (i = 0; i < vpu->variant->num_clocks; i++)
399 vpu->clocks[i].id = vpu->variant->clk_names[i];
400 ret = devm_clk_bulk_get(&pdev->dev, vpu->variant->num_clocks,
405 res = platform_get_resource(vpu->pdev, IORESOURCE_MEM, 0);
406 vpu->base = devm_ioremap_resource(vpu->dev, res);
407 if (IS_ERR(vpu->base))
408 return PTR_ERR(vpu->base);
409 vpu->enc_base = vpu->base + vpu->variant->enc_offset;
411 ret = dma_set_coherent_mask(vpu->dev, DMA_BIT_MASK(32));
413 dev_err(vpu->dev, "Could not set DMA coherent mask.\n");
417 if (vpu->variant->vepu_irq) {
420 irq = platform_get_irq_byname(vpu->pdev, "vepu");
422 dev_err(vpu->dev, "Could not get vepu IRQ.\n");
426 ret = devm_request_irq(vpu->dev, irq, vpu->variant->vepu_irq,
427 0, dev_name(vpu->dev), vpu);
429 dev_err(vpu->dev, "Could not request vepu IRQ.\n");
434 ret = vpu->variant->init(vpu);
436 dev_err(&pdev->dev, "Failed to init VPU hardware\n");
440 pm_runtime_set_autosuspend_delay(vpu->dev, 100);
441 pm_runtime_use_autosuspend(vpu->dev);
442 pm_runtime_enable(vpu->dev);
444 ret = clk_bulk_prepare(vpu->variant->num_clocks, vpu->clocks);
446 dev_err(&pdev->dev, "Failed to prepare clocks\n");
450 ret = v4l2_device_register(&pdev->dev, &vpu->v4l2_dev);
452 dev_err(&pdev->dev, "Failed to register v4l2 device\n");
453 goto err_clk_unprepare;
455 platform_set_drvdata(pdev, vpu);
457 vpu->m2m_dev = v4l2_m2m_init(&vpu_m2m_ops);
458 if (IS_ERR(vpu->m2m_dev)) {
459 v4l2_err(&vpu->v4l2_dev, "Failed to init mem2mem device\n");
460 ret = PTR_ERR(vpu->m2m_dev);
464 vpu->mdev.dev = vpu->dev;
465 strscpy(vpu->mdev.model, DRIVER_NAME, sizeof(vpu->mdev.model));
466 media_device_init(&vpu->mdev);
467 vpu->v4l2_dev.mdev = &vpu->mdev;
469 ret = rockchip_vpu_video_device_register(vpu);
471 dev_err(&pdev->dev, "Failed to register encoder\n");
475 ret = media_device_register(&vpu->mdev);
477 v4l2_err(&vpu->v4l2_dev, "Failed to register mem2mem media device\n");
478 goto err_video_dev_unreg;
483 video_unregister_device(vpu->vfd_enc);
484 video_device_release(vpu->vfd_enc);
487 v4l2_m2m_release(vpu->m2m_dev);
489 v4l2_device_unregister(&vpu->v4l2_dev);
491 clk_bulk_unprepare(vpu->variant->num_clocks, vpu->clocks);
492 pm_runtime_disable(vpu->dev);
496 static int rockchip_vpu_remove(struct platform_device *pdev)
498 struct rockchip_vpu_dev *vpu = platform_get_drvdata(pdev);
500 v4l2_info(&vpu->v4l2_dev, "Removing %s\n", pdev->name);
502 media_device_unregister(&vpu->mdev);
503 v4l2_m2m_unregister_media_controller(vpu->m2m_dev);
504 v4l2_m2m_release(vpu->m2m_dev);
505 media_device_cleanup(&vpu->mdev);
507 video_unregister_device(vpu->vfd_enc);
508 video_device_release(vpu->vfd_enc);
510 v4l2_device_unregister(&vpu->v4l2_dev);
511 clk_bulk_unprepare(vpu->variant->num_clocks, vpu->clocks);
512 pm_runtime_disable(vpu->dev);
516 static const struct dev_pm_ops rockchip_vpu_pm_ops = {
517 SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
518 pm_runtime_force_resume)
521 static struct platform_driver rockchip_vpu_driver = {
522 .probe = rockchip_vpu_probe,
523 .remove = rockchip_vpu_remove,
526 .of_match_table = of_match_ptr(of_rockchip_vpu_match),
527 .pm = &rockchip_vpu_pm_ops,
530 module_platform_driver(rockchip_vpu_driver);
532 MODULE_LICENSE("GPL v2");
533 MODULE_AUTHOR("Alpha Lin <Alpha.Lin@Rock-Chips.com>");
534 MODULE_AUTHOR("Tomasz Figa <tfiga@chromium.org>");
535 MODULE_AUTHOR("Ezequiel Garcia <ezequiel@collabora.com>");
536 MODULE_DESCRIPTION("Rockchip VPU codec driver");