drivers/gpu/host1x/hw/channel_hw.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Tegra host1x Channel
   4  *
   5  * Copyright (c) 2010-2013, NVIDIA Corporation.
   6  */
   7
   8 #include <linux/host1x.h>
   9 #include <linux/iommu.h>
  10 #include <linux/slab.h>
  11
  12 #include <trace/events/host1x.h>
  13
  14 #include "../channel.h"
  15 #include "../dev.h"
  16 #include "../intr.h"
  17 #include "../job.h"
  18
  19 #define TRACE_MAX_LENGTH 128U
  20
  21 static void trace_write_gather(struct host1x_cdma *cdma, struct host1x_bo *bo,
  22                                u32 offset, u32 words)
  23 {
  24         struct device *dev = cdma_to_channel(cdma)->dev;
  25         void *mem = NULL;
  26
  27         if (host1x_debug_trace_cmdbuf)
  28                 mem = host1x_bo_mmap(bo);
  29
  30         if (mem) {
  31                 u32 i;
  32                 /*
  33                  * Write in batches of 128 as there seems to be a limit
  34                  * of how much you can output to ftrace at once.
  35                  */
  36                 for (i = 0; i < words; i += TRACE_MAX_LENGTH) {
  37                         u32 num_words = min(words - i, TRACE_MAX_LENGTH);
  38
  39                         offset += i * sizeof(u32);
  40
  41                         trace_host1x_cdma_push_gather(dev_name(dev), bo,
  42                                                       num_words, offset,
  43                                                       mem);
  44                 }
  45
  46                 host1x_bo_munmap(bo, mem);
  47         }
  48 }
  49
  50 static void submit_wait(struct host1x_cdma *cdma, u32 id, u32 threshold,
  51                         u32 next_class)
  52 {
  53 #if HOST1X_HW >= 2
  54         host1x_cdma_push_wide(cdma,
  55                 host1x_opcode_setclass(
  56                         HOST1X_CLASS_HOST1X,
  57                         HOST1X_UCLASS_LOAD_SYNCPT_PAYLOAD_32,
  58                         /* WAIT_SYNCPT_32 is at SYNCPT_PAYLOAD_32+2 */
  59                         BIT(0) | BIT(2)
  60                 ),
  61                 threshold,
  62                 id,
  63                 host1x_opcode_setclass(next_class, 0, 0)
  64         );
  65 #else
  66         /* TODO add waitchk or use waitbases or other mitigation */
  67         host1x_cdma_push(cdma,
  68                 host1x_opcode_setclass(
  69                         HOST1X_CLASS_HOST1X,
  70                         host1x_uclass_wait_syncpt_r(),
  71                         BIT(0)
  72                 ),
  73                 host1x_class_host_wait_syncpt(id, threshold)
  74         );
  75         host1x_cdma_push(cdma,
  76                 host1x_opcode_setclass(next_class, 0, 0),
  77                 HOST1X_OPCODE_NOP
  78         );
  79 #endif
  80 }
  81
  82 static void submit_gathers(struct host1x_job *job, u32 job_syncpt_base)
  83 {
  84         struct host1x_cdma *cdma = &job->channel->cdma;
  85 #if HOST1X_HW < 6
  86         struct device *dev = job->channel->dev;
  87 #endif
  88         unsigned int i;
  89         u32 threshold;
  90
  91         for (i = 0; i < job->num_cmds; i++) {
  92                 struct host1x_job_cmd *cmd = &job->cmds[i];
  93
  94                 if (cmd->is_wait) {
  95                         if (cmd->wait.relative)
  96                                 threshold = job_syncpt_base + cmd->wait.threshold;
  97                         else
  98                                 threshold = cmd->wait.threshold;
  99
 100                         submit_wait(cdma, cmd->wait.id, threshold, cmd->wait.next_class);
 101                 } else {
 102                         struct host1x_job_gather *g = &cmd->gather;
 103
 104                         dma_addr_t addr = g->base + g->offset;
 105                         u32 op2, op3;
 106
 107                         op2 = lower_32_bits(addr);
 108                         op3 = upper_32_bits(addr);
 109
 110                         trace_write_gather(cdma, g->bo, g->offset, g->words);
 111
 112                         if (op3 != 0) {
 113 #if HOST1X_HW >= 6
 114                                 u32 op1 = host1x_opcode_gather_wide(g->words);
 115                                 u32 op4 = HOST1X_OPCODE_NOP;
 116
 117                                 host1x_cdma_push_wide(cdma, op1, op2, op3, op4);
 118 #else
 119                                 dev_err(dev, "invalid gather for push buffer %pad\n",
 120                                         &addr);
 121                                 continue;
 122 #endif
 123                         } else {
 124                                 u32 op1 = host1x_opcode_gather(g->words);
 125
 126                                 host1x_cdma_push(cdma, op1, op2);
 127                         }
 128                 }
 129         }
 130 }
 131
 132 static inline void synchronize_syncpt_base(struct host1x_job *job)
 133 {
 134         struct host1x_syncpt *sp = job->syncpt;
 135         unsigned int id;
 136         u32 value;
 137
 138         value = host1x_syncpt_read_max(sp);
 139         id = sp->base->id;
 140
 141         host1x_cdma_push(&job->channel->cdma,
 142                          host1x_opcode_setclass(HOST1X_CLASS_HOST1X,
 143                                 HOST1X_UCLASS_LOAD_SYNCPT_BASE, 1),
 144                          HOST1X_UCLASS_LOAD_SYNCPT_BASE_BASE_INDX_F(id) |
 145                          HOST1X_UCLASS_LOAD_SYNCPT_BASE_VALUE_F(value));
 146 }
 147
 148 static void host1x_channel_set_streamid(struct host1x_channel *channel)
 149 {
 150 #if HOST1X_HW >= 6
 151         u32 sid = 0x7f;
 152 #ifdef CONFIG_IOMMU_API
 153         struct iommu_fwspec *spec = dev_iommu_fwspec_get(channel->dev->parent);
 154         if (spec)
 155                 sid = spec->ids[0] & 0xffff;
 156 #endif
 157
 158         host1x_ch_writel(channel, sid, HOST1X_CHANNEL_SMMU_STREAMID);
 159 #endif
 160 }
 161
 162 static int channel_submit(struct host1x_job *job)
 163 {
 164         struct host1x_channel *ch = job->channel;
 165         struct host1x_syncpt *sp = job->syncpt;
 166         u32 user_syncpt_incrs = job->syncpt_incrs;
 167         u32 prev_max = 0;
 168         u32 syncval;
 169         int err;
 170         struct host1x_waitlist *completed_waiter = NULL;
 171         struct host1x *host = dev_get_drvdata(ch->dev->parent);
 172
 173         trace_host1x_channel_submit(dev_name(ch->dev),
 174                                     job->num_cmds, job->num_relocs,
 175                                     job->syncpt->id, job->syncpt_incrs);
 176
 177         /* before error checks, return current max */
 178         prev_max = job->syncpt_end = host1x_syncpt_read_max(sp);
 179
 180         /* get submit lock */
 181         err = mutex_lock_interruptible(&ch->submitlock);
 182         if (err)
 183                 goto error;
 184
 185         completed_waiter = kzalloc(sizeof(*completed_waiter), GFP_KERNEL);
 186         if (!completed_waiter) {
 187                 mutex_unlock(&ch->submitlock);
 188                 err = -ENOMEM;
 189                 goto error;
 190         }
 191
 192         host1x_channel_set_streamid(ch);
 193
 194         /* begin a CDMA submit */
 195         err = host1x_cdma_begin(&ch->cdma, job);
 196         if (err) {
 197                 mutex_unlock(&ch->submitlock);
 198                 goto error;
 199         }
 200
 201         if (job->serialize) {
 202                 /*
 203                  * Force serialization by inserting a host wait for the
 204                  * previous job to finish before this one can commence.
 205                  */
 206                 host1x_cdma_push(&ch->cdma,
 207                                  host1x_opcode_setclass(HOST1X_CLASS_HOST1X,
 208                                         host1x_uclass_wait_syncpt_r(), 1),
 209                                  host1x_class_host_wait_syncpt(job->syncpt->id,
 210                                         host1x_syncpt_read_max(sp)));
 211         }
 212
 213         /* Synchronize base register to allow using it for relative waiting */
 214         if (sp->base)
 215                 synchronize_syncpt_base(job);
 216
 217         syncval = host1x_syncpt_incr_max(sp, user_syncpt_incrs);
 218
 219         host1x_hw_syncpt_assign_to_channel(host, sp, ch);
 220
 221         job->syncpt_end = syncval;
 222
 223         /* add a setclass for modules that require it */
 224         if (job->class)
 225                 host1x_cdma_push(&ch->cdma,
 226                                  host1x_opcode_setclass(job->class, 0, 0),
 227                                  HOST1X_OPCODE_NOP);
 228
 229         submit_gathers(job, syncval - user_syncpt_incrs);
 230
 231         /* end CDMA submit & stash pinned hMems into sync queue */
 232         host1x_cdma_end(&ch->cdma, job);
 233
 234         trace_host1x_channel_submitted(dev_name(ch->dev), prev_max, syncval);
 235
 236         /* schedule a submit complete interrupt */
 237         err = host1x_intr_add_action(host, sp, syncval,
 238                                      HOST1X_INTR_ACTION_SUBMIT_COMPLETE, ch,
 239                                      completed_waiter, &job->waiter);
 240         completed_waiter = NULL;
 241         WARN(err, "Failed to set submit complete interrupt");
 242
 243         mutex_unlock(&ch->submitlock);
 244
 245         return 0;
 246
 247 error:
 248         kfree(completed_waiter);
 249         return err;
 250 }
 251
 252 static void enable_gather_filter(struct host1x *host,
 253                                  struct host1x_channel *ch)
 254 {
 255 #if HOST1X_HW >= 6
 256         u32 val;
 257
 258         if (!host->hv_regs)
 259                 return;
 260
 261         val = host1x_hypervisor_readl(
 262                 host, HOST1X_HV_CH_KERNEL_FILTER_GBUFFER(ch->id / 32));
 263         val |= BIT(ch->id % 32);
 264         host1x_hypervisor_writel(
 265                 host, val, HOST1X_HV_CH_KERNEL_FILTER_GBUFFER(ch->id / 32));
 266 #elif HOST1X_HW >= 4
 267         host1x_ch_writel(ch,
 268                          HOST1X_CHANNEL_CHANNELCTRL_KERNEL_FILTER_GBUFFER(1),
 269                          HOST1X_CHANNEL_CHANNELCTRL);
 270 #endif
 271 }
 272
 273 static int host1x_channel_init(struct host1x_channel *ch, struct host1x *dev,
 274                                unsigned int index)
 275 {
 276 #if HOST1X_HW < 6
 277         ch->regs = dev->regs + index * 0x4000;
 278 #else
 279         ch->regs = dev->regs + index * 0x100;
 280 #endif
 281         enable_gather_filter(dev, ch);
 282         return 0;
 283 }
 284
 285 static const struct host1x_channel_ops host1x_channel_ops = {
 286         .init = host1x_channel_init,
 287         .submit = channel_submit,
 288 };