drivers/i3c/master/svc-i3c-master.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Silvaco dual-role I3C master driver
   4  *
   5  * Copyright (C) 2020 Silvaco
   6  * Author: Miquel RAYNAL <miquel.raynal@bootlin.com>
   7  * Based on a work from: Conor Culhane <conor.culhane@silvaco.com>
   8  */
   9
  10 #include <linux/bitfield.h>
  11 #include <linux/clk.h>
  12 #include <linux/completion.h>
  13 #include <linux/errno.h>
  14 #include <linux/i3c/master.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/iopoll.h>
  17 #include <linux/list.h>
  18 #include <linux/module.h>
  19 #include <linux/of.h>
  20 #include <linux/pinctrl/consumer.h>
  21 #include <linux/platform_device.h>
  22 #include <linux/pm_runtime.h>
  23
  24 /* Master Mode Registers */
  25 #define SVC_I3C_MCONFIG      0x000
  26 #define   SVC_I3C_MCONFIG_MASTER_EN BIT(0)
  27 #define   SVC_I3C_MCONFIG_DISTO(x) FIELD_PREP(BIT(3), (x))
  28 #define   SVC_I3C_MCONFIG_HKEEP(x) FIELD_PREP(GENMASK(5, 4), (x))
  29 #define   SVC_I3C_MCONFIG_ODSTOP(x) FIELD_PREP(BIT(6), (x))
  30 #define   SVC_I3C_MCONFIG_PPBAUD(x) FIELD_PREP(GENMASK(11, 8), (x))
  31 #define   SVC_I3C_MCONFIG_PPLOW(x) FIELD_PREP(GENMASK(15, 12), (x))
  32 #define   SVC_I3C_MCONFIG_ODBAUD(x) FIELD_PREP(GENMASK(23, 16), (x))
  33 #define   SVC_I3C_MCONFIG_ODHPP(x) FIELD_PREP(BIT(24), (x))
  34 #define   SVC_I3C_MCONFIG_SKEW(x) FIELD_PREP(GENMASK(27, 25), (x))
  35 #define   SVC_I3C_MCONFIG_I2CBAUD(x) FIELD_PREP(GENMASK(31, 28), (x))
  36
  37 #define SVC_I3C_MCTRL        0x084
  38 #define   SVC_I3C_MCTRL_REQUEST_MASK GENMASK(2, 0)
  39 #define   SVC_I3C_MCTRL_REQUEST_NONE 0
  40 #define   SVC_I3C_MCTRL_REQUEST_START_ADDR 1
  41 #define   SVC_I3C_MCTRL_REQUEST_STOP 2
  42 #define   SVC_I3C_MCTRL_REQUEST_IBI_ACKNACK 3
  43 #define   SVC_I3C_MCTRL_REQUEST_PROC_DAA 4
  44 #define   SVC_I3C_MCTRL_REQUEST_AUTO_IBI 7
  45 #define   SVC_I3C_MCTRL_TYPE_I3C 0
  46 #define   SVC_I3C_MCTRL_TYPE_I2C BIT(4)
  47 #define   SVC_I3C_MCTRL_IBIRESP_AUTO 0
  48 #define   SVC_I3C_MCTRL_IBIRESP_ACK_WITHOUT_BYTE 0
  49 #define   SVC_I3C_MCTRL_IBIRESP_ACK_WITH_BYTE BIT(7)
  50 #define   SVC_I3C_MCTRL_IBIRESP_NACK BIT(6)
  51 #define   SVC_I3C_MCTRL_IBIRESP_MANUAL GENMASK(7, 6)
  52 #define   SVC_I3C_MCTRL_DIR(x) FIELD_PREP(BIT(8), (x))
  53 #define   SVC_I3C_MCTRL_DIR_WRITE 0
  54 #define   SVC_I3C_MCTRL_DIR_READ 1
  55 #define   SVC_I3C_MCTRL_ADDR(x) FIELD_PREP(GENMASK(15, 9), (x))
  56 #define   SVC_I3C_MCTRL_RDTERM(x) FIELD_PREP(GENMASK(23, 16), (x))
  57
  58 #define SVC_I3C_MSTATUS      0x088
  59 #define   SVC_I3C_MSTATUS_STATE(x) FIELD_GET(GENMASK(2, 0), (x))
  60 #define   SVC_I3C_MSTATUS_STATE_DAA(x) (SVC_I3C_MSTATUS_STATE(x) == 5)
  61 #define   SVC_I3C_MSTATUS_STATE_IDLE(x) (SVC_I3C_MSTATUS_STATE(x) == 0)
  62 #define   SVC_I3C_MSTATUS_BETWEEN(x) FIELD_GET(BIT(4), (x))
  63 #define   SVC_I3C_MSTATUS_NACKED(x) FIELD_GET(BIT(5), (x))
  64 #define   SVC_I3C_MSTATUS_IBITYPE(x) FIELD_GET(GENMASK(7, 6), (x))
  65 #define   SVC_I3C_MSTATUS_IBITYPE_IBI 1
  66 #define   SVC_I3C_MSTATUS_IBITYPE_MASTER_REQUEST 2
  67 #define   SVC_I3C_MSTATUS_IBITYPE_HOT_JOIN 3
  68 #define   SVC_I3C_MINT_SLVSTART BIT(8)
  69 #define   SVC_I3C_MINT_MCTRLDONE BIT(9)
  70 #define   SVC_I3C_MINT_COMPLETE BIT(10)
  71 #define   SVC_I3C_MINT_RXPEND BIT(11)
  72 #define   SVC_I3C_MINT_TXNOTFULL BIT(12)
  73 #define   SVC_I3C_MINT_IBIWON BIT(13)
  74 #define   SVC_I3C_MINT_ERRWARN BIT(15)
  75 #define   SVC_I3C_MSTATUS_SLVSTART(x) FIELD_GET(SVC_I3C_MINT_SLVSTART, (x))
  76 #define   SVC_I3C_MSTATUS_MCTRLDONE(x) FIELD_GET(SVC_I3C_MINT_MCTRLDONE, (x))
  77 #define   SVC_I3C_MSTATUS_COMPLETE(x) FIELD_GET(SVC_I3C_MINT_COMPLETE, (x))
  78 #define   SVC_I3C_MSTATUS_RXPEND(x) FIELD_GET(SVC_I3C_MINT_RXPEND, (x))
  79 #define   SVC_I3C_MSTATUS_TXNOTFULL(x) FIELD_GET(SVC_I3C_MINT_TXNOTFULL, (x))
  80 #define   SVC_I3C_MSTATUS_IBIWON(x) FIELD_GET(SVC_I3C_MINT_IBIWON, (x))
  81 #define   SVC_I3C_MSTATUS_ERRWARN(x) FIELD_GET(SVC_I3C_MINT_ERRWARN, (x))
  82 #define   SVC_I3C_MSTATUS_IBIADDR(x) FIELD_GET(GENMASK(30, 24), (x))
  83
  84 #define SVC_I3C_IBIRULES     0x08C
  85 #define   SVC_I3C_IBIRULES_ADDR(slot, addr) FIELD_PREP(GENMASK(29, 0), \
  86                                                        ((addr) & 0x3F) << ((slot) * 6))
  87 #define   SVC_I3C_IBIRULES_ADDRS 5
  88 #define   SVC_I3C_IBIRULES_MSB0 BIT(30)
  89 #define   SVC_I3C_IBIRULES_NOBYTE BIT(31)
  90 #define   SVC_I3C_IBIRULES_MANDBYTE 0
  91 #define SVC_I3C_MINTSET      0x090
  92 #define SVC_I3C_MINTCLR      0x094
  93 #define SVC_I3C_MINTMASKED   0x098
  94 #define SVC_I3C_MERRWARN     0x09C
  95 #define   SVC_I3C_MERRWARN_NACK BIT(2)
  96 #define SVC_I3C_MDMACTRL     0x0A0
  97 #define SVC_I3C_MDATACTRL    0x0AC
  98 #define   SVC_I3C_MDATACTRL_FLUSHTB BIT(0)
  99 #define   SVC_I3C_MDATACTRL_FLUSHRB BIT(1)
 100 #define   SVC_I3C_MDATACTRL_UNLOCK_TRIG BIT(3)
 101 #define   SVC_I3C_MDATACTRL_TXTRIG_FIFO_NOT_FULL GENMASK(5, 4)
 102 #define   SVC_I3C_MDATACTRL_RXTRIG_FIFO_NOT_EMPTY 0
 103 #define   SVC_I3C_MDATACTRL_RXCOUNT(x) FIELD_GET(GENMASK(28, 24), (x))
 104 #define   SVC_I3C_MDATACTRL_TXFULL BIT(30)
 105 #define   SVC_I3C_MDATACTRL_RXEMPTY BIT(31)
 106
 107 #define SVC_I3C_MWDATAB      0x0B0
 108 #define   SVC_I3C_MWDATAB_END BIT(8)
 109
 110 #define SVC_I3C_MWDATABE     0x0B4
 111 #define SVC_I3C_MWDATAH      0x0B8
 112 #define SVC_I3C_MWDATAHE     0x0BC
 113 #define SVC_I3C_MRDATAB      0x0C0
 114 #define SVC_I3C_MRDATAH      0x0C8
 115 #define SVC_I3C_MWMSG_SDR    0x0D0
 116 #define SVC_I3C_MRMSG_SDR    0x0D4
 117 #define SVC_I3C_MWMSG_DDR    0x0D8
 118 #define SVC_I3C_MRMSG_DDR    0x0DC
 119
 120 #define SVC_I3C_MDYNADDR     0x0E4
 121 #define   SVC_MDYNADDR_VALID BIT(0)
 122 #define   SVC_MDYNADDR_ADDR(x) FIELD_PREP(GENMASK(7, 1), (x))
 123
 124 #define SVC_I3C_MAX_DEVS 32
 125 #define SVC_I3C_PM_TIMEOUT_MS 1000
 126
 127 /* This parameter depends on the implementation and may be tuned */
 128 #define SVC_I3C_FIFO_SIZE 16
 129
 130 struct svc_i3c_cmd {
 131         u8 addr;
 132         bool rnw;
 133         u8 *in;
 134         const void *out;
 135         unsigned int len;
 136         unsigned int read_len;
 137         bool continued;
 138 };
 139
 140 struct svc_i3c_xfer {
 141         struct list_head node;
 142         struct completion comp;
 143         int ret;
 144         unsigned int type;
 145         unsigned int ncmds;
 146         struct svc_i3c_cmd cmds[];
 147 };
 148
 149 struct svc_i3c_regs_save {
 150         u32 mconfig;
 151         u32 mdynaddr;
 152 };
 153
 154 /**
 155  * struct svc_i3c_master - Silvaco I3C Master structure
 156  * @base: I3C master controller
 157  * @dev: Corresponding device
 158  * @regs: Memory mapping
 159  * @free_slots: Bit array of available slots
 160  * @addrs: Array containing the dynamic addresses of each attached device
 161  * @descs: Array of descriptors, one per attached device
 162  * @hj_work: Hot-join work
 163  * @ibi_work: IBI work
 164  * @irq: Main interrupt
 165  * @pclk: System clock
 166  * @fclk: Fast clock (bus)
 167  * @sclk: Slow clock (other events)
 168  * @xferqueue: Transfer queue structure
 169  * @xferqueue.list: List member
 170  * @xferqueue.cur: Current ongoing transfer
 171  * @xferqueue.lock: Queue lock
 172  * @ibi: IBI structure
 173  * @ibi.num_slots: Number of slots available in @ibi.slots
 174  * @ibi.slots: Available IBI slots
 175  * @ibi.tbq_slot: To be queued IBI slot
 176  * @ibi.lock: IBI lock
 177  */
 178 struct svc_i3c_master {
 179         struct i3c_master_controller base;
 180         struct device *dev;
 181         void __iomem *regs;
 182         struct svc_i3c_regs_save saved_regs;
 183         u32 free_slots;
 184         u8 addrs[SVC_I3C_MAX_DEVS];
 185         struct i3c_dev_desc *descs[SVC_I3C_MAX_DEVS];
 186         struct work_struct hj_work;
 187         struct work_struct ibi_work;
 188         int irq;
 189         struct clk *pclk;
 190         struct clk *fclk;
 191         struct clk *sclk;
 192         struct {
 193                 struct list_head list;
 194                 struct svc_i3c_xfer *cur;
 195                 /* Prevent races between transfers */
 196                 spinlock_t lock;
 197         } xferqueue;
 198         struct {
 199                 unsigned int num_slots;
 200                 struct i3c_dev_desc **slots;
 201                 struct i3c_ibi_slot *tbq_slot;
 202                 /* Prevent races within IBI handlers */
 203                 spinlock_t lock;
 204         } ibi;
 205 };
 206
 207 /**
 208  * struct svc_i3c_i2c_dev_data - Device specific data
 209  * @index: Index in the master tables corresponding to this device
 210  * @ibi: IBI slot index in the master structure
 211  * @ibi_pool: IBI pool associated to this device
 212  */
 213 struct svc_i3c_i2c_dev_data {
 214         u8 index;
 215         int ibi;
 216         struct i3c_generic_ibi_pool *ibi_pool;
 217 };
 218
 219 static bool svc_i3c_master_error(struct svc_i3c_master *master)
 220 {
 221         u32 mstatus, merrwarn;
 222
 223         mstatus = readl(master->regs + SVC_I3C_MSTATUS);
 224         if (SVC_I3C_MSTATUS_ERRWARN(mstatus)) {
 225                 merrwarn = readl(master->regs + SVC_I3C_MERRWARN);
 226                 writel(merrwarn, master->regs + SVC_I3C_MERRWARN);
 227                 dev_err(master->dev,
 228                         "Error condition: MSTATUS 0x%08x, MERRWARN 0x%08x\n",
 229                         mstatus, merrwarn);
 230
 231                 return true;
 232         }
 233
 234         return false;
 235 }
 236
 237 static void svc_i3c_master_enable_interrupts(struct svc_i3c_master *master, u32 mask)
 238 {
 239         writel(mask, master->regs + SVC_I3C_MINTSET);
 240 }
 241
 242 static void svc_i3c_master_disable_interrupts(struct svc_i3c_master *master)
 243 {
 244         u32 mask = readl(master->regs + SVC_I3C_MINTSET);
 245
 246         writel(mask, master->regs + SVC_I3C_MINTCLR);
 247 }
 248
 249 static void svc_i3c_master_clear_merrwarn(struct svc_i3c_master *master)
 250 {
 251         /* Clear pending warnings */
 252         writel(readl(master->regs + SVC_I3C_MERRWARN),
 253                master->regs + SVC_I3C_MERRWARN);
 254 }
 255
 256 static void svc_i3c_master_flush_fifo(struct svc_i3c_master *master)
 257 {
 258         /* Flush FIFOs */
 259         writel(SVC_I3C_MDATACTRL_FLUSHTB | SVC_I3C_MDATACTRL_FLUSHRB,
 260                master->regs + SVC_I3C_MDATACTRL);
 261 }
 262
 263 static void svc_i3c_master_reset_fifo_trigger(struct svc_i3c_master *master)
 264 {
 265         u32 reg;
 266
 267         /* Set RX and TX tigger levels, flush FIFOs */
 268         reg = SVC_I3C_MDATACTRL_FLUSHTB |
 269               SVC_I3C_MDATACTRL_FLUSHRB |
 270               SVC_I3C_MDATACTRL_UNLOCK_TRIG |
 271               SVC_I3C_MDATACTRL_TXTRIG_FIFO_NOT_FULL |
 272               SVC_I3C_MDATACTRL_RXTRIG_FIFO_NOT_EMPTY;
 273         writel(reg, master->regs + SVC_I3C_MDATACTRL);
 274 }
 275
 276 static void svc_i3c_master_reset(struct svc_i3c_master *master)
 277 {
 278         svc_i3c_master_clear_merrwarn(master);
 279         svc_i3c_master_reset_fifo_trigger(master);
 280         svc_i3c_master_disable_interrupts(master);
 281 }
 282
 283 static inline struct svc_i3c_master *
 284 to_svc_i3c_master(struct i3c_master_controller *master)
 285 {
 286         return container_of(master, struct svc_i3c_master, base);
 287 }
 288
 289 static void svc_i3c_master_hj_work(struct work_struct *work)
 290 {
 291         struct svc_i3c_master *master;
 292
 293         master = container_of(work, struct svc_i3c_master, hj_work);
 294         i3c_master_do_daa(&master->base);
 295 }
 296
 297 static struct i3c_dev_desc *
 298 svc_i3c_master_dev_from_addr(struct svc_i3c_master *master,
 299                              unsigned int ibiaddr)
 300 {
 301         int i;
 302
 303         for (i = 0; i < SVC_I3C_MAX_DEVS; i++)
 304                 if (master->addrs[i] == ibiaddr)
 305                         break;
 306
 307         if (i == SVC_I3C_MAX_DEVS)
 308                 return NULL;
 309
 310         return master->descs[i];
 311 }
 312
 313 static void svc_i3c_master_emit_stop(struct svc_i3c_master *master)
 314 {
 315         writel(SVC_I3C_MCTRL_REQUEST_STOP, master->regs + SVC_I3C_MCTRL);
 316
 317         /*
 318          * This delay is necessary after the emission of a stop, otherwise eg.
 319          * repeating IBIs do not get detected. There is a note in the manual
 320          * about it, stating that the stop condition might not be settled
 321          * correctly if a start condition follows too rapidly.
 322          */
 323         udelay(1);
 324 }
 325
 326 static int svc_i3c_master_handle_ibi(struct svc_i3c_master *master,
 327                                      struct i3c_dev_desc *dev)
 328 {
 329         struct svc_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev);
 330         struct i3c_ibi_slot *slot;
 331         unsigned int count;
 332         u32 mdatactrl;
 333         u8 *buf;
 334
 335         slot = i3c_generic_ibi_get_free_slot(data->ibi_pool);
 336         if (!slot)
 337                 return -ENOSPC;
 338
 339         slot->len = 0;
 340         buf = slot->data;
 341
 342         while (SVC_I3C_MSTATUS_RXPEND(readl(master->regs + SVC_I3C_MSTATUS))  &&
 343                slot->len < SVC_I3C_FIFO_SIZE) {
 344                 mdatactrl = readl(master->regs + SVC_I3C_MDATACTRL);
 345                 count = SVC_I3C_MDATACTRL_RXCOUNT(mdatactrl);
 346                 readsl(master->regs + SVC_I3C_MRDATAB, buf, count);
 347                 slot->len += count;
 348                 buf += count;
 349         }
 350
 351         master->ibi.tbq_slot = slot;
 352
 353         return 0;
 354 }
 355
 356 static void svc_i3c_master_ack_ibi(struct svc_i3c_master *master,
 357                                    bool mandatory_byte)
 358 {
 359         unsigned int ibi_ack_nack;
 360
 361         ibi_ack_nack = SVC_I3C_MCTRL_REQUEST_IBI_ACKNACK;
 362         if (mandatory_byte)
 363                 ibi_ack_nack |= SVC_I3C_MCTRL_IBIRESP_ACK_WITH_BYTE;
 364         else
 365                 ibi_ack_nack |= SVC_I3C_MCTRL_IBIRESP_ACK_WITHOUT_BYTE;
 366
 367         writel(ibi_ack_nack, master->regs + SVC_I3C_MCTRL);
 368 }
 369
 370 static void svc_i3c_master_nack_ibi(struct svc_i3c_master *master)
 371 {
 372         writel(SVC_I3C_MCTRL_REQUEST_IBI_ACKNACK |
 373                SVC_I3C_MCTRL_IBIRESP_NACK,
 374                master->regs + SVC_I3C_MCTRL);
 375 }
 376
 377 static void svc_i3c_master_ibi_work(struct work_struct *work)
 378 {
 379         struct svc_i3c_master *master = container_of(work, struct svc_i3c_master, ibi_work);
 380         struct svc_i3c_i2c_dev_data *data;
 381         unsigned int ibitype, ibiaddr;
 382         struct i3c_dev_desc *dev;
 383         u32 status, val;
 384         int ret;
 385
 386         /* Acknowledge the incoming interrupt with the AUTOIBI mechanism */
 387         writel(SVC_I3C_MCTRL_REQUEST_AUTO_IBI |
 388                SVC_I3C_MCTRL_IBIRESP_AUTO,
 389                master->regs + SVC_I3C_MCTRL);
 390
 391         /* Wait for IBIWON, should take approximately 100us */
 392         ret = readl_relaxed_poll_timeout(master->regs + SVC_I3C_MSTATUS, val,
 393                                          SVC_I3C_MSTATUS_IBIWON(val), 0, 1000);
 394         if (ret) {
 395                 dev_err(master->dev, "Timeout when polling for IBIWON\n");
 396                 goto reenable_ibis;
 397         }
 398
 399         /* Clear the interrupt status */
 400         writel(SVC_I3C_MINT_IBIWON, master->regs + SVC_I3C_MSTATUS);
 401
 402         status = readl(master->regs + SVC_I3C_MSTATUS);
 403         ibitype = SVC_I3C_MSTATUS_IBITYPE(status);
 404         ibiaddr = SVC_I3C_MSTATUS_IBIADDR(status);
 405
 406         /* Handle the critical responses to IBI's */
 407         switch (ibitype) {
 408         case SVC_I3C_MSTATUS_IBITYPE_IBI:
 409                 dev = svc_i3c_master_dev_from_addr(master, ibiaddr);
 410                 if (!dev)
 411                         svc_i3c_master_nack_ibi(master);
 412                 else
 413                         svc_i3c_master_handle_ibi(master, dev);
 414                 break;
 415         case SVC_I3C_MSTATUS_IBITYPE_HOT_JOIN:
 416                 svc_i3c_master_ack_ibi(master, false);
 417                 break;
 418         case SVC_I3C_MSTATUS_IBITYPE_MASTER_REQUEST:
 419                 svc_i3c_master_nack_ibi(master);
 420                 break;
 421         default:
 422                 break;
 423         }
 424
 425         /*
 426          * If an error happened, we probably got interrupted and the exchange
 427          * timedout. In this case we just drop everything, emit a stop and wait
 428          * for the slave to interrupt again.
 429          */
 430         if (svc_i3c_master_error(master)) {
 431                 if (master->ibi.tbq_slot) {
 432                         data = i3c_dev_get_master_data(dev);
 433                         i3c_generic_ibi_recycle_slot(data->ibi_pool,
 434                                                      master->ibi.tbq_slot);
 435                         master->ibi.tbq_slot = NULL;
 436                 }
 437
 438                 svc_i3c_master_emit_stop(master);
 439
 440                 goto reenable_ibis;
 441         }
 442
 443         /* Handle the non critical tasks */
 444         switch (ibitype) {
 445         case SVC_I3C_MSTATUS_IBITYPE_IBI:
 446                 if (dev) {
 447                         i3c_master_queue_ibi(dev, master->ibi.tbq_slot);
 448                         master->ibi.tbq_slot = NULL;
 449                 }
 450                 svc_i3c_master_emit_stop(master);
 451                 break;
 452         case SVC_I3C_MSTATUS_IBITYPE_HOT_JOIN:
 453                 queue_work(master->base.wq, &master->hj_work);
 454                 break;
 455         case SVC_I3C_MSTATUS_IBITYPE_MASTER_REQUEST:
 456         default:
 457                 break;
 458         }
 459
 460 reenable_ibis:
 461         svc_i3c_master_enable_interrupts(master, SVC_I3C_MINT_SLVSTART);
 462 }
 463
 464 static irqreturn_t svc_i3c_master_irq_handler(int irq, void *dev_id)
 465 {
 466         struct svc_i3c_master *master = (struct svc_i3c_master *)dev_id;
 467         u32 active = readl(master->regs + SVC_I3C_MINTMASKED);
 468
 469         if (!SVC_I3C_MSTATUS_SLVSTART(active))
 470                 return IRQ_NONE;
 471
 472         /* Clear the interrupt status */
 473         writel(SVC_I3C_MINT_SLVSTART, master->regs + SVC_I3C_MSTATUS);
 474
 475         svc_i3c_master_disable_interrupts(master);
 476
 477         /* Handle the interrupt in a non atomic context */
 478         queue_work(master->base.wq, &master->ibi_work);
 479
 480         return IRQ_HANDLED;
 481 }
 482
 483 static int svc_i3c_master_bus_init(struct i3c_master_controller *m)
 484 {
 485         struct svc_i3c_master *master = to_svc_i3c_master(m);
 486         struct i3c_bus *bus = i3c_master_get_bus(m);
 487         struct i3c_device_info info = {};
 488         unsigned long fclk_rate, fclk_period_ns;
 489         unsigned int high_period_ns, od_low_period_ns;
 490         u32 ppbaud, pplow, odhpp, odbaud, odstop, i2cbaud, reg;
 491         int ret;
 492
 493         ret = pm_runtime_resume_and_get(master->dev);
 494         if (ret < 0) {
 495                 dev_err(master->dev,
 496                         "<%s> cannot resume i3c bus master, err: %d\n",
 497                         __func__, ret);
 498                 return ret;
 499         }
 500
 501         /* Timings derivation */
 502         fclk_rate = clk_get_rate(master->fclk);
 503         if (!fclk_rate) {
 504                 ret = -EINVAL;
 505                 goto rpm_out;
 506         }
 507
 508         fclk_period_ns = DIV_ROUND_UP(1000000000, fclk_rate);
 509
 510         /*
 511          * Using I3C Push-Pull mode, target is 12.5MHz/80ns period.
 512          * Simplest configuration is using a 50% duty-cycle of 40ns.
 513          */
 514         ppbaud = DIV_ROUND_UP(40, fclk_period_ns) - 1;
 515         pplow = 0;
 516
 517         /*
 518          * Using I3C Open-Drain mode, target is 4.17MHz/240ns with a
 519          * duty-cycle tuned so that high levels are filetered out by
 520          * the 50ns filter (target being 40ns).
 521          */
 522         odhpp = 1;
 523         high_period_ns = (ppbaud + 1) * fclk_period_ns;
 524         odbaud = DIV_ROUND_UP(240 - high_period_ns, high_period_ns) - 1;
 525         od_low_period_ns = (odbaud + 1) * high_period_ns;
 526
 527         switch (bus->mode) {
 528         case I3C_BUS_MODE_PURE:
 529                 i2cbaud = 0;
 530                 odstop = 0;
 531                 break;
 532         case I3C_BUS_MODE_MIXED_FAST:
 533         case I3C_BUS_MODE_MIXED_LIMITED:
 534                 /*
 535                  * Using I2C Fm+ mode, target is 1MHz/1000ns, the difference
 536                  * between the high and low period does not really matter.
 537                  */
 538                 i2cbaud = DIV_ROUND_UP(1000, od_low_period_ns) - 2;
 539                 odstop = 1;
 540                 break;
 541         case I3C_BUS_MODE_MIXED_SLOW:
 542                 /*
 543                  * Using I2C Fm mode, target is 0.4MHz/2500ns, with the same
 544                  * constraints as the FM+ mode.
 545                  */
 546                 i2cbaud = DIV_ROUND_UP(2500, od_low_period_ns) - 2;
 547                 odstop = 1;
 548                 break;
 549         default:
 550                 goto rpm_out;
 551         }
 552
 553         reg = SVC_I3C_MCONFIG_MASTER_EN |
 554               SVC_I3C_MCONFIG_DISTO(0) |
 555               SVC_I3C_MCONFIG_HKEEP(0) |
 556               SVC_I3C_MCONFIG_ODSTOP(odstop) |
 557               SVC_I3C_MCONFIG_PPBAUD(ppbaud) |
 558               SVC_I3C_MCONFIG_PPLOW(pplow) |
 559               SVC_I3C_MCONFIG_ODBAUD(odbaud) |
 560               SVC_I3C_MCONFIG_ODHPP(odhpp) |
 561               SVC_I3C_MCONFIG_SKEW(0) |
 562               SVC_I3C_MCONFIG_I2CBAUD(i2cbaud);
 563         writel(reg, master->regs + SVC_I3C_MCONFIG);
 564
 565         /* Master core's registration */
 566         ret = i3c_master_get_free_addr(m, 0);
 567         if (ret < 0)
 568                 goto rpm_out;
 569
 570         info.dyn_addr = ret;
 571
 572         writel(SVC_MDYNADDR_VALID | SVC_MDYNADDR_ADDR(info.dyn_addr),
 573                master->regs + SVC_I3C_MDYNADDR);
 574
 575         ret = i3c_master_set_info(&master->base, &info);
 576         if (ret)
 577                 goto rpm_out;
 578
 579 rpm_out:
 580         pm_runtime_mark_last_busy(master->dev);
 581         pm_runtime_put_autosuspend(master->dev);
 582
 583         return ret;
 584 }
 585
 586 static void svc_i3c_master_bus_cleanup(struct i3c_master_controller *m)
 587 {
 588         struct svc_i3c_master *master = to_svc_i3c_master(m);
 589         int ret;
 590
 591         ret = pm_runtime_resume_and_get(master->dev);
 592         if (ret < 0) {
 593                 dev_err(master->dev, "<%s> Cannot get runtime PM.\n", __func__);
 594                 return;
 595         }
 596
 597         svc_i3c_master_disable_interrupts(master);
 598
 599         /* Disable master */
 600         writel(0, master->regs + SVC_I3C_MCONFIG);
 601
 602         pm_runtime_mark_last_busy(master->dev);
 603         pm_runtime_put_autosuspend(master->dev);
 604 }
 605
 606 static int svc_i3c_master_reserve_slot(struct svc_i3c_master *master)
 607 {
 608         unsigned int slot;
 609
 610         if (!(master->free_slots & GENMASK(SVC_I3C_MAX_DEVS - 1, 0)))
 611                 return -ENOSPC;
 612
 613         slot = ffs(master->free_slots) - 1;
 614
 615         master->free_slots &= ~BIT(slot);
 616
 617         return slot;
 618 }
 619
 620 static void svc_i3c_master_release_slot(struct svc_i3c_master *master,
 621                                         unsigned int slot)
 622 {
 623         master->free_slots |= BIT(slot);
 624 }
 625
 626 static int svc_i3c_master_attach_i3c_dev(struct i3c_dev_desc *dev)
 627 {
 628         struct i3c_master_controller *m = i3c_dev_get_master(dev);
 629         struct svc_i3c_master *master = to_svc_i3c_master(m);
 630         struct svc_i3c_i2c_dev_data *data;
 631         int slot;
 632
 633         slot = svc_i3c_master_reserve_slot(master);
 634         if (slot < 0)
 635                 return slot;
 636
 637         data = kzalloc(sizeof(*data), GFP_KERNEL);
 638         if (!data) {
 639                 svc_i3c_master_release_slot(master, slot);
 640                 return -ENOMEM;
 641         }
 642
 643         data->ibi = -1;
 644         data->index = slot;
 645         master->addrs[slot] = dev->info.dyn_addr ? dev->info.dyn_addr :
 646                                                    dev->info.static_addr;
 647         master->descs[slot] = dev;
 648
 649         i3c_dev_set_master_data(dev, data);
 650
 651         return 0;
 652 }
 653
 654 static int svc_i3c_master_reattach_i3c_dev(struct i3c_dev_desc *dev,
 655                                            u8 old_dyn_addr)
 656 {
 657         struct i3c_master_controller *m = i3c_dev_get_master(dev);
 658         struct svc_i3c_master *master = to_svc_i3c_master(m);
 659         struct svc_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev);
 660
 661         master->addrs[data->index] = dev->info.dyn_addr ? dev->info.dyn_addr :
 662                                                           dev->info.static_addr;
 663
 664         return 0;
 665 }
 666
 667 static void svc_i3c_master_detach_i3c_dev(struct i3c_dev_desc *dev)
 668 {
 669         struct svc_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev);
 670         struct i3c_master_controller *m = i3c_dev_get_master(dev);
 671         struct svc_i3c_master *master = to_svc_i3c_master(m);
 672
 673         master->addrs[data->index] = 0;
 674         svc_i3c_master_release_slot(master, data->index);
 675
 676         kfree(data);
 677 }
 678
 679 static int svc_i3c_master_attach_i2c_dev(struct i2c_dev_desc *dev)
 680 {
 681         struct i3c_master_controller *m = i2c_dev_get_master(dev);
 682         struct svc_i3c_master *master = to_svc_i3c_master(m);
 683         struct svc_i3c_i2c_dev_data *data;
 684         int slot;
 685
 686         slot = svc_i3c_master_reserve_slot(master);
 687         if (slot < 0)
 688                 return slot;
 689
 690         data = kzalloc(sizeof(*data), GFP_KERNEL);
 691         if (!data) {
 692                 svc_i3c_master_release_slot(master, slot);
 693                 return -ENOMEM;
 694         }
 695
 696         data->index = slot;
 697         master->addrs[slot] = dev->addr;
 698
 699         i2c_dev_set_master_data(dev, data);
 700
 701         return 0;
 702 }
 703
 704 static void svc_i3c_master_detach_i2c_dev(struct i2c_dev_desc *dev)
 705 {
 706         struct svc_i3c_i2c_dev_data *data = i2c_dev_get_master_data(dev);
 707         struct i3c_master_controller *m = i2c_dev_get_master(dev);
 708         struct svc_i3c_master *master = to_svc_i3c_master(m);
 709
 710         svc_i3c_master_release_slot(master, data->index);
 711
 712         kfree(data);
 713 }
 714
 715 static int svc_i3c_master_readb(struct svc_i3c_master *master, u8 *dst,
 716                                 unsigned int len)
 717 {
 718         int ret, i;
 719         u32 reg;
 720
 721         for (i = 0; i < len; i++) {
 722                 ret = readl_poll_timeout_atomic(master->regs + SVC_I3C_MSTATUS,
 723                                                 reg,
 724                                                 SVC_I3C_MSTATUS_RXPEND(reg),
 725                                                 0, 1000);
 726                 if (ret)
 727                         return ret;
 728
 729                 dst[i] = readl(master->regs + SVC_I3C_MRDATAB);
 730         }
 731
 732         return 0;
 733 }
 734
 735 static int svc_i3c_master_do_daa_locked(struct svc_i3c_master *master,
 736                                         u8 *addrs, unsigned int *count)
 737 {
 738         u64 prov_id[SVC_I3C_MAX_DEVS] = {}, nacking_prov_id = 0;
 739         unsigned int dev_nb = 0, last_addr = 0;
 740         u32 reg;
 741         int ret, i;
 742
 743         while (true) {
 744                 /* Enter/proceed with DAA */
 745                 writel(SVC_I3C_MCTRL_REQUEST_PROC_DAA |
 746                        SVC_I3C_MCTRL_TYPE_I3C |
 747                        SVC_I3C_MCTRL_IBIRESP_NACK |
 748                        SVC_I3C_MCTRL_DIR(SVC_I3C_MCTRL_DIR_WRITE),
 749                        master->regs + SVC_I3C_MCTRL);
 750
 751                 /*
 752                  * Either one slave will send its ID, or the assignment process
 753                  * is done.
 754                  */
 755                 ret = readl_poll_timeout_atomic(master->regs + SVC_I3C_MSTATUS,
 756                                                 reg,
 757                                                 SVC_I3C_MSTATUS_RXPEND(reg) |
 758                                                 SVC_I3C_MSTATUS_MCTRLDONE(reg),
 759                                                 1, 1000);
 760                 if (ret)
 761                         return ret;
 762
 763                 if (SVC_I3C_MSTATUS_RXPEND(reg)) {
 764                         u8 data[6];
 765
 766                         /*
 767                          * We only care about the 48-bit provisional ID yet to
 768                          * be sure a device does not nack an address twice.
 769                          * Otherwise, we would just need to flush the RX FIFO.
 770                          */
 771                         ret = svc_i3c_master_readb(master, data, 6);
 772                         if (ret)
 773                                 return ret;
 774
 775                         for (i = 0; i < 6; i++)
 776                                 prov_id[dev_nb] |= (u64)(data[i]) << (8 * (5 - i));
 777
 778                         /* We do not care about the BCR and DCR yet */
 779                         ret = svc_i3c_master_readb(master, data, 2);
 780                         if (ret)
 781                                 return ret;
 782                 } else if (SVC_I3C_MSTATUS_MCTRLDONE(reg)) {
 783                         if (SVC_I3C_MSTATUS_STATE_IDLE(reg) &&
 784                             SVC_I3C_MSTATUS_COMPLETE(reg)) {
 785                                 /*
 786                                  * All devices received and acked they dynamic
 787                                  * address, this is the natural end of the DAA
 788                                  * procedure.
 789                                  */
 790                                 break;
 791                         } else if (SVC_I3C_MSTATUS_NACKED(reg)) {
 792                                 /*
 793                                  * A slave device nacked the address, this is
 794                                  * allowed only once, DAA will be stopped and
 795                                  * then resumed. The same device is supposed to
 796                                  * answer again immediately and shall ack the
 797                                  * address this time.
 798                                  */
 799                                 if (prov_id[dev_nb] == nacking_prov_id)
 800                                         return -EIO;
 801
 802                                 dev_nb--;
 803                                 nacking_prov_id = prov_id[dev_nb];
 804                                 svc_i3c_master_emit_stop(master);
 805
 806                                 continue;
 807                         } else {
 808                                 return -EIO;
 809                         }
 810                 }
 811
 812                 /* Wait for the slave to be ready to receive its address */
 813                 ret = readl_poll_timeout_atomic(master->regs + SVC_I3C_MSTATUS,
 814                                                 reg,
 815                                                 SVC_I3C_MSTATUS_MCTRLDONE(reg) &&
 816                                                 SVC_I3C_MSTATUS_STATE_DAA(reg) &&
 817                                                 SVC_I3C_MSTATUS_BETWEEN(reg),
 818                                                 0, 1000);
 819                 if (ret)
 820                         return ret;
 821
 822                 /* Give the slave device a suitable dynamic address */
 823                 ret = i3c_master_get_free_addr(&master->base, last_addr + 1);
 824                 if (ret < 0)
 825                         return ret;
 826
 827                 addrs[dev_nb] = ret;
 828                 dev_dbg(master->dev, "DAA: device %d assigned to 0x%02x\n",
 829                         dev_nb, addrs[dev_nb]);
 830
 831                 writel(addrs[dev_nb], master->regs + SVC_I3C_MWDATAB);
 832                 last_addr = addrs[dev_nb++];
 833         }
 834
 835         *count = dev_nb;
 836
 837         return 0;
 838 }
 839
 840 static int svc_i3c_update_ibirules(struct svc_i3c_master *master)
 841 {
 842         struct i3c_dev_desc *dev;
 843         u32 reg_mbyte = 0, reg_nobyte = SVC_I3C_IBIRULES_NOBYTE;
 844         unsigned int mbyte_addr_ok = 0, mbyte_addr_ko = 0, nobyte_addr_ok = 0,
 845                 nobyte_addr_ko = 0;
 846         bool list_mbyte = false, list_nobyte = false;
 847
 848         /* Create the IBIRULES register for both cases */
 849         i3c_bus_for_each_i3cdev(&master->base.bus, dev) {
 850                 if (I3C_BCR_DEVICE_ROLE(dev->info.bcr) == I3C_BCR_I3C_MASTER)
 851                         continue;
 852
 853                 if (dev->info.bcr & I3C_BCR_IBI_PAYLOAD) {
 854                         reg_mbyte |= SVC_I3C_IBIRULES_ADDR(mbyte_addr_ok,
 855                                                            dev->info.dyn_addr);
 856
 857                         /* IBI rules cannot be applied to devices with MSb=1 */
 858                         if (dev->info.dyn_addr & BIT(7))
 859                                 mbyte_addr_ko++;
 860                         else
 861                                 mbyte_addr_ok++;
 862                 } else {
 863                         reg_nobyte |= SVC_I3C_IBIRULES_ADDR(nobyte_addr_ok,
 864                                                             dev->info.dyn_addr);
 865
 866                         /* IBI rules cannot be applied to devices with MSb=1 */
 867                         if (dev->info.dyn_addr & BIT(7))
 868                                 nobyte_addr_ko++;
 869                         else
 870                                 nobyte_addr_ok++;
 871                 }
 872         }
 873
 874         /* Device list cannot be handled by hardware */
 875         if (!mbyte_addr_ko && mbyte_addr_ok <= SVC_I3C_IBIRULES_ADDRS)
 876                 list_mbyte = true;
 877
 878         if (!nobyte_addr_ko && nobyte_addr_ok <= SVC_I3C_IBIRULES_ADDRS)
 879                 list_nobyte = true;
 880
 881         /* No list can be properly handled, return an error */
 882         if (!list_mbyte && !list_nobyte)
 883                 return -ERANGE;
 884
 885         /* Pick the first list that can be handled by hardware, randomly */
 886         if (list_mbyte)
 887                 writel(reg_mbyte, master->regs + SVC_I3C_IBIRULES);
 888         else
 889                 writel(reg_nobyte, master->regs + SVC_I3C_IBIRULES);
 890
 891         return 0;
 892 }
 893
 894 static int svc_i3c_master_do_daa(struct i3c_master_controller *m)
 895 {
 896         struct svc_i3c_master *master = to_svc_i3c_master(m);
 897         u8 addrs[SVC_I3C_MAX_DEVS];
 898         unsigned long flags;
 899         unsigned int dev_nb;
 900         int ret, i;
 901
 902         ret = pm_runtime_resume_and_get(master->dev);
 903         if (ret < 0) {
 904                 dev_err(master->dev, "<%s> Cannot get runtime PM.\n", __func__);
 905                 return ret;
 906         }
 907
 908         spin_lock_irqsave(&master->xferqueue.lock, flags);
 909         ret = svc_i3c_master_do_daa_locked(master, addrs, &dev_nb);
 910         spin_unlock_irqrestore(&master->xferqueue.lock, flags);
 911         if (ret) {
 912                 svc_i3c_master_emit_stop(master);
 913                 svc_i3c_master_clear_merrwarn(master);
 914                 goto rpm_out;
 915         }
 916
 917         /* Register all devices who participated to the core */
 918         for (i = 0; i < dev_nb; i++) {
 919                 ret = i3c_master_add_i3c_dev_locked(m, addrs[i]);
 920                 if (ret)
 921                         goto rpm_out;
 922         }
 923
 924         /* Configure IBI auto-rules */
 925         ret = svc_i3c_update_ibirules(master);
 926         if (ret)
 927                 dev_err(master->dev, "Cannot handle such a list of devices");
 928
 929 rpm_out:
 930         pm_runtime_mark_last_busy(master->dev);
 931         pm_runtime_put_autosuspend(master->dev);
 932
 933         return ret;
 934 }
 935
 936 static int svc_i3c_master_read(struct svc_i3c_master *master,
 937                                u8 *in, unsigned int len)
 938 {
 939         int offset = 0, i;
 940         u32 mdctrl, mstatus;
 941         bool completed = false;
 942         unsigned int count;
 943         unsigned long start = jiffies;
 944
 945         while (!completed) {
 946                 mstatus = readl(master->regs + SVC_I3C_MSTATUS);
 947                 if (SVC_I3C_MSTATUS_COMPLETE(mstatus) != 0)
 948                         completed = true;
 949
 950                 if (time_after(jiffies, start + msecs_to_jiffies(1000))) {
 951                         dev_dbg(master->dev, "I3C read timeout\n");
 952                         return -ETIMEDOUT;
 953                 }
 954
 955                 mdctrl = readl(master->regs + SVC_I3C_MDATACTRL);
 956                 count = SVC_I3C_MDATACTRL_RXCOUNT(mdctrl);
 957                 if (offset + count > len) {
 958                         dev_err(master->dev, "I3C receive length too long!\n");
 959                         return -EINVAL;
 960                 }
 961                 for (i = 0; i < count; i++)
 962                         in[offset + i] = readl(master->regs + SVC_I3C_MRDATAB);
 963
 964                 offset += count;
 965         }
 966
 967         return offset;
 968 }
 969
 970 static int svc_i3c_master_write(struct svc_i3c_master *master,
 971                                 const u8 *out, unsigned int len)
 972 {
 973         int offset = 0, ret;
 974         u32 mdctrl;
 975
 976         while (offset < len) {
 977                 ret = readl_poll_timeout(master->regs + SVC_I3C_MDATACTRL,
 978                                          mdctrl,
 979                                          !(mdctrl & SVC_I3C_MDATACTRL_TXFULL),
 980                                          0, 1000);
 981                 if (ret)
 982                         return ret;
 983
 984                 /*
 985                  * The last byte to be sent over the bus must either have the
 986                  * "end" bit set or be written in MWDATABE.
 987                  */
 988                 if (likely(offset < (len - 1)))
 989                         writel(out[offset++], master->regs + SVC_I3C_MWDATAB);
 990                 else
 991                         writel(out[offset++], master->regs + SVC_I3C_MWDATABE);
 992         }
 993
 994         return 0;
 995 }
 996
 997 static int svc_i3c_master_xfer(struct svc_i3c_master *master,
 998                                bool rnw, unsigned int xfer_type, u8 addr,
 999                                u8 *in, const u8 *out, unsigned int xfer_len,
1000                                unsigned int *read_len, bool continued)
1001 {
1002         u32 reg;
1003         int ret;
1004
1005         writel(SVC_I3C_MCTRL_REQUEST_START_ADDR |
1006                xfer_type |
1007                SVC_I3C_MCTRL_IBIRESP_NACK |
1008                SVC_I3C_MCTRL_DIR(rnw) |
1009                SVC_I3C_MCTRL_ADDR(addr) |
1010                SVC_I3C_MCTRL_RDTERM(*read_len),
1011                master->regs + SVC_I3C_MCTRL);
1012
1013         ret = readl_poll_timeout(master->regs + SVC_I3C_MSTATUS, reg,
1014                                  SVC_I3C_MSTATUS_MCTRLDONE(reg), 0, 1000);
1015         if (ret)
1016                 goto emit_stop;
1017
1018         if (readl(master->regs + SVC_I3C_MERRWARN) & SVC_I3C_MERRWARN_NACK) {
1019                 ret = -ENXIO;
1020                 goto emit_stop;
1021         }
1022
1023         if (rnw)
1024                 ret = svc_i3c_master_read(master, in, xfer_len);
1025         else
1026                 ret = svc_i3c_master_write(master, out, xfer_len);
1027         if (ret < 0)
1028                 goto emit_stop;
1029
1030         if (rnw)
1031                 *read_len = ret;
1032
1033         ret = readl_poll_timeout(master->regs + SVC_I3C_MSTATUS, reg,
1034                                  SVC_I3C_MSTATUS_COMPLETE(reg), 0, 1000);
1035         if (ret)
1036                 goto emit_stop;
1037
1038         writel(SVC_I3C_MINT_COMPLETE, master->regs + SVC_I3C_MSTATUS);
1039
1040         if (!continued) {
1041                 svc_i3c_master_emit_stop(master);
1042
1043                 /* Wait idle if stop is sent. */
1044                 readl_poll_timeout(master->regs + SVC_I3C_MSTATUS, reg,
1045                                    SVC_I3C_MSTATUS_STATE_IDLE(reg), 0, 1000);
1046         }
1047
1048         return 0;
1049
1050 emit_stop:
1051         svc_i3c_master_emit_stop(master);
1052         svc_i3c_master_clear_merrwarn(master);
1053
1054         return ret;
1055 }
1056
1057 static struct svc_i3c_xfer *
1058 svc_i3c_master_alloc_xfer(struct svc_i3c_master *master, unsigned int ncmds)
1059 {
1060         struct svc_i3c_xfer *xfer;
1061
1062         xfer = kzalloc(struct_size(xfer, cmds, ncmds), GFP_KERNEL);
1063         if (!xfer)
1064                 return NULL;
1065
1066         INIT_LIST_HEAD(&xfer->node);
1067         xfer->ncmds = ncmds;
1068         xfer->ret = -ETIMEDOUT;
1069
1070         return xfer;
1071 }
1072
1073 static void svc_i3c_master_free_xfer(struct svc_i3c_xfer *xfer)
1074 {
1075         kfree(xfer);
1076 }
1077
1078 static void svc_i3c_master_dequeue_xfer_locked(struct svc_i3c_master *master,
1079                                                struct svc_i3c_xfer *xfer)
1080 {
1081         if (master->xferqueue.cur == xfer)
1082                 master->xferqueue.cur = NULL;
1083         else
1084                 list_del_init(&xfer->node);
1085 }
1086
1087 static void svc_i3c_master_dequeue_xfer(struct svc_i3c_master *master,
1088                                         struct svc_i3c_xfer *xfer)
1089 {
1090         unsigned long flags;
1091
1092         spin_lock_irqsave(&master->xferqueue.lock, flags);
1093         svc_i3c_master_dequeue_xfer_locked(master, xfer);
1094         spin_unlock_irqrestore(&master->xferqueue.lock, flags);
1095 }
1096
1097 static void svc_i3c_master_start_xfer_locked(struct svc_i3c_master *master)
1098 {
1099         struct svc_i3c_xfer *xfer = master->xferqueue.cur;
1100         int ret, i;
1101
1102         if (!xfer)
1103                 return;
1104
1105         svc_i3c_master_clear_merrwarn(master);
1106         svc_i3c_master_flush_fifo(master);
1107
1108         for (i = 0; i < xfer->ncmds; i++) {
1109                 struct svc_i3c_cmd *cmd = &xfer->cmds[i];
1110
1111                 ret = svc_i3c_master_xfer(master, cmd->rnw, xfer->type,
1112                                           cmd->addr, cmd->in, cmd->out,
1113                                           cmd->len, &cmd->read_len,
1114                                           cmd->continued);
1115                 if (ret)
1116                         break;
1117         }
1118
1119         xfer->ret = ret;
1120         complete(&xfer->comp);
1121
1122         if (ret < 0)
1123                 svc_i3c_master_dequeue_xfer_locked(master, xfer);
1124
1125         xfer = list_first_entry_or_null(&master->xferqueue.list,
1126                                         struct svc_i3c_xfer,
1127                                         node);
1128         if (xfer)
1129                 list_del_init(&xfer->node);
1130
1131         master->xferqueue.cur = xfer;
1132         svc_i3c_master_start_xfer_locked(master);
1133 }
1134
1135 static void svc_i3c_master_enqueue_xfer(struct svc_i3c_master *master,
1136                                         struct svc_i3c_xfer *xfer)
1137 {
1138         unsigned long flags;
1139         int ret;
1140
1141         ret = pm_runtime_resume_and_get(master->dev);
1142         if (ret < 0) {
1143                 dev_err(master->dev, "<%s> Cannot get runtime PM.\n", __func__);
1144                 return;
1145         }
1146
1147         init_completion(&xfer->comp);
1148         spin_lock_irqsave(&master->xferqueue.lock, flags);
1149         if (master->xferqueue.cur) {
1150                 list_add_tail(&xfer->node, &master->xferqueue.list);
1151         } else {
1152                 master->xferqueue.cur = xfer;
1153                 svc_i3c_master_start_xfer_locked(master);
1154         }
1155         spin_unlock_irqrestore(&master->xferqueue.lock, flags);
1156
1157         pm_runtime_mark_last_busy(master->dev);
1158         pm_runtime_put_autosuspend(master->dev);
1159 }
1160
1161 static bool
1162 svc_i3c_master_supports_ccc_cmd(struct i3c_master_controller *master,
1163                                 const struct i3c_ccc_cmd *cmd)
1164 {
1165         /* No software support for CCC commands targeting more than one slave */
1166         return (cmd->ndests == 1);
1167 }
1168
1169 static int svc_i3c_master_send_bdcast_ccc_cmd(struct svc_i3c_master *master,
1170                                               struct i3c_ccc_cmd *ccc)
1171 {
1172         unsigned int xfer_len = ccc->dests[0].payload.len + 1;
1173         struct svc_i3c_xfer *xfer;
1174         struct svc_i3c_cmd *cmd;
1175         u8 *buf;
1176         int ret;
1177
1178         xfer = svc_i3c_master_alloc_xfer(master, 1);
1179         if (!xfer)
1180                 return -ENOMEM;
1181
1182         buf = kmalloc(xfer_len, GFP_KERNEL);
1183         if (!buf) {
1184                 svc_i3c_master_free_xfer(xfer);
1185                 return -ENOMEM;
1186         }
1187
1188         buf[0] = ccc->id;
1189         memcpy(&buf[1], ccc->dests[0].payload.data, ccc->dests[0].payload.len);
1190
1191         xfer->type = SVC_I3C_MCTRL_TYPE_I3C;
1192
1193         cmd = &xfer->cmds[0];
1194         cmd->addr = ccc->dests[0].addr;
1195         cmd->rnw = ccc->rnw;
1196         cmd->in = NULL;
1197         cmd->out = buf;
1198         cmd->len = xfer_len;
1199         cmd->read_len = 0;
1200         cmd->continued = false;
1201
1202         svc_i3c_master_enqueue_xfer(master, xfer);
1203         if (!wait_for_completion_timeout(&xfer->comp, msecs_to_jiffies(1000)))
1204                 svc_i3c_master_dequeue_xfer(master, xfer);
1205
1206         ret = xfer->ret;
1207         kfree(buf);
1208         svc_i3c_master_free_xfer(xfer);
1209
1210         return ret;
1211 }
1212
1213 static int svc_i3c_master_send_direct_ccc_cmd(struct svc_i3c_master *master,
1214                                               struct i3c_ccc_cmd *ccc)
1215 {
1216         unsigned int xfer_len = ccc->dests[0].payload.len;
1217         unsigned int read_len = ccc->rnw ? xfer_len : 0;
1218         struct svc_i3c_xfer *xfer;
1219         struct svc_i3c_cmd *cmd;
1220         int ret;
1221
1222         xfer = svc_i3c_master_alloc_xfer(master, 2);
1223         if (!xfer)
1224                 return -ENOMEM;
1225
1226         xfer->type = SVC_I3C_MCTRL_TYPE_I3C;
1227
1228         /* Broadcasted message */
1229         cmd = &xfer->cmds[0];
1230         cmd->addr = I3C_BROADCAST_ADDR;
1231         cmd->rnw = 0;
1232         cmd->in = NULL;
1233         cmd->out = &ccc->id;
1234         cmd->len = 1;
1235         cmd->read_len = 0;
1236         cmd->continued = true;
1237
1238         /* Directed message */
1239         cmd = &xfer->cmds[1];
1240         cmd->addr = ccc->dests[0].addr;
1241         cmd->rnw = ccc->rnw;
1242         cmd->in = ccc->rnw ? ccc->dests[0].payload.data : NULL;
1243         cmd->out = ccc->rnw ? NULL : ccc->dests[0].payload.data,
1244         cmd->len = xfer_len;
1245         cmd->read_len = read_len;
1246         cmd->continued = false;
1247
1248         svc_i3c_master_enqueue_xfer(master, xfer);
1249         if (!wait_for_completion_timeout(&xfer->comp, msecs_to_jiffies(1000)))
1250                 svc_i3c_master_dequeue_xfer(master, xfer);
1251
1252         if (cmd->read_len != xfer_len)
1253                 ccc->dests[0].payload.len = cmd->read_len;
1254
1255         ret = xfer->ret;
1256         svc_i3c_master_free_xfer(xfer);
1257
1258         return ret;
1259 }
1260
1261 static int svc_i3c_master_send_ccc_cmd(struct i3c_master_controller *m,
1262                                        struct i3c_ccc_cmd *cmd)
1263 {
1264         struct svc_i3c_master *master = to_svc_i3c_master(m);
1265         bool broadcast = cmd->id < 0x80;
1266
1267         if (broadcast)
1268                 return svc_i3c_master_send_bdcast_ccc_cmd(master, cmd);
1269         else
1270                 return svc_i3c_master_send_direct_ccc_cmd(master, cmd);
1271 }
1272
1273 static int svc_i3c_master_priv_xfers(struct i3c_dev_desc *dev,
1274                                      struct i3c_priv_xfer *xfers,
1275                                      int nxfers)
1276 {
1277         struct i3c_master_controller *m = i3c_dev_get_master(dev);
1278         struct svc_i3c_master *master = to_svc_i3c_master(m);
1279         struct svc_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev);
1280         struct svc_i3c_xfer *xfer;
1281         int ret, i;
1282
1283         xfer = svc_i3c_master_alloc_xfer(master, nxfers);
1284         if (!xfer)
1285                 return -ENOMEM;
1286
1287         xfer->type = SVC_I3C_MCTRL_TYPE_I3C;
1288
1289         for (i = 0; i < nxfers; i++) {
1290                 struct svc_i3c_cmd *cmd = &xfer->cmds[i];
1291
1292                 cmd->addr = master->addrs[data->index];
1293                 cmd->rnw = xfers[i].rnw;
1294                 cmd->in = xfers[i].rnw ? xfers[i].data.in : NULL;
1295                 cmd->out = xfers[i].rnw ? NULL : xfers[i].data.out;
1296                 cmd->len = xfers[i].len;
1297                 cmd->read_len = xfers[i].rnw ? xfers[i].len : 0;
1298                 cmd->continued = (i + 1) < nxfers;
1299         }
1300
1301         svc_i3c_master_enqueue_xfer(master, xfer);
1302         if (!wait_for_completion_timeout(&xfer->comp, msecs_to_jiffies(1000)))
1303                 svc_i3c_master_dequeue_xfer(master, xfer);
1304
1305         ret = xfer->ret;
1306         svc_i3c_master_free_xfer(xfer);
1307
1308         return ret;
1309 }
1310
1311 static int svc_i3c_master_i2c_xfers(struct i2c_dev_desc *dev,
1312                                     const struct i2c_msg *xfers,
1313                                     int nxfers)
1314 {
1315         struct i3c_master_controller *m = i2c_dev_get_master(dev);
1316         struct svc_i3c_master *master = to_svc_i3c_master(m);
1317         struct svc_i3c_i2c_dev_data *data = i2c_dev_get_master_data(dev);
1318         struct svc_i3c_xfer *xfer;
1319         int ret, i;
1320
1321         xfer = svc_i3c_master_alloc_xfer(master, nxfers);
1322         if (!xfer)
1323                 return -ENOMEM;
1324
1325         xfer->type = SVC_I3C_MCTRL_TYPE_I2C;
1326
1327         for (i = 0; i < nxfers; i++) {
1328                 struct svc_i3c_cmd *cmd = &xfer->cmds[i];
1329
1330                 cmd->addr = master->addrs[data->index];
1331                 cmd->rnw = xfers[i].flags & I2C_M_RD;
1332                 cmd->in = cmd->rnw ? xfers[i].buf : NULL;
1333                 cmd->out = cmd->rnw ? NULL : xfers[i].buf;
1334                 cmd->len = xfers[i].len;
1335                 cmd->read_len = cmd->rnw ? xfers[i].len : 0;
1336                 cmd->continued = (i + 1 < nxfers);
1337         }
1338
1339         svc_i3c_master_enqueue_xfer(master, xfer);
1340         if (!wait_for_completion_timeout(&xfer->comp, msecs_to_jiffies(1000)))
1341                 svc_i3c_master_dequeue_xfer(master, xfer);
1342
1343         ret = xfer->ret;
1344         svc_i3c_master_free_xfer(xfer);
1345
1346         return ret;
1347 }
1348
1349 static int svc_i3c_master_request_ibi(struct i3c_dev_desc *dev,
1350                                       const struct i3c_ibi_setup *req)
1351 {
1352         struct i3c_master_controller *m = i3c_dev_get_master(dev);
1353         struct svc_i3c_master *master = to_svc_i3c_master(m);
1354         struct svc_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev);
1355         unsigned long flags;
1356         unsigned int i;
1357
1358         if (dev->ibi->max_payload_len > SVC_I3C_FIFO_SIZE) {
1359                 dev_err(master->dev, "IBI max payload %d should be < %d\n",
1360                         dev->ibi->max_payload_len, SVC_I3C_FIFO_SIZE);
1361                 return -ERANGE;
1362         }
1363
1364         data->ibi_pool = i3c_generic_ibi_alloc_pool(dev, req);
1365         if (IS_ERR(data->ibi_pool))
1366                 return PTR_ERR(data->ibi_pool);
1367
1368         spin_lock_irqsave(&master->ibi.lock, flags);
1369         for (i = 0; i < master->ibi.num_slots; i++) {
1370                 if (!master->ibi.slots[i]) {
1371                         data->ibi = i;
1372                         master->ibi.slots[i] = dev;
1373                         break;
1374                 }
1375         }
1376         spin_unlock_irqrestore(&master->ibi.lock, flags);
1377
1378         if (i < master->ibi.num_slots)
1379                 return 0;
1380
1381         i3c_generic_ibi_free_pool(data->ibi_pool);
1382         data->ibi_pool = NULL;
1383
1384         return -ENOSPC;
1385 }
1386
1387 static void svc_i3c_master_free_ibi(struct i3c_dev_desc *dev)
1388 {
1389         struct i3c_master_controller *m = i3c_dev_get_master(dev);
1390         struct svc_i3c_master *master = to_svc_i3c_master(m);
1391         struct svc_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev);
1392         unsigned long flags;
1393
1394         spin_lock_irqsave(&master->ibi.lock, flags);
1395         master->ibi.slots[data->ibi] = NULL;
1396         data->ibi = -1;
1397         spin_unlock_irqrestore(&master->ibi.lock, flags);
1398
1399         i3c_generic_ibi_free_pool(data->ibi_pool);
1400 }
1401
1402 static int svc_i3c_master_enable_ibi(struct i3c_dev_desc *dev)
1403 {
1404         struct i3c_master_controller *m = i3c_dev_get_master(dev);
1405         struct svc_i3c_master *master = to_svc_i3c_master(m);
1406         int ret;
1407
1408         ret = pm_runtime_resume_and_get(master->dev);
1409         if (ret < 0) {
1410                 dev_err(master->dev, "<%s> Cannot get runtime PM.\n", __func__);
1411                 return ret;
1412         }
1413
1414         svc_i3c_master_enable_interrupts(master, SVC_I3C_MINT_SLVSTART);
1415
1416         return i3c_master_enec_locked(m, dev->info.dyn_addr, I3C_CCC_EVENT_SIR);
1417 }
1418
1419 static int svc_i3c_master_disable_ibi(struct i3c_dev_desc *dev)
1420 {
1421         struct i3c_master_controller *m = i3c_dev_get_master(dev);
1422         struct svc_i3c_master *master = to_svc_i3c_master(m);
1423         int ret;
1424
1425         svc_i3c_master_disable_interrupts(master);
1426
1427         ret = i3c_master_disec_locked(m, dev->info.dyn_addr, I3C_CCC_EVENT_SIR);
1428
1429         pm_runtime_mark_last_busy(master->dev);
1430         pm_runtime_put_autosuspend(master->dev);
1431
1432         return ret;
1433 }
1434
1435 static void svc_i3c_master_recycle_ibi_slot(struct i3c_dev_desc *dev,
1436                                             struct i3c_ibi_slot *slot)
1437 {
1438         struct svc_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev);
1439
1440         i3c_generic_ibi_recycle_slot(data->ibi_pool, slot);
1441 }
1442
1443 static const struct i3c_master_controller_ops svc_i3c_master_ops = {
1444         .bus_init = svc_i3c_master_bus_init,
1445         .bus_cleanup = svc_i3c_master_bus_cleanup,
1446         .attach_i3c_dev = svc_i3c_master_attach_i3c_dev,
1447         .detach_i3c_dev = svc_i3c_master_detach_i3c_dev,
1448         .reattach_i3c_dev = svc_i3c_master_reattach_i3c_dev,
1449         .attach_i2c_dev = svc_i3c_master_attach_i2c_dev,
1450         .detach_i2c_dev = svc_i3c_master_detach_i2c_dev,
1451         .do_daa = svc_i3c_master_do_daa,
1452         .supports_ccc_cmd = svc_i3c_master_supports_ccc_cmd,
1453         .send_ccc_cmd = svc_i3c_master_send_ccc_cmd,
1454         .priv_xfers = svc_i3c_master_priv_xfers,
1455         .i2c_xfers = svc_i3c_master_i2c_xfers,
1456         .request_ibi = svc_i3c_master_request_ibi,
1457         .free_ibi = svc_i3c_master_free_ibi,
1458         .recycle_ibi_slot = svc_i3c_master_recycle_ibi_slot,
1459         .enable_ibi = svc_i3c_master_enable_ibi,
1460         .disable_ibi = svc_i3c_master_disable_ibi,
1461 };
1462
1463 static int svc_i3c_master_prepare_clks(struct svc_i3c_master *master)
1464 {
1465         int ret = 0;
1466
1467         ret = clk_prepare_enable(master->pclk);
1468         if (ret)
1469                 return ret;
1470
1471         ret = clk_prepare_enable(master->fclk);
1472         if (ret) {
1473                 clk_disable_unprepare(master->pclk);
1474                 return ret;
1475         }
1476
1477         ret = clk_prepare_enable(master->sclk);
1478         if (ret) {
1479                 clk_disable_unprepare(master->pclk);
1480                 clk_disable_unprepare(master->fclk);
1481                 return ret;
1482         }
1483
1484         return 0;
1485 }
1486
1487 static void svc_i3c_master_unprepare_clks(struct svc_i3c_master *master)
1488 {
1489         clk_disable_unprepare(master->pclk);
1490         clk_disable_unprepare(master->fclk);
1491         clk_disable_unprepare(master->sclk);
1492 }
1493
1494 static int svc_i3c_master_probe(struct platform_device *pdev)
1495 {
1496         struct device *dev = &pdev->dev;
1497         struct svc_i3c_master *master;
1498         int ret;
1499
1500         master = devm_kzalloc(dev, sizeof(*master), GFP_KERNEL);
1501         if (!master)
1502                 return -ENOMEM;
1503
1504         master->regs = devm_platform_ioremap_resource(pdev, 0);
1505         if (IS_ERR(master->regs))
1506                 return PTR_ERR(master->regs);
1507
1508         master->pclk = devm_clk_get(dev, "pclk");
1509         if (IS_ERR(master->pclk))
1510                 return PTR_ERR(master->pclk);
1511
1512         master->fclk = devm_clk_get(dev, "fast_clk");
1513         if (IS_ERR(master->fclk))
1514                 return PTR_ERR(master->fclk);
1515
1516         master->sclk = devm_clk_get(dev, "slow_clk");
1517         if (IS_ERR(master->sclk))
1518                 return PTR_ERR(master->sclk);
1519
1520         master->irq = platform_get_irq(pdev, 0);
1521         if (master->irq <= 0)
1522                 return -ENOENT;
1523
1524         master->dev = dev;
1525
1526         ret = svc_i3c_master_prepare_clks(master);
1527         if (ret)
1528                 return ret;
1529
1530         INIT_WORK(&master->hj_work, svc_i3c_master_hj_work);
1531         INIT_WORK(&master->ibi_work, svc_i3c_master_ibi_work);
1532         ret = devm_request_irq(dev, master->irq, svc_i3c_master_irq_handler,
1533                                IRQF_NO_SUSPEND, "svc-i3c-irq", master);
1534         if (ret)
1535                 goto err_disable_clks;
1536
1537         master->free_slots = GENMASK(SVC_I3C_MAX_DEVS - 1, 0);
1538
1539         spin_lock_init(&master->xferqueue.lock);
1540         INIT_LIST_HEAD(&master->xferqueue.list);
1541
1542         spin_lock_init(&master->ibi.lock);
1543         master->ibi.num_slots = SVC_I3C_MAX_DEVS;
1544         master->ibi.slots = devm_kcalloc(&pdev->dev, master->ibi.num_slots,
1545                                          sizeof(*master->ibi.slots),
1546                                          GFP_KERNEL);
1547         if (!master->ibi.slots) {
1548                 ret = -ENOMEM;
1549                 goto err_disable_clks;
1550         }
1551
1552         platform_set_drvdata(pdev, master);
1553
1554         pm_runtime_set_autosuspend_delay(&pdev->dev, SVC_I3C_PM_TIMEOUT_MS);
1555         pm_runtime_use_autosuspend(&pdev->dev);
1556         pm_runtime_get_noresume(&pdev->dev);
1557         pm_runtime_set_active(&pdev->dev);
1558         pm_runtime_enable(&pdev->dev);
1559
1560         svc_i3c_master_reset(master);
1561
1562         /* Register the master */
1563         ret = i3c_master_register(&master->base, &pdev->dev,
1564                                   &svc_i3c_master_ops, false);
1565         if (ret)
1566                 goto rpm_disable;
1567
1568         pm_runtime_mark_last_busy(&pdev->dev);
1569         pm_runtime_put_autosuspend(&pdev->dev);
1570
1571         return 0;
1572
1573 rpm_disable:
1574         pm_runtime_dont_use_autosuspend(&pdev->dev);
1575         pm_runtime_put_noidle(&pdev->dev);
1576         pm_runtime_set_suspended(&pdev->dev);
1577         pm_runtime_disable(&pdev->dev);
1578
1579 err_disable_clks:
1580         svc_i3c_master_unprepare_clks(master);
1581
1582         return ret;
1583 }
1584
1585 static void svc_i3c_master_remove(struct platform_device *pdev)
1586 {
1587         struct svc_i3c_master *master = platform_get_drvdata(pdev);
1588
1589         i3c_master_unregister(&master->base);
1590
1591         pm_runtime_dont_use_autosuspend(&pdev->dev);
1592         pm_runtime_disable(&pdev->dev);
1593 }
1594
1595 static void svc_i3c_save_regs(struct svc_i3c_master *master)
1596 {
1597         master->saved_regs.mconfig = readl(master->regs + SVC_I3C_MCONFIG);
1598         master->saved_regs.mdynaddr = readl(master->regs + SVC_I3C_MDYNADDR);
1599 }
1600
1601 static void svc_i3c_restore_regs(struct svc_i3c_master *master)
1602 {
1603         if (readl(master->regs + SVC_I3C_MDYNADDR) !=
1604             master->saved_regs.mdynaddr) {
1605                 writel(master->saved_regs.mconfig,
1606                        master->regs + SVC_I3C_MCONFIG);
1607                 writel(master->saved_regs.mdynaddr,
1608                        master->regs + SVC_I3C_MDYNADDR);
1609         }
1610 }
1611
1612 static int __maybe_unused svc_i3c_runtime_suspend(struct device *dev)
1613 {
1614         struct svc_i3c_master *master = dev_get_drvdata(dev);
1615
1616         svc_i3c_save_regs(master);
1617         svc_i3c_master_unprepare_clks(master);
1618         pinctrl_pm_select_sleep_state(dev);
1619
1620         return 0;
1621 }
1622
1623 static int __maybe_unused svc_i3c_runtime_resume(struct device *dev)
1624 {
1625         struct svc_i3c_master *master = dev_get_drvdata(dev);
1626
1627         pinctrl_pm_select_default_state(dev);
1628         svc_i3c_master_prepare_clks(master);
1629
1630         svc_i3c_restore_regs(master);
1631
1632         return 0;
1633 }
1634
1635 static const struct dev_pm_ops svc_i3c_pm_ops = {
1636         SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
1637                                       pm_runtime_force_resume)
1638         SET_RUNTIME_PM_OPS(svc_i3c_runtime_suspend,
1639                            svc_i3c_runtime_resume, NULL)
1640 };
1641
1642 static const struct of_device_id svc_i3c_master_of_match_tbl[] = {
1643         { .compatible = "silvaco,i3c-master" },
1644         { /* sentinel */ },
1645 };
1646 MODULE_DEVICE_TABLE(of, svc_i3c_master_of_match_tbl);
1647
1648 static struct platform_driver svc_i3c_master = {
1649         .probe = svc_i3c_master_probe,
1650         .remove_new = svc_i3c_master_remove,
1651         .driver = {
1652                 .name = "silvaco-i3c-master",
1653                 .of_match_table = svc_i3c_master_of_match_tbl,
1654                 .pm = &svc_i3c_pm_ops,
1655         },
1656 };
1657 module_platform_driver(svc_i3c_master);
1658
1659 MODULE_AUTHOR("Conor Culhane <conor.culhane@silvaco.com>");
1660 MODULE_AUTHOR("Miquel Raynal <miquel.raynal@bootlin.com>");
1661 MODULE_DESCRIPTION("Silvaco dual-role I3C master driver");
1662 MODULE_LICENSE("GPL v2");