sound/soc/sof/ipc.c

   1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
   2 //
   3 // This file is provided under a dual BSD/GPLv2 license.  When using or
   4 // redistributing this file, you may do so under either license.
   5 //
   6 // Copyright(c) 2018 Intel Corporation. All rights reserved.
   7 //
   8 // Author: Liam Girdwood <liam.r.girdwood@linux.intel.com>
   9 //
  10 // Generic IPC layer that can work over MMIO and SPI/I2C. PHY layer provided
  11 // by platform driver code.
  12 //
  13
  14 #include <linux/mutex.h>
  15 #include <linux/types.h>
  16
  17 #include "sof-priv.h"
  18 #include "sof-audio.h"
  19 #include "ops.h"
  20 #include "ipc3-ops.h"
  21
  22 typedef void (*ipc_rx_callback)(struct snd_sof_dev *sdev, void *msg_buf);
  23
  24 static void ipc_trace_message(struct snd_sof_dev *sdev, void *msg_buf);
  25 static void ipc_stream_message(struct snd_sof_dev *sdev, void *msg_buf);
  26
  27 /*
  28  * IPC message Tx/Rx message handling.
  29  */
  30
  31 struct sof_ipc_ctrl_data_params {
  32         size_t msg_bytes;
  33         size_t hdr_bytes;
  34         size_t pl_size;
  35         size_t elems;
  36         u32 num_msg;
  37         u8 *src;
  38         u8 *dst;
  39 };
  40
  41 #if IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_VERBOSE_IPC)
  42 static void ipc_log_header(struct device *dev, u8 *text, u32 cmd)
  43 {
  44         u8 *str;
  45         u8 *str2 = NULL;
  46         u32 glb;
  47         u32 type;
  48         bool vdbg = false;
  49
  50         glb = cmd & SOF_GLB_TYPE_MASK;
  51         type = cmd & SOF_CMD_TYPE_MASK;
  52
  53         switch (glb) {
  54         case SOF_IPC_GLB_REPLY:
  55                 str = "GLB_REPLY"; break;
  56         case SOF_IPC_GLB_COMPOUND:
  57                 str = "GLB_COMPOUND"; break;
  58         case SOF_IPC_GLB_TPLG_MSG:
  59                 str = "GLB_TPLG_MSG";
  60                 switch (type) {
  61                 case SOF_IPC_TPLG_COMP_NEW:
  62                         str2 = "COMP_NEW"; break;
  63                 case SOF_IPC_TPLG_COMP_FREE:
  64                         str2 = "COMP_FREE"; break;
  65                 case SOF_IPC_TPLG_COMP_CONNECT:
  66                         str2 = "COMP_CONNECT"; break;
  67                 case SOF_IPC_TPLG_PIPE_NEW:
  68                         str2 = "PIPE_NEW"; break;
  69                 case SOF_IPC_TPLG_PIPE_FREE:
  70                         str2 = "PIPE_FREE"; break;
  71                 case SOF_IPC_TPLG_PIPE_CONNECT:
  72                         str2 = "PIPE_CONNECT"; break;
  73                 case SOF_IPC_TPLG_PIPE_COMPLETE:
  74                         str2 = "PIPE_COMPLETE"; break;
  75                 case SOF_IPC_TPLG_BUFFER_NEW:
  76                         str2 = "BUFFER_NEW"; break;
  77                 case SOF_IPC_TPLG_BUFFER_FREE:
  78                         str2 = "BUFFER_FREE"; break;
  79                 default:
  80                         str2 = "unknown type"; break;
  81                 }
  82                 break;
  83         case SOF_IPC_GLB_PM_MSG:
  84                 str = "GLB_PM_MSG";
  85                 switch (type) {
  86                 case SOF_IPC_PM_CTX_SAVE:
  87                         str2 = "CTX_SAVE"; break;
  88                 case SOF_IPC_PM_CTX_RESTORE:
  89                         str2 = "CTX_RESTORE"; break;
  90                 case SOF_IPC_PM_CTX_SIZE:
  91                         str2 = "CTX_SIZE"; break;
  92                 case SOF_IPC_PM_CLK_SET:
  93                         str2 = "CLK_SET"; break;
  94                 case SOF_IPC_PM_CLK_GET:
  95                         str2 = "CLK_GET"; break;
  96                 case SOF_IPC_PM_CLK_REQ:
  97                         str2 = "CLK_REQ"; break;
  98                 case SOF_IPC_PM_CORE_ENABLE:
  99                         str2 = "CORE_ENABLE"; break;
 100                 case SOF_IPC_PM_GATE:
 101                         str2 = "GATE"; break;
 102                 default:
 103                         str2 = "unknown type"; break;
 104                 }
 105                 break;
 106         case SOF_IPC_GLB_COMP_MSG:
 107                 str = "GLB_COMP_MSG";
 108                 switch (type) {
 109                 case SOF_IPC_COMP_SET_VALUE:
 110                         str2 = "SET_VALUE"; break;
 111                 case SOF_IPC_COMP_GET_VALUE:
 112                         str2 = "GET_VALUE"; break;
 113                 case SOF_IPC_COMP_SET_DATA:
 114                         str2 = "SET_DATA"; break;
 115                 case SOF_IPC_COMP_GET_DATA:
 116                         str2 = "GET_DATA"; break;
 117                 default:
 118                         str2 = "unknown type"; break;
 119                 }
 120                 break;
 121         case SOF_IPC_GLB_STREAM_MSG:
 122                 str = "GLB_STREAM_MSG";
 123                 switch (type) {
 124                 case SOF_IPC_STREAM_PCM_PARAMS:
 125                         str2 = "PCM_PARAMS"; break;
 126                 case SOF_IPC_STREAM_PCM_PARAMS_REPLY:
 127                         str2 = "PCM_REPLY"; break;
 128                 case SOF_IPC_STREAM_PCM_FREE:
 129                         str2 = "PCM_FREE"; break;
 130                 case SOF_IPC_STREAM_TRIG_START:
 131                         str2 = "TRIG_START"; break;
 132                 case SOF_IPC_STREAM_TRIG_STOP:
 133                         str2 = "TRIG_STOP"; break;
 134                 case SOF_IPC_STREAM_TRIG_PAUSE:
 135                         str2 = "TRIG_PAUSE"; break;
 136                 case SOF_IPC_STREAM_TRIG_RELEASE:
 137                         str2 = "TRIG_RELEASE"; break;
 138                 case SOF_IPC_STREAM_TRIG_DRAIN:
 139                         str2 = "TRIG_DRAIN"; break;
 140                 case SOF_IPC_STREAM_TRIG_XRUN:
 141                         str2 = "TRIG_XRUN"; break;
 142                 case SOF_IPC_STREAM_POSITION:
 143                         vdbg = true;
 144                         str2 = "POSITION"; break;
 145                 case SOF_IPC_STREAM_VORBIS_PARAMS:
 146                         str2 = "VORBIS_PARAMS"; break;
 147                 case SOF_IPC_STREAM_VORBIS_FREE:
 148                         str2 = "VORBIS_FREE"; break;
 149                 default:
 150                         str2 = "unknown type"; break;
 151                 }
 152                 break;
 153         case SOF_IPC_FW_READY:
 154                 str = "FW_READY"; break;
 155         case SOF_IPC_GLB_DAI_MSG:
 156                 str = "GLB_DAI_MSG";
 157                 switch (type) {
 158                 case SOF_IPC_DAI_CONFIG:
 159                         str2 = "CONFIG"; break;
 160                 case SOF_IPC_DAI_LOOPBACK:
 161                         str2 = "LOOPBACK"; break;
 162                 default:
 163                         str2 = "unknown type"; break;
 164                 }
 165                 break;
 166         case SOF_IPC_GLB_TRACE_MSG:
 167                 str = "GLB_TRACE_MSG";
 168                 switch (type) {
 169                 case SOF_IPC_TRACE_DMA_PARAMS:
 170                         str2 = "DMA_PARAMS"; break;
 171                 case SOF_IPC_TRACE_DMA_POSITION:
 172                         str2 = "DMA_POSITION"; break;
 173                 case SOF_IPC_TRACE_DMA_PARAMS_EXT:
 174                         str2 = "DMA_PARAMS_EXT"; break;
 175                 case SOF_IPC_TRACE_FILTER_UPDATE:
 176                         str2 = "FILTER_UPDATE"; break;
 177                 case SOF_IPC_TRACE_DMA_FREE:
 178                         str2 = "DMA_FREE"; break;
 179                 default:
 180                         str2 = "unknown type"; break;
 181                 }
 182                 break;
 183         case SOF_IPC_GLB_TEST_MSG:
 184                 str = "GLB_TEST_MSG";
 185                 switch (type) {
 186                 case SOF_IPC_TEST_IPC_FLOOD:
 187                         str2 = "IPC_FLOOD"; break;
 188                 default:
 189                         str2 = "unknown type"; break;
 190                 }
 191                 break;
 192         case SOF_IPC_GLB_DEBUG:
 193                 str = "GLB_DEBUG";
 194                 switch (type) {
 195                 case SOF_IPC_DEBUG_MEM_USAGE:
 196                         str2 = "MEM_USAGE"; break;
 197                 default:
 198                         str2 = "unknown type"; break;
 199                 }
 200                 break;
 201         case SOF_IPC_GLB_PROBE:
 202                 str = "GLB_PROBE";
 203                 switch (type) {
 204                 case SOF_IPC_PROBE_INIT:
 205                         str2 = "INIT"; break;
 206                 case SOF_IPC_PROBE_DEINIT:
 207                         str2 = "DEINIT"; break;
 208                 case SOF_IPC_PROBE_DMA_ADD:
 209                         str2 = "DMA_ADD"; break;
 210                 case SOF_IPC_PROBE_DMA_INFO:
 211                         str2 = "DMA_INFO"; break;
 212                 case SOF_IPC_PROBE_DMA_REMOVE:
 213                         str2 = "DMA_REMOVE"; break;
 214                 case SOF_IPC_PROBE_POINT_ADD:
 215                         str2 = "POINT_ADD"; break;
 216                 case SOF_IPC_PROBE_POINT_INFO:
 217                         str2 = "POINT_INFO"; break;
 218                 case SOF_IPC_PROBE_POINT_REMOVE:
 219                         str2 = "POINT_REMOVE"; break;
 220                 default:
 221                         str2 = "unknown type"; break;
 222                 }
 223                 break;
 224         default:
 225                 str = "unknown GLB command"; break;
 226         }
 227
 228         if (str2) {
 229                 if (vdbg)
 230                         dev_vdbg(dev, "%s: 0x%x: %s: %s\n", text, cmd, str, str2);
 231                 else
 232                         dev_dbg(dev, "%s: 0x%x: %s: %s\n", text, cmd, str, str2);
 233         } else {
 234                 dev_dbg(dev, "%s: 0x%x: %s\n", text, cmd, str);
 235         }
 236 }
 237 #else
 238 static inline void ipc_log_header(struct device *dev, u8 *text, u32 cmd)
 239 {
 240         if ((cmd & SOF_GLB_TYPE_MASK) != SOF_IPC_GLB_TRACE_MSG)
 241                 dev_dbg(dev, "%s: 0x%x\n", text, cmd);
 242 }
 243 #endif
 244
 245 /* wait for IPC message reply */
 246 static int tx_wait_done(struct snd_sof_ipc *ipc, struct snd_sof_ipc_msg *msg,
 247                         void *reply_data)
 248 {
 249         struct snd_sof_dev *sdev = ipc->sdev;
 250         struct sof_ipc_cmd_hdr *hdr = msg->msg_data;
 251         int ret;
 252
 253         /* wait for DSP IPC completion */
 254         ret = wait_event_timeout(msg->waitq, msg->ipc_complete,
 255                                  msecs_to_jiffies(sdev->ipc_timeout));
 256
 257         if (ret == 0) {
 258                 dev_err(sdev->dev,
 259                         "ipc tx timed out for %#x (msg/reply size: %d/%zu)\n",
 260                         hdr->cmd, hdr->size, msg->reply_size);
 261                 snd_sof_handle_fw_exception(ipc->sdev);
 262                 ret = -ETIMEDOUT;
 263         } else {
 264                 ret = msg->reply_error;
 265                 if (ret < 0) {
 266                         dev_err(sdev->dev,
 267                                 "ipc tx error for %#x (msg/reply size: %d/%zu): %d\n",
 268                                 hdr->cmd, hdr->size, msg->reply_size, ret);
 269                 } else {
 270                         ipc_log_header(sdev->dev, "ipc tx succeeded", hdr->cmd);
 271                         if (msg->reply_size)
 272                                 /* copy the data returned from DSP */
 273                                 memcpy(reply_data, msg->reply_data,
 274                                        msg->reply_size);
 275                 }
 276
 277                 /* re-enable dumps after successful IPC tx */
 278                 if (sdev->ipc_dump_printed) {
 279                         sdev->dbg_dump_printed = false;
 280                         sdev->ipc_dump_printed = false;
 281                 }
 282         }
 283
 284         return ret;
 285 }
 286
 287 /* send IPC message from host to DSP */
 288 static int sof_ipc_tx_message_unlocked(struct snd_sof_ipc *ipc,
 289                                        void *msg_data, size_t msg_bytes,
 290                                        void *reply_data, size_t reply_bytes)
 291 {
 292         struct sof_ipc_cmd_hdr *hdr = msg_data;
 293         struct snd_sof_dev *sdev = ipc->sdev;
 294         struct snd_sof_ipc_msg *msg;
 295         int ret;
 296
 297         if (!msg_data || msg_bytes < sizeof(*hdr)) {
 298                 dev_err_ratelimited(sdev->dev, "No IPC message to send\n");
 299                 return -EINVAL;
 300         }
 301
 302         if (ipc->disable_ipc_tx || sdev->fw_state != SOF_FW_BOOT_COMPLETE)
 303                 return -ENODEV;
 304
 305         /*
 306          * The spin-lock is also still needed to protect message objects against
 307          * other atomic contexts.
 308          */
 309         spin_lock_irq(&sdev->ipc_lock);
 310
 311         /* initialise the message */
 312         msg = &ipc->msg;
 313
 314         /* attach message data */
 315         msg->msg_data = msg_data;
 316         msg->msg_size = msg_bytes;
 317
 318         msg->reply_size = reply_bytes;
 319         msg->reply_error = 0;
 320
 321         sdev->msg = msg;
 322
 323         ret = snd_sof_dsp_send_msg(sdev, msg);
 324         /* Next reply that we receive will be related to this message */
 325         if (!ret)
 326                 msg->ipc_complete = false;
 327
 328         spin_unlock_irq(&sdev->ipc_lock);
 329
 330         if (ret) {
 331                 dev_err_ratelimited(sdev->dev,
 332                                     "error: ipc tx failed with error %d\n",
 333                                     ret);
 334                 return ret;
 335         }
 336
 337         ipc_log_header(sdev->dev, "ipc tx", hdr->cmd);
 338
 339         /* now wait for completion */
 340         return tx_wait_done(ipc, msg, reply_data);
 341 }
 342
 343 /* send IPC message from host to DSP */
 344 int sof_ipc_tx_message(struct snd_sof_ipc *ipc, u32 header,
 345                        void *msg_data, size_t msg_bytes, void *reply_data,
 346                        size_t reply_bytes)
 347 {
 348         const struct sof_dsp_power_state target_state = {
 349                 .state = SOF_DSP_PM_D0,
 350         };
 351         int ret;
 352
 353         /* ensure the DSP is in D0 before sending a new IPC */
 354         ret = snd_sof_dsp_set_power_state(ipc->sdev, &target_state);
 355         if (ret < 0) {
 356                 dev_err(ipc->sdev->dev, "error: resuming DSP %d\n", ret);
 357                 return ret;
 358         }
 359
 360         return sof_ipc_tx_message_no_pm(ipc, header, msg_data, msg_bytes,
 361                                         reply_data, reply_bytes);
 362 }
 363 EXPORT_SYMBOL(sof_ipc_tx_message);
 364
 365 /*
 366  * send IPC message from host to DSP without modifying the DSP state.
 367  * This will be used for IPC's that can be handled by the DSP
 368  * even in a low-power D0 substate.
 369  */
 370 int sof_ipc_tx_message_no_pm(struct snd_sof_ipc *ipc, u32 header,
 371                              void *msg_data, size_t msg_bytes,
 372                              void *reply_data, size_t reply_bytes)
 373 {
 374         int ret;
 375
 376         if (msg_bytes > SOF_IPC_MSG_MAX_SIZE ||
 377             reply_bytes > SOF_IPC_MSG_MAX_SIZE)
 378                 return -ENOBUFS;
 379
 380         /* Serialise IPC TX */
 381         mutex_lock(&ipc->tx_mutex);
 382
 383         ret = sof_ipc_tx_message_unlocked(ipc, msg_data, msg_bytes,
 384                                           reply_data, reply_bytes);
 385
 386         mutex_unlock(&ipc->tx_mutex);
 387
 388         return ret;
 389 }
 390 EXPORT_SYMBOL(sof_ipc_tx_message_no_pm);
 391
 392 /* Generic helper function to retrieve the reply */
 393 void snd_sof_ipc_get_reply(struct snd_sof_dev *sdev)
 394 {
 395         struct snd_sof_ipc_msg *msg = sdev->msg;
 396         struct sof_ipc_reply reply;
 397         int ret = 0;
 398
 399         /*
 400          * Sometimes, there is unexpected reply ipc arriving. The reply
 401          * ipc belongs to none of the ipcs sent from driver.
 402          * In this case, the driver must ignore the ipc.
 403          */
 404         if (!msg) {
 405                 dev_warn(sdev->dev, "unexpected ipc interrupt raised!\n");
 406                 return;
 407         }
 408
 409         /* get the generic reply */
 410         snd_sof_dsp_mailbox_read(sdev, sdev->host_box.offset, &reply,
 411                                  sizeof(reply));
 412
 413         if (reply.error < 0) {
 414                 memcpy(msg->reply_data, &reply, sizeof(reply));
 415                 ret = reply.error;
 416         } else if (!reply.hdr.size) {
 417                 /* Reply should always be >= sizeof(struct sof_ipc_reply) */
 418                 if (msg->reply_size)
 419                         dev_err(sdev->dev,
 420                                 "empty reply received, expected %zu bytes\n",
 421                                 msg->reply_size);
 422                 else
 423                         dev_err(sdev->dev, "empty reply received\n");
 424
 425                 ret = -EINVAL;
 426         } else if (msg->reply_size > 0) {
 427                 if (reply.hdr.size == msg->reply_size) {
 428                         ret = 0;
 429                 } else if (reply.hdr.size < msg->reply_size) {
 430                         dev_dbg(sdev->dev,
 431                                 "reply size (%u) is less than expected (%zu)\n",
 432                                 reply.hdr.size, msg->reply_size);
 433
 434                         msg->reply_size = reply.hdr.size;
 435                         ret = 0;
 436                 } else {
 437                         dev_err(sdev->dev,
 438                                 "reply size (%u) exceeds the buffer size (%zu)\n",
 439                                 reply.hdr.size, msg->reply_size);
 440                         ret = -EINVAL;
 441                 }
 442
 443                 /* get the full message if reply.hdr.size <= msg->reply_size */
 444                 if (!ret)
 445                         snd_sof_dsp_mailbox_read(sdev, sdev->host_box.offset,
 446                                                  msg->reply_data, msg->reply_size);
 447         }
 448
 449         msg->reply_error = ret;
 450 }
 451 EXPORT_SYMBOL(snd_sof_ipc_get_reply);
 452
 453 /* handle reply message from DSP */
 454 void snd_sof_ipc_reply(struct snd_sof_dev *sdev, u32 msg_id)
 455 {
 456         struct snd_sof_ipc_msg *msg = &sdev->ipc->msg;
 457
 458         if (msg->ipc_complete) {
 459                 dev_dbg(sdev->dev,
 460                         "no reply expected, received 0x%x, will be ignored",
 461                         msg_id);
 462                 return;
 463         }
 464
 465         /* wake up and return the error if we have waiters on this message ? */
 466         msg->ipc_complete = true;
 467         wake_up(&msg->waitq);
 468 }
 469 EXPORT_SYMBOL(snd_sof_ipc_reply);
 470
 471 static void ipc_comp_notification(struct snd_sof_dev *sdev, void *msg_buf)
 472 {
 473         const struct sof_ipc_tplg_ops *tplg_ops = sdev->ipc->ops->tplg;
 474         struct sof_ipc_cmd_hdr *hdr = msg_buf;
 475         u32 msg_type = hdr->cmd & SOF_CMD_TYPE_MASK;
 476
 477         switch (msg_type) {
 478         case SOF_IPC_COMP_GET_VALUE:
 479         case SOF_IPC_COMP_GET_DATA:
 480                 break;
 481         default:
 482                 dev_err(sdev->dev, "error: unhandled component message %#x\n", msg_type);
 483                 return;
 484         }
 485
 486         if (tplg_ops->control->update)
 487                 tplg_ops->control->update(sdev, msg_buf);
 488 }
 489
 490 /* DSP firmware has sent host a message  */
 491 void snd_sof_ipc_msgs_rx(struct snd_sof_dev *sdev)
 492 {
 493         ipc_rx_callback rx_callback = NULL;
 494         struct sof_ipc_cmd_hdr hdr;
 495         void *msg_buf;
 496         u32 cmd;
 497         int err;
 498
 499         /* read back header */
 500         err = snd_sof_ipc_msg_data(sdev, NULL, &hdr, sizeof(hdr));
 501         if (err < 0) {
 502                 dev_warn(sdev->dev, "failed to read IPC header: %d\n", err);
 503                 return;
 504         }
 505
 506         if (hdr.size < sizeof(hdr)) {
 507                 dev_err(sdev->dev, "The received message size is invalid\n");
 508                 return;
 509         }
 510
 511         ipc_log_header(sdev->dev, "ipc rx", hdr.cmd);
 512
 513         cmd = hdr.cmd & SOF_GLB_TYPE_MASK;
 514
 515         /* check message type */
 516         switch (cmd) {
 517         case SOF_IPC_GLB_REPLY:
 518                 dev_err(sdev->dev, "error: ipc reply unknown\n");
 519                 break;
 520         case SOF_IPC_FW_READY:
 521                 /* check for FW boot completion */
 522                 if (sdev->fw_state == SOF_FW_BOOT_IN_PROGRESS) {
 523                         err = sof_ops(sdev)->fw_ready(sdev, cmd);
 524                         if (err < 0)
 525                                 sof_set_fw_state(sdev, SOF_FW_BOOT_READY_FAILED);
 526                         else
 527                                 sof_set_fw_state(sdev, SOF_FW_BOOT_READY_OK);
 528
 529                         /* wake up firmware loader */
 530                         wake_up(&sdev->boot_wait);
 531                 }
 532                 break;
 533         case SOF_IPC_GLB_COMPOUND:
 534         case SOF_IPC_GLB_TPLG_MSG:
 535         case SOF_IPC_GLB_PM_MSG:
 536                 break;
 537         case SOF_IPC_GLB_COMP_MSG:
 538                 rx_callback = ipc_comp_notification;
 539                 break;
 540         case SOF_IPC_GLB_STREAM_MSG:
 541                 rx_callback = ipc_stream_message;
 542                 break;
 543         case SOF_IPC_GLB_TRACE_MSG:
 544                 rx_callback = ipc_trace_message;
 545                 break;
 546         default:
 547                 dev_err(sdev->dev, "%s: Unknown DSP message: 0x%x\n", __func__, cmd);
 548                 break;
 549         }
 550
 551         /* read the full message */
 552         msg_buf = kmalloc(hdr.size, GFP_KERNEL);
 553         if (!msg_buf)
 554                 return;
 555
 556         err = snd_sof_ipc_msg_data(sdev, NULL, msg_buf, hdr.size);
 557         if (err < 0) {
 558                 dev_err(sdev->dev, "%s: Failed to read message: %d\n", __func__, err);
 559         } else {
 560                 /* Call local handler for the message */
 561                 if (rx_callback)
 562                         rx_callback(sdev, msg_buf);
 563
 564                 /* Notify registered clients */
 565                 sof_client_ipc_rx_dispatcher(sdev, msg_buf);
 566         }
 567
 568         kfree(msg_buf);
 569
 570         ipc_log_header(sdev->dev, "ipc rx done", hdr.cmd);
 571 }
 572 EXPORT_SYMBOL(snd_sof_ipc_msgs_rx);
 573
 574 /*
 575  * IPC trace mechanism.
 576  */
 577
 578 static void ipc_trace_message(struct snd_sof_dev *sdev, void *msg_buf)
 579 {
 580         struct sof_ipc_cmd_hdr *hdr = msg_buf;
 581         u32 msg_type = hdr->cmd & SOF_CMD_TYPE_MASK;
 582
 583         switch (msg_type) {
 584         case SOF_IPC_TRACE_DMA_POSITION:
 585                 snd_sof_trace_update_pos(sdev, msg_buf);
 586                 break;
 587         default:
 588                 dev_err(sdev->dev, "error: unhandled trace message %#x\n", msg_type);
 589                 break;
 590         }
 591 }
 592
 593 /*
 594  * IPC stream position.
 595  */
 596
 597 static void ipc_period_elapsed(struct snd_sof_dev *sdev, u32 msg_id)
 598 {
 599         struct snd_soc_component *scomp = sdev->component;
 600         struct snd_sof_pcm_stream *stream;
 601         struct sof_ipc_stream_posn posn;
 602         struct snd_sof_pcm *spcm;
 603         int direction, ret;
 604
 605         spcm = snd_sof_find_spcm_comp(scomp, msg_id, &direction);
 606         if (!spcm) {
 607                 dev_err(sdev->dev,
 608                         "error: period elapsed for unknown stream, msg_id %d\n",
 609                         msg_id);
 610                 return;
 611         }
 612
 613         stream = &spcm->stream[direction];
 614         ret = snd_sof_ipc_msg_data(sdev, stream->substream, &posn, sizeof(posn));
 615         if (ret < 0) {
 616                 dev_warn(sdev->dev, "failed to read stream position: %d\n", ret);
 617                 return;
 618         }
 619
 620         dev_vdbg(sdev->dev, "posn : host 0x%llx dai 0x%llx wall 0x%llx\n",
 621                  posn.host_posn, posn.dai_posn, posn.wallclock);
 622
 623         memcpy(&stream->posn, &posn, sizeof(posn));
 624
 625         if (spcm->pcm.compress)
 626                 snd_sof_compr_fragment_elapsed(stream->cstream);
 627         else if (stream->substream->runtime &&
 628                  !stream->substream->runtime->no_period_wakeup)
 629                 /* only inform ALSA for period_wakeup mode */
 630                 snd_sof_pcm_period_elapsed(stream->substream);
 631 }
 632
 633 /* DSP notifies host of an XRUN within FW */
 634 static void ipc_xrun(struct snd_sof_dev *sdev, u32 msg_id)
 635 {
 636         struct snd_soc_component *scomp = sdev->component;
 637         struct snd_sof_pcm_stream *stream;
 638         struct sof_ipc_stream_posn posn;
 639         struct snd_sof_pcm *spcm;
 640         int direction, ret;
 641
 642         spcm = snd_sof_find_spcm_comp(scomp, msg_id, &direction);
 643         if (!spcm) {
 644                 dev_err(sdev->dev, "error: XRUN for unknown stream, msg_id %d\n",
 645                         msg_id);
 646                 return;
 647         }
 648
 649         stream = &spcm->stream[direction];
 650         ret = snd_sof_ipc_msg_data(sdev, stream->substream, &posn, sizeof(posn));
 651         if (ret < 0) {
 652                 dev_warn(sdev->dev, "failed to read overrun position: %d\n", ret);
 653                 return;
 654         }
 655
 656         dev_dbg(sdev->dev,  "posn XRUN: host %llx comp %d size %d\n",
 657                 posn.host_posn, posn.xrun_comp_id, posn.xrun_size);
 658
 659 #if defined(CONFIG_SND_SOC_SOF_DEBUG_XRUN_STOP)
 660         /* stop PCM on XRUN - used for pipeline debug */
 661         memcpy(&stream->posn, &posn, sizeof(posn));
 662         snd_pcm_stop_xrun(stream->substream);
 663 #endif
 664 }
 665
 666 /* stream notifications from DSP FW */
 667 static void ipc_stream_message(struct snd_sof_dev *sdev, void *msg_buf)
 668 {
 669         struct sof_ipc_cmd_hdr *hdr = msg_buf;
 670         u32 msg_type = hdr->cmd & SOF_CMD_TYPE_MASK;
 671         u32 msg_id = SOF_IPC_MESSAGE_ID(hdr->cmd);
 672
 673         switch (msg_type) {
 674         case SOF_IPC_STREAM_POSITION:
 675                 ipc_period_elapsed(sdev, msg_id);
 676                 break;
 677         case SOF_IPC_STREAM_TRIG_XRUN:
 678                 ipc_xrun(sdev, msg_id);
 679                 break;
 680         default:
 681                 dev_err(sdev->dev, "error: unhandled stream message %#x\n",
 682                         msg_id);
 683                 break;
 684         }
 685 }
 686
 687 /* get stream position IPC - use faster MMIO method if available on platform */
 688 int snd_sof_ipc_stream_posn(struct snd_soc_component *scomp,
 689                             struct snd_sof_pcm *spcm, int direction,
 690                             struct sof_ipc_stream_posn *posn)
 691 {
 692         struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
 693         struct sof_ipc_stream stream;
 694         int err;
 695
 696         /* read position via slower IPC */
 697         stream.hdr.size = sizeof(stream);
 698         stream.hdr.cmd = SOF_IPC_GLB_STREAM_MSG | SOF_IPC_STREAM_POSITION;
 699         stream.comp_id = spcm->stream[direction].comp_id;
 700
 701         /* send IPC to the DSP */
 702         err = sof_ipc_tx_message(sdev->ipc,
 703                                  stream.hdr.cmd, &stream, sizeof(stream), posn,
 704                                  sizeof(*posn));
 705         if (err < 0) {
 706                 dev_err(sdev->dev, "error: failed to get stream %d position\n",
 707                         stream.comp_id);
 708                 return err;
 709         }
 710
 711         return 0;
 712 }
 713 EXPORT_SYMBOL(snd_sof_ipc_stream_posn);
 714
 715 static int sof_get_ctrl_copy_params(enum sof_ipc_ctrl_type ctrl_type,
 716                                     struct sof_ipc_ctrl_data *src,
 717                                     struct sof_ipc_ctrl_data *dst,
 718                                     struct sof_ipc_ctrl_data_params *sparams)
 719 {
 720         switch (ctrl_type) {
 721         case SOF_CTRL_TYPE_VALUE_CHAN_GET:
 722         case SOF_CTRL_TYPE_VALUE_CHAN_SET:
 723                 sparams->src = (u8 *)src->chanv;
 724                 sparams->dst = (u8 *)dst->chanv;
 725                 break;
 726         case SOF_CTRL_TYPE_DATA_GET:
 727         case SOF_CTRL_TYPE_DATA_SET:
 728                 sparams->src = (u8 *)src->data->data;
 729                 sparams->dst = (u8 *)dst->data->data;
 730                 break;
 731         default:
 732                 return -EINVAL;
 733         }
 734
 735         /* calculate payload size and number of messages */
 736         sparams->pl_size = SOF_IPC_MSG_MAX_SIZE - sparams->hdr_bytes;
 737         sparams->num_msg = DIV_ROUND_UP(sparams->msg_bytes, sparams->pl_size);
 738
 739         return 0;
 740 }
 741
 742 static int sof_set_get_large_ctrl_data(struct snd_sof_dev *sdev,
 743                                        struct sof_ipc_ctrl_data *cdata,
 744                                        struct sof_ipc_ctrl_data_params *sparams,
 745                                        bool set)
 746 {
 747         struct sof_ipc_ctrl_data *partdata;
 748         size_t send_bytes;
 749         size_t offset = 0;
 750         size_t msg_bytes;
 751         size_t pl_size;
 752         int err;
 753         int i;
 754
 755         /* allocate max ipc size because we have at least one */
 756         partdata = kzalloc(SOF_IPC_MSG_MAX_SIZE, GFP_KERNEL);
 757         if (!partdata)
 758                 return -ENOMEM;
 759
 760         if (set)
 761                 err = sof_get_ctrl_copy_params(cdata->type, cdata, partdata,
 762                                                sparams);
 763         else
 764                 err = sof_get_ctrl_copy_params(cdata->type, partdata, cdata,
 765                                                sparams);
 766         if (err < 0) {
 767                 kfree(partdata);
 768                 return err;
 769         }
 770
 771         msg_bytes = sparams->msg_bytes;
 772         pl_size = sparams->pl_size;
 773
 774         /* copy the header data */
 775         memcpy(partdata, cdata, sparams->hdr_bytes);
 776
 777         /* Serialise IPC TX */
 778         mutex_lock(&sdev->ipc->tx_mutex);
 779
 780         /* copy the payload data in a loop */
 781         for (i = 0; i < sparams->num_msg; i++) {
 782                 send_bytes = min(msg_bytes, pl_size);
 783                 partdata->num_elems = send_bytes;
 784                 partdata->rhdr.hdr.size = sparams->hdr_bytes + send_bytes;
 785                 partdata->msg_index = i;
 786                 msg_bytes -= send_bytes;
 787                 partdata->elems_remaining = msg_bytes;
 788
 789                 if (set)
 790                         memcpy(sparams->dst, sparams->src + offset, send_bytes);
 791
 792                 err = sof_ipc_tx_message_unlocked(sdev->ipc,
 793                                                   partdata,
 794                                                   partdata->rhdr.hdr.size,
 795                                                   partdata,
 796                                                   partdata->rhdr.hdr.size);
 797                 if (err < 0)
 798                         break;
 799
 800                 if (!set)
 801                         memcpy(sparams->dst + offset, sparams->src, send_bytes);
 802
 803                 offset += pl_size;
 804         }
 805
 806         mutex_unlock(&sdev->ipc->tx_mutex);
 807
 808         kfree(partdata);
 809         return err;
 810 }
 811
 812 /*
 813  * IPC get()/set() for kcontrols.
 814  */
 815 int snd_sof_ipc_set_get_comp_data(struct snd_sof_control *scontrol, bool set)
 816 {
 817         struct snd_soc_component *scomp = scontrol->scomp;
 818         struct sof_ipc_ctrl_data *cdata = scontrol->ipc_control_data;
 819         struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
 820         struct sof_ipc_fw_ready *ready = &sdev->fw_ready;
 821         struct sof_ipc_fw_version *v = &ready->version;
 822         struct sof_ipc_ctrl_data_params sparams;
 823         enum sof_ipc_ctrl_type ctrl_type;
 824         struct snd_sof_widget *swidget;
 825         bool widget_found = false;
 826         size_t send_bytes;
 827         u32 ipc_cmd;
 828         int err;
 829
 830         list_for_each_entry(swidget, &sdev->widget_list, list) {
 831                 if (swidget->comp_id == scontrol->comp_id) {
 832                         widget_found = true;
 833                         break;
 834                 }
 835         }
 836
 837         if (!widget_found) {
 838                 dev_err(sdev->dev, "error: can't find widget with id %d\n", scontrol->comp_id);
 839                 return -EINVAL;
 840         }
 841
 842         /*
 843          * Volatile controls should always be part of static pipelines and the widget use_count
 844          * would always be > 0 in this case. For the others, just return the cached value if the
 845          * widget is not set up.
 846          */
 847         if (!swidget->use_count)
 848                 return 0;
 849
 850         /* read or write firmware volume */
 851         if (scontrol->readback_offset != 0) {
 852                 /* write/read value header via mmaped region */
 853                 send_bytes = sizeof(struct sof_ipc_ctrl_value_chan) *
 854                 cdata->num_elems;
 855                 if (set)
 856                         err = snd_sof_dsp_block_write(sdev, SOF_FW_BLK_TYPE_IRAM,
 857                                                       scontrol->readback_offset,
 858                                                       cdata->chanv, send_bytes);
 859
 860                 else
 861                         err = snd_sof_dsp_block_read(sdev, SOF_FW_BLK_TYPE_IRAM,
 862                                                      scontrol->readback_offset,
 863                                                      cdata->chanv, send_bytes);
 864
 865                 if (err)
 866                         dev_err_once(sdev->dev, "error: %s TYPE_IRAM failed\n",
 867                                      set ? "write to" :  "read from");
 868                 return err;
 869         }
 870
 871         /*
 872          * Select the IPC cmd and the ctrl_type based on the ctrl_cmd and the
 873          * direction
 874          * Note: SOF_CTRL_TYPE_VALUE_COMP_* is not used and supported currently
 875          *       for ctrl_type
 876          */
 877         if (cdata->cmd == SOF_CTRL_CMD_BINARY) {
 878                 ipc_cmd = set ? SOF_IPC_COMP_SET_DATA : SOF_IPC_COMP_GET_DATA;
 879                 ctrl_type = set ? SOF_CTRL_TYPE_DATA_SET : SOF_CTRL_TYPE_DATA_GET;
 880         } else {
 881                 ipc_cmd = set ? SOF_IPC_COMP_SET_VALUE : SOF_IPC_COMP_GET_VALUE;
 882                 ctrl_type = set ? SOF_CTRL_TYPE_VALUE_CHAN_SET : SOF_CTRL_TYPE_VALUE_CHAN_GET;
 883         }
 884
 885         cdata->rhdr.hdr.cmd = SOF_IPC_GLB_COMP_MSG | ipc_cmd;
 886         cdata->type = ctrl_type;
 887         cdata->comp_id = scontrol->comp_id;
 888         cdata->msg_index = 0;
 889
 890         /* calculate header and data size */
 891         switch (cdata->type) {
 892         case SOF_CTRL_TYPE_VALUE_CHAN_GET:
 893         case SOF_CTRL_TYPE_VALUE_CHAN_SET:
 894                 sparams.msg_bytes = scontrol->num_channels *
 895                         sizeof(struct sof_ipc_ctrl_value_chan);
 896                 sparams.hdr_bytes = sizeof(struct sof_ipc_ctrl_data);
 897                 sparams.elems = scontrol->num_channels;
 898                 break;
 899         case SOF_CTRL_TYPE_DATA_GET:
 900         case SOF_CTRL_TYPE_DATA_SET:
 901                 sparams.msg_bytes = cdata->data->size;
 902                 sparams.hdr_bytes = sizeof(struct sof_ipc_ctrl_data) +
 903                         sizeof(struct sof_abi_hdr);
 904                 sparams.elems = cdata->data->size;
 905                 break;
 906         default:
 907                 return -EINVAL;
 908         }
 909
 910         cdata->rhdr.hdr.size = sparams.msg_bytes + sparams.hdr_bytes;
 911         cdata->num_elems = sparams.elems;
 912         cdata->elems_remaining = 0;
 913
 914         /* send normal size ipc in one part */
 915         if (cdata->rhdr.hdr.size <= SOF_IPC_MSG_MAX_SIZE) {
 916                 err = sof_ipc_tx_message(sdev->ipc, cdata->rhdr.hdr.cmd, cdata,
 917                                          cdata->rhdr.hdr.size, cdata,
 918                                          cdata->rhdr.hdr.size);
 919
 920                 if (err < 0)
 921                         dev_err(sdev->dev, "error: set/get ctrl ipc comp %d\n",
 922                                 cdata->comp_id);
 923
 924                 return err;
 925         }
 926
 927         /* data is bigger than max ipc size, chop into smaller pieces */
 928         dev_dbg(sdev->dev, "large ipc size %u, control size %u\n",
 929                 cdata->rhdr.hdr.size, scontrol->size);
 930
 931         /* large messages is only supported from ABI 3.3.0 onwards */
 932         if (v->abi_version < SOF_ABI_VER(3, 3, 0)) {
 933                 dev_err(sdev->dev, "error: incompatible FW ABI version\n");
 934                 return -EINVAL;
 935         }
 936
 937         err = sof_set_get_large_ctrl_data(sdev, cdata, &sparams, set);
 938
 939         if (err < 0)
 940                 dev_err(sdev->dev, "error: set/get large ctrl ipc comp %d\n",
 941                         cdata->comp_id);
 942
 943         return err;
 944 }
 945 EXPORT_SYMBOL(snd_sof_ipc_set_get_comp_data);
 946
 947 int snd_sof_ipc_valid(struct snd_sof_dev *sdev)
 948 {
 949         struct sof_ipc_fw_ready *ready = &sdev->fw_ready;
 950         struct sof_ipc_fw_version *v = &ready->version;
 951
 952         dev_info(sdev->dev,
 953                  "Firmware info: version %d:%d:%d-%s\n",  v->major, v->minor,
 954                  v->micro, v->tag);
 955         dev_info(sdev->dev,
 956                  "Firmware: ABI %d:%d:%d Kernel ABI %d:%d:%d\n",
 957                  SOF_ABI_VERSION_MAJOR(v->abi_version),
 958                  SOF_ABI_VERSION_MINOR(v->abi_version),
 959                  SOF_ABI_VERSION_PATCH(v->abi_version),
 960                  SOF_ABI_MAJOR, SOF_ABI_MINOR, SOF_ABI_PATCH);
 961
 962         if (SOF_ABI_VERSION_INCOMPATIBLE(SOF_ABI_VERSION, v->abi_version)) {
 963                 dev_err(sdev->dev, "error: incompatible FW ABI version\n");
 964                 return -EINVAL;
 965         }
 966
 967         if (SOF_ABI_VERSION_MINOR(v->abi_version) > SOF_ABI_MINOR) {
 968                 if (!IS_ENABLED(CONFIG_SND_SOC_SOF_STRICT_ABI_CHECKS)) {
 969                         dev_warn(sdev->dev, "warn: FW ABI is more recent than kernel\n");
 970                 } else {
 971                         dev_err(sdev->dev, "error: FW ABI is more recent than kernel\n");
 972                         return -EINVAL;
 973                 }
 974         }
 975
 976         if (ready->flags & SOF_IPC_INFO_BUILD) {
 977                 dev_info(sdev->dev,
 978                          "Firmware debug build %d on %s-%s - options:\n"
 979                          " GDB: %s\n"
 980                          " lock debug: %s\n"
 981                          " lock vdebug: %s\n",
 982                          v->build, v->date, v->time,
 983                          (ready->flags & SOF_IPC_INFO_GDB) ?
 984                                 "enabled" : "disabled",
 985                          (ready->flags & SOF_IPC_INFO_LOCKS) ?
 986                                 "enabled" : "disabled",
 987                          (ready->flags & SOF_IPC_INFO_LOCKSV) ?
 988                                 "enabled" : "disabled");
 989         }
 990
 991         /* copy the fw_version into debugfs at first boot */
 992         memcpy(&sdev->fw_version, v, sizeof(*v));
 993
 994         return 0;
 995 }
 996 EXPORT_SYMBOL(snd_sof_ipc_valid);
 997
 998 int sof_ipc_init_msg_memory(struct snd_sof_dev *sdev)
 999 {
1000         struct snd_sof_ipc_msg *msg;
1001
1002         msg = &sdev->ipc->msg;
1003
1004         msg->reply_data = devm_kzalloc(sdev->dev, SOF_IPC_MSG_MAX_SIZE, GFP_KERNEL);
1005         if (!msg->reply_data)
1006                 return -ENOMEM;
1007
1008         return 0;
1009 }
1010
1011 struct snd_sof_ipc *snd_sof_ipc_init(struct snd_sof_dev *sdev)
1012 {
1013         struct snd_sof_ipc *ipc;
1014         struct snd_sof_ipc_msg *msg;
1015
1016         ipc = devm_kzalloc(sdev->dev, sizeof(*ipc), GFP_KERNEL);
1017         if (!ipc)
1018                 return NULL;
1019
1020         mutex_init(&ipc->tx_mutex);
1021         ipc->sdev = sdev;
1022         msg = &ipc->msg;
1023
1024         /* indicate that we aren't sending a message ATM */
1025         msg->ipc_complete = true;
1026
1027         init_waitqueue_head(&msg->waitq);
1028
1029         /*
1030          * Use IPC3 ops as it is the only available version now. With the addition of new IPC
1031          * versions, this will need to be modified to use the selected version at runtime.
1032          */
1033         ipc->ops = &ipc3_ops;
1034
1035         /* check for mandatory ops */
1036         if (!ipc->ops->tplg || !ipc->ops->tplg->widget || !ipc->ops->tplg->control) {
1037                 dev_err(sdev->dev, "Invalid topology IPC ops\n");
1038                 return NULL;
1039         }
1040
1041         return ipc;
1042 }
1043 EXPORT_SYMBOL(snd_sof_ipc_init);
1044
1045 void snd_sof_ipc_free(struct snd_sof_dev *sdev)
1046 {
1047         struct snd_sof_ipc *ipc = sdev->ipc;
1048
1049         if (!ipc)
1050                 return;
1051
1052         /* disable sending of ipc's */
1053         mutex_lock(&ipc->tx_mutex);
1054         ipc->disable_ipc_tx = true;
1055         mutex_unlock(&ipc->tx_mutex);
1056 }
1057 EXPORT_SYMBOL(snd_sof_ipc_free);