drivers/rpmsg/qcom_glink_native.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2016-2017, Linaro Ltd
   4  */
   5
   6 #include <linux/idr.h>
   7 #include <linux/interrupt.h>
   8 #include <linux/io.h>
   9 #include <linux/list.h>
  10 #include <linux/mfd/syscon.h>
  11 #include <linux/module.h>
  12 #include <linux/of.h>
  13 #include <linux/of_address.h>
  14 #include <linux/platform_device.h>
  15 #include <linux/regmap.h>
  16 #include <linux/rpmsg.h>
  17 #include <linux/sizes.h>
  18 #include <linux/slab.h>
  19 #include <linux/workqueue.h>
  20 #include <linux/mailbox_client.h>
  21
  22 #include "rpmsg_internal.h"
  23 #include "qcom_glink_native.h"
  24
  25 #define GLINK_NAME_SIZE         32
  26 #define GLINK_VERSION_1         1
  27
  28 #define RPM_GLINK_CID_MIN       1
  29 #define RPM_GLINK_CID_MAX       65536
  30
  31 struct glink_msg {
  32         __le16 cmd;
  33         __le16 param1;
  34         __le32 param2;
  35         u8 data[];
  36 } __packed;
  37
  38 /**
  39  * struct glink_defer_cmd - deferred incoming control message
  40  * @node:       list node
  41  * @msg:        message header
  42  * @data:       payload of the message
  43  *
  44  * Copy of a received control message, to be added to @rx_queue and processed
  45  * by @rx_work of @qcom_glink.
  46  */
  47 struct glink_defer_cmd {
  48         struct list_head node;
  49
  50         struct glink_msg msg;
  51         u8 data[];
  52 };
  53
  54 /**
  55  * struct glink_core_rx_intent - RX intent
  56  * RX intent
  57  *
  58  * @data: pointer to the data (may be NULL for zero-copy)
  59  * @id: remote or local intent ID
  60  * @size: size of the original intent (do not modify)
  61  * @reuse: To mark if the intent can be reused after first use
  62  * @in_use: To mark if intent is already in use for the channel
  63  * @offset: next write offset (initially 0)
  64  * @node:       list node
  65  */
  66 struct glink_core_rx_intent {
  67         void *data;
  68         u32 id;
  69         size_t size;
  70         bool reuse;
  71         bool in_use;
  72         u32 offset;
  73
  74         struct list_head node;
  75 };
  76
  77 /**
  78  * struct qcom_glink - driver context, relates to one remote subsystem
  79  * @dev:        reference to the associated struct device
  80  * @rx_pipe:    pipe object for receive FIFO
  81  * @tx_pipe:    pipe object for transmit FIFO
  82  * @rx_work:    worker for handling received control messages
  83  * @rx_lock:    protects the @rx_queue
  84  * @rx_queue:   queue of received control messages to be processed in @rx_work
  85  * @tx_lock:    synchronizes operations on the tx fifo
  86  * @idr_lock:   synchronizes @lcids and @rcids modifications
  87  * @lcids:      idr of all channels with a known local channel id
  88  * @rcids:      idr of all channels with a known remote channel id
  89  * @features:   remote features
  90  * @intentless: flag to indicate that there is no intent
  91  * @tx_avail_notify: Waitqueue for pending tx tasks
  92  * @sent_read_notify: flag to check cmd sent or not
  93  */
  94 struct qcom_glink {
  95         struct device *dev;
  96
  97         struct qcom_glink_pipe *rx_pipe;
  98         struct qcom_glink_pipe *tx_pipe;
  99
 100         struct work_struct rx_work;
 101         spinlock_t rx_lock;
 102         struct list_head rx_queue;
 103
 104         spinlock_t tx_lock;
 105
 106         spinlock_t idr_lock;
 107         struct idr lcids;
 108         struct idr rcids;
 109         unsigned long features;
 110
 111         bool intentless;
 112         wait_queue_head_t tx_avail_notify;
 113         bool sent_read_notify;
 114 };
 115
 116 enum {
 117         GLINK_STATE_CLOSED,
 118         GLINK_STATE_OPENING,
 119         GLINK_STATE_OPEN,
 120         GLINK_STATE_CLOSING,
 121 };
 122
 123 /**
 124  * struct glink_channel - internal representation of a channel
 125  * @rpdev:      rpdev reference, only used for primary endpoints
 126  * @ept:        rpmsg endpoint this channel is associated with
 127  * @glink:      qcom_glink context handle
 128  * @refcount:   refcount for the channel object
 129  * @recv_lock:  guard for @ept.cb
 130  * @name:       unique channel name/identifier
 131  * @lcid:       channel id, in local space
 132  * @rcid:       channel id, in remote space
 133  * @intent_lock: lock for protection of @liids, @riids
 134  * @liids:      idr of all local intents
 135  * @riids:      idr of all remote intents
 136  * @intent_work: worker responsible for transmitting rx_done packets
 137  * @done_intents: list of intents that needs to be announced rx_done
 138  * @buf:        receive buffer, for gathering fragments
 139  * @buf_offset: write offset in @buf
 140  * @buf_size:   size of current @buf
 141  * @open_ack:   completed once remote has acked the open-request
 142  * @open_req:   completed once open-request has been received
 143  * @intent_req_lock: Synchronises multiple intent requests
 144  * @intent_req_result: Result of intent request
 145  * @intent_req_comp: Completion for intent_req signalling
 146  */
 147 struct glink_channel {
 148         struct rpmsg_endpoint ept;
 149
 150         struct rpmsg_device *rpdev;
 151         struct qcom_glink *glink;
 152
 153         struct kref refcount;
 154
 155         spinlock_t recv_lock;
 156
 157         char *name;
 158         unsigned int lcid;
 159         unsigned int rcid;
 160
 161         spinlock_t intent_lock;
 162         struct idr liids;
 163         struct idr riids;
 164         struct work_struct intent_work;
 165         struct list_head done_intents;
 166
 167         struct glink_core_rx_intent *buf;
 168         int buf_offset;
 169         int buf_size;
 170
 171         struct completion open_ack;
 172         struct completion open_req;
 173
 174         struct mutex intent_req_lock;
 175         bool intent_req_result;
 176         struct completion intent_req_comp;
 177 };
 178
 179 #define to_glink_channel(_ept) container_of(_ept, struct glink_channel, ept)
 180
 181 static const struct rpmsg_endpoint_ops glink_endpoint_ops;
 182
 183 #define RPM_CMD_VERSION                 0
 184 #define RPM_CMD_VERSION_ACK             1
 185 #define RPM_CMD_OPEN                    2
 186 #define RPM_CMD_CLOSE                   3
 187 #define RPM_CMD_OPEN_ACK                4
 188 #define RPM_CMD_INTENT                  5
 189 #define RPM_CMD_RX_DONE                 6
 190 #define RPM_CMD_RX_INTENT_REQ           7
 191 #define RPM_CMD_RX_INTENT_REQ_ACK       8
 192 #define RPM_CMD_TX_DATA                 9
 193 #define RPM_CMD_CLOSE_ACK               11
 194 #define RPM_CMD_TX_DATA_CONT            12
 195 #define RPM_CMD_READ_NOTIF              13
 196 #define RPM_CMD_RX_DONE_W_REUSE         14
 197
 198 #define GLINK_FEATURE_INTENTLESS        BIT(1)
 199
 200 static void qcom_glink_rx_done_work(struct work_struct *work);
 201
 202 static struct glink_channel *qcom_glink_alloc_channel(struct qcom_glink *glink,
 203                                                       const char *name)
 204 {
 205         struct glink_channel *channel;
 206
 207         channel = kzalloc(sizeof(*channel), GFP_KERNEL);
 208         if (!channel)
 209                 return ERR_PTR(-ENOMEM);
 210
 211         /* Setup glink internal glink_channel data */
 212         spin_lock_init(&channel->recv_lock);
 213         spin_lock_init(&channel->intent_lock);
 214         mutex_init(&channel->intent_req_lock);
 215
 216         channel->glink = glink;
 217         channel->name = kstrdup(name, GFP_KERNEL);
 218
 219         init_completion(&channel->open_req);
 220         init_completion(&channel->open_ack);
 221         init_completion(&channel->intent_req_comp);
 222
 223         INIT_LIST_HEAD(&channel->done_intents);
 224         INIT_WORK(&channel->intent_work, qcom_glink_rx_done_work);
 225
 226         idr_init(&channel->liids);
 227         idr_init(&channel->riids);
 228         kref_init(&channel->refcount);
 229
 230         return channel;
 231 }
 232
 233 static void qcom_glink_channel_release(struct kref *ref)
 234 {
 235         struct glink_channel *channel = container_of(ref, struct glink_channel,
 236                                                      refcount);
 237         struct glink_core_rx_intent *intent;
 238         struct glink_core_rx_intent *tmp;
 239         unsigned long flags;
 240         int iid;
 241
 242         /* cancel pending rx_done work */
 243         cancel_work_sync(&channel->intent_work);
 244
 245         spin_lock_irqsave(&channel->intent_lock, flags);
 246         /* Free all non-reuse intents pending rx_done work */
 247         list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) {
 248                 if (!intent->reuse) {
 249                         kfree(intent->data);
 250                         kfree(intent);
 251                 }
 252         }
 253
 254         idr_for_each_entry(&channel->liids, tmp, iid) {
 255                 kfree(tmp->data);
 256                 kfree(tmp);
 257         }
 258         idr_destroy(&channel->liids);
 259
 260         idr_for_each_entry(&channel->riids, tmp, iid)
 261                 kfree(tmp);
 262         idr_destroy(&channel->riids);
 263         spin_unlock_irqrestore(&channel->intent_lock, flags);
 264
 265         kfree(channel->name);
 266         kfree(channel);
 267 }
 268
 269 static size_t qcom_glink_rx_avail(struct qcom_glink *glink)
 270 {
 271         return glink->rx_pipe->avail(glink->rx_pipe);
 272 }
 273
 274 static void qcom_glink_rx_peak(struct qcom_glink *glink,
 275                                void *data, unsigned int offset, size_t count)
 276 {
 277         glink->rx_pipe->peak(glink->rx_pipe, data, offset, count);
 278 }
 279
 280 static void qcom_glink_rx_advance(struct qcom_glink *glink, size_t count)
 281 {
 282         glink->rx_pipe->advance(glink->rx_pipe, count);
 283 }
 284
 285 static size_t qcom_glink_tx_avail(struct qcom_glink *glink)
 286 {
 287         return glink->tx_pipe->avail(glink->tx_pipe);
 288 }
 289
 290 static void qcom_glink_tx_write(struct qcom_glink *glink,
 291                                 const void *hdr, size_t hlen,
 292                                 const void *data, size_t dlen)
 293 {
 294         glink->tx_pipe->write(glink->tx_pipe, hdr, hlen, data, dlen);
 295 }
 296
 297 static void qcom_glink_tx_kick(struct qcom_glink *glink)
 298 {
 299         glink->tx_pipe->kick(glink->tx_pipe);
 300 }
 301
 302 static void qcom_glink_send_read_notify(struct qcom_glink *glink)
 303 {
 304         struct glink_msg msg;
 305
 306         msg.cmd = cpu_to_le16(RPM_CMD_READ_NOTIF);
 307         msg.param1 = 0;
 308         msg.param2 = 0;
 309
 310         qcom_glink_tx_write(glink, &msg, sizeof(msg), NULL, 0);
 311
 312         qcom_glink_tx_kick(glink);
 313 }
 314
 315 static int qcom_glink_tx(struct qcom_glink *glink,
 316                          const void *hdr, size_t hlen,
 317                          const void *data, size_t dlen, bool wait)
 318 {
 319         unsigned int tlen = hlen + dlen;
 320         unsigned long flags;
 321         int ret = 0;
 322
 323         /* Reject packets that are too big */
 324         if (tlen >= glink->tx_pipe->length)
 325                 return -EINVAL;
 326
 327         spin_lock_irqsave(&glink->tx_lock, flags);
 328
 329         while (qcom_glink_tx_avail(glink) < tlen) {
 330                 if (!wait) {
 331                         ret = -EAGAIN;
 332                         goto out;
 333                 }
 334
 335                 if (!glink->sent_read_notify) {
 336                         glink->sent_read_notify = true;
 337                         qcom_glink_send_read_notify(glink);
 338                 }
 339
 340                 /* Wait without holding the tx_lock */
 341                 spin_unlock_irqrestore(&glink->tx_lock, flags);
 342
 343                 wait_event_timeout(glink->tx_avail_notify,
 344                                    qcom_glink_tx_avail(glink) >= tlen, 10 * HZ);
 345
 346                 spin_lock_irqsave(&glink->tx_lock, flags);
 347
 348                 if (qcom_glink_tx_avail(glink) >= tlen)
 349                         glink->sent_read_notify = false;
 350         }
 351
 352         qcom_glink_tx_write(glink, hdr, hlen, data, dlen);
 353         qcom_glink_tx_kick(glink);
 354
 355 out:
 356         spin_unlock_irqrestore(&glink->tx_lock, flags);
 357
 358         return ret;
 359 }
 360
 361 static int qcom_glink_send_version(struct qcom_glink *glink)
 362 {
 363         struct glink_msg msg;
 364
 365         msg.cmd = cpu_to_le16(RPM_CMD_VERSION);
 366         msg.param1 = cpu_to_le16(GLINK_VERSION_1);
 367         msg.param2 = cpu_to_le32(glink->features);
 368
 369         return qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
 370 }
 371
 372 static void qcom_glink_send_version_ack(struct qcom_glink *glink)
 373 {
 374         struct glink_msg msg;
 375
 376         msg.cmd = cpu_to_le16(RPM_CMD_VERSION_ACK);
 377         msg.param1 = cpu_to_le16(GLINK_VERSION_1);
 378         msg.param2 = cpu_to_le32(glink->features);
 379
 380         qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
 381 }
 382
 383 static void qcom_glink_send_open_ack(struct qcom_glink *glink,
 384                                      struct glink_channel *channel)
 385 {
 386         struct glink_msg msg;
 387
 388         msg.cmd = cpu_to_le16(RPM_CMD_OPEN_ACK);
 389         msg.param1 = cpu_to_le16(channel->rcid);
 390         msg.param2 = cpu_to_le32(0);
 391
 392         qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
 393 }
 394
 395 static void qcom_glink_handle_intent_req_ack(struct qcom_glink *glink,
 396                                              unsigned int cid, bool granted)
 397 {
 398         struct glink_channel *channel;
 399         unsigned long flags;
 400
 401         spin_lock_irqsave(&glink->idr_lock, flags);
 402         channel = idr_find(&glink->rcids, cid);
 403         spin_unlock_irqrestore(&glink->idr_lock, flags);
 404         if (!channel) {
 405                 dev_err(glink->dev, "unable to find channel\n");
 406                 return;
 407         }
 408
 409         channel->intent_req_result = granted;
 410         complete(&channel->intent_req_comp);
 411 }
 412
 413 /**
 414  * qcom_glink_send_open_req() - send a RPM_CMD_OPEN request to the remote
 415  * @glink: Ptr to the glink edge
 416  * @channel: Ptr to the channel that the open req is sent
 417  *
 418  * Allocates a local channel id and sends a RPM_CMD_OPEN message to the remote.
 419  * Will return with refcount held, regardless of outcome.
 420  *
 421  * Return: 0 on success, negative errno otherwise.
 422  */
 423 static int qcom_glink_send_open_req(struct qcom_glink *glink,
 424                                     struct glink_channel *channel)
 425 {
 426         struct {
 427                 struct glink_msg msg;
 428                 u8 name[GLINK_NAME_SIZE];
 429         } __packed req;
 430         int name_len = strlen(channel->name) + 1;
 431         int req_len = ALIGN(sizeof(req.msg) + name_len, 8);
 432         int ret;
 433         unsigned long flags;
 434
 435         kref_get(&channel->refcount);
 436
 437         spin_lock_irqsave(&glink->idr_lock, flags);
 438         ret = idr_alloc_cyclic(&glink->lcids, channel,
 439                                RPM_GLINK_CID_MIN, RPM_GLINK_CID_MAX,
 440                                GFP_ATOMIC);
 441         spin_unlock_irqrestore(&glink->idr_lock, flags);
 442         if (ret < 0)
 443                 return ret;
 444
 445         channel->lcid = ret;
 446
 447         req.msg.cmd = cpu_to_le16(RPM_CMD_OPEN);
 448         req.msg.param1 = cpu_to_le16(channel->lcid);
 449         req.msg.param2 = cpu_to_le32(name_len);
 450         strcpy(req.name, channel->name);
 451
 452         ret = qcom_glink_tx(glink, &req, req_len, NULL, 0, true);
 453         if (ret)
 454                 goto remove_idr;
 455
 456         return 0;
 457
 458 remove_idr:
 459         spin_lock_irqsave(&glink->idr_lock, flags);
 460         idr_remove(&glink->lcids, channel->lcid);
 461         channel->lcid = 0;
 462         spin_unlock_irqrestore(&glink->idr_lock, flags);
 463
 464         return ret;
 465 }
 466
 467 static void qcom_glink_send_close_req(struct qcom_glink *glink,
 468                                       struct glink_channel *channel)
 469 {
 470         struct glink_msg req;
 471
 472         req.cmd = cpu_to_le16(RPM_CMD_CLOSE);
 473         req.param1 = cpu_to_le16(channel->lcid);
 474         req.param2 = 0;
 475
 476         qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true);
 477 }
 478
 479 static void qcom_glink_send_close_ack(struct qcom_glink *glink,
 480                                       unsigned int rcid)
 481 {
 482         struct glink_msg req;
 483
 484         req.cmd = cpu_to_le16(RPM_CMD_CLOSE_ACK);
 485         req.param1 = cpu_to_le16(rcid);
 486         req.param2 = 0;
 487
 488         qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true);
 489 }
 490
 491 static void qcom_glink_rx_done_work(struct work_struct *work)
 492 {
 493         struct glink_channel *channel = container_of(work, struct glink_channel,
 494                                                      intent_work);
 495         struct qcom_glink *glink = channel->glink;
 496         struct glink_core_rx_intent *intent, *tmp;
 497         struct {
 498                 u16 id;
 499                 u16 lcid;
 500                 u32 liid;
 501         } __packed cmd;
 502
 503         unsigned int cid = channel->lcid;
 504         unsigned int iid;
 505         bool reuse;
 506         unsigned long flags;
 507
 508         spin_lock_irqsave(&channel->intent_lock, flags);
 509         list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) {
 510                 list_del(&intent->node);
 511                 spin_unlock_irqrestore(&channel->intent_lock, flags);
 512                 iid = intent->id;
 513                 reuse = intent->reuse;
 514
 515                 cmd.id = reuse ? RPM_CMD_RX_DONE_W_REUSE : RPM_CMD_RX_DONE;
 516                 cmd.lcid = cid;
 517                 cmd.liid = iid;
 518
 519                 qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true);
 520                 if (!reuse) {
 521                         kfree(intent->data);
 522                         kfree(intent);
 523                 }
 524                 spin_lock_irqsave(&channel->intent_lock, flags);
 525         }
 526         spin_unlock_irqrestore(&channel->intent_lock, flags);
 527 }
 528
 529 static void qcom_glink_rx_done(struct qcom_glink *glink,
 530                                struct glink_channel *channel,
 531                                struct glink_core_rx_intent *intent)
 532 {
 533         /* We don't send RX_DONE to intentless systems */
 534         if (glink->intentless) {
 535                 kfree(intent->data);
 536                 kfree(intent);
 537                 return;
 538         }
 539
 540         /* Take it off the tree of receive intents */
 541         if (!intent->reuse) {
 542                 spin_lock(&channel->intent_lock);
 543                 idr_remove(&channel->liids, intent->id);
 544                 spin_unlock(&channel->intent_lock);
 545         }
 546
 547         /* Schedule the sending of a rx_done indication */
 548         spin_lock(&channel->intent_lock);
 549         list_add_tail(&intent->node, &channel->done_intents);
 550         spin_unlock(&channel->intent_lock);
 551
 552         schedule_work(&channel->intent_work);
 553 }
 554
 555 /**
 556  * qcom_glink_receive_version() - receive version/features from remote system
 557  *
 558  * @glink:      pointer to transport interface
 559  * @version:    remote version
 560  * @features:   remote features
 561  *
 562  * This function is called in response to a remote-initiated version/feature
 563  * negotiation sequence.
 564  */
 565 static void qcom_glink_receive_version(struct qcom_glink *glink,
 566                                        u32 version,
 567                                        u32 features)
 568 {
 569         switch (version) {
 570         case 0:
 571                 break;
 572         case GLINK_VERSION_1:
 573                 glink->features &= features;
 574                 fallthrough;
 575         default:
 576                 qcom_glink_send_version_ack(glink);
 577                 break;
 578         }
 579 }
 580
 581 /**
 582  * qcom_glink_receive_version_ack() - receive negotiation ack from remote system
 583  *
 584  * @glink:      pointer to transport interface
 585  * @version:    remote version response
 586  * @features:   remote features response
 587  *
 588  * This function is called in response to a local-initiated version/feature
 589  * negotiation sequence and is the counter-offer from the remote side based
 590  * upon the initial version and feature set requested.
 591  */
 592 static void qcom_glink_receive_version_ack(struct qcom_glink *glink,
 593                                            u32 version,
 594                                            u32 features)
 595 {
 596         switch (version) {
 597         case 0:
 598                 /* Version negotiation failed */
 599                 break;
 600         case GLINK_VERSION_1:
 601                 if (features == glink->features)
 602                         break;
 603
 604                 glink->features &= features;
 605                 fallthrough;
 606         default:
 607                 qcom_glink_send_version(glink);
 608                 break;
 609         }
 610 }
 611
 612 /**
 613  * qcom_glink_send_intent_req_ack() - convert an rx intent request ack cmd to
 614  *      wire format and transmit
 615  * @glink:      The transport to transmit on.
 616  * @channel:    The glink channel
 617  * @granted:    The request response to encode.
 618  *
 619  * Return: 0 on success or standard Linux error code.
 620  */
 621 static int qcom_glink_send_intent_req_ack(struct qcom_glink *glink,
 622                                           struct glink_channel *channel,
 623                                           bool granted)
 624 {
 625         struct glink_msg msg;
 626
 627         msg.cmd = cpu_to_le16(RPM_CMD_RX_INTENT_REQ_ACK);
 628         msg.param1 = cpu_to_le16(channel->lcid);
 629         msg.param2 = cpu_to_le32(granted);
 630
 631         qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
 632
 633         return 0;
 634 }
 635
 636 /**
 637  * qcom_glink_advertise_intent - convert an rx intent cmd to wire format and
 638  *                         transmit
 639  * @glink:      The transport to transmit on.
 640  * @channel:    The local channel
 641  * @intent:     The intent to pass on to remote.
 642  *
 643  * Return: 0 on success or standard Linux error code.
 644  */
 645 static int qcom_glink_advertise_intent(struct qcom_glink *glink,
 646                                        struct glink_channel *channel,
 647                                        struct glink_core_rx_intent *intent)
 648 {
 649         struct command {
 650                 __le16 id;
 651                 __le16 lcid;
 652                 __le32 count;
 653                 __le32 size;
 654                 __le32 liid;
 655         } __packed;
 656         struct command cmd;
 657
 658         cmd.id = cpu_to_le16(RPM_CMD_INTENT);
 659         cmd.lcid = cpu_to_le16(channel->lcid);
 660         cmd.count = cpu_to_le32(1);
 661         cmd.size = cpu_to_le32(intent->size);
 662         cmd.liid = cpu_to_le32(intent->id);
 663
 664         qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true);
 665
 666         return 0;
 667 }
 668
 669 static struct glink_core_rx_intent *
 670 qcom_glink_alloc_intent(struct qcom_glink *glink,
 671                         struct glink_channel *channel,
 672                         size_t size,
 673                         bool reuseable)
 674 {
 675         struct glink_core_rx_intent *intent;
 676         int ret;
 677         unsigned long flags;
 678
 679         intent = kzalloc(sizeof(*intent), GFP_KERNEL);
 680         if (!intent)
 681                 return NULL;
 682
 683         intent->data = kzalloc(size, GFP_KERNEL);
 684         if (!intent->data)
 685                 goto free_intent;
 686
 687         spin_lock_irqsave(&channel->intent_lock, flags);
 688         ret = idr_alloc_cyclic(&channel->liids, intent, 1, -1, GFP_ATOMIC);
 689         if (ret < 0) {
 690                 spin_unlock_irqrestore(&channel->intent_lock, flags);
 691                 goto free_data;
 692         }
 693         spin_unlock_irqrestore(&channel->intent_lock, flags);
 694
 695         intent->id = ret;
 696         intent->size = size;
 697         intent->reuse = reuseable;
 698
 699         return intent;
 700
 701 free_data:
 702         kfree(intent->data);
 703 free_intent:
 704         kfree(intent);
 705         return NULL;
 706 }
 707
 708 static void qcom_glink_handle_rx_done(struct qcom_glink *glink,
 709                                       u32 cid, uint32_t iid,
 710                                       bool reuse)
 711 {
 712         struct glink_core_rx_intent *intent;
 713         struct glink_channel *channel;
 714         unsigned long flags;
 715
 716         spin_lock_irqsave(&glink->idr_lock, flags);
 717         channel = idr_find(&glink->rcids, cid);
 718         spin_unlock_irqrestore(&glink->idr_lock, flags);
 719         if (!channel) {
 720                 dev_err(glink->dev, "invalid channel id received\n");
 721                 return;
 722         }
 723
 724         spin_lock_irqsave(&channel->intent_lock, flags);
 725         intent = idr_find(&channel->riids, iid);
 726
 727         if (!intent) {
 728                 spin_unlock_irqrestore(&channel->intent_lock, flags);
 729                 dev_err(glink->dev, "invalid intent id received\n");
 730                 return;
 731         }
 732
 733         intent->in_use = false;
 734
 735         if (!reuse) {
 736                 idr_remove(&channel->riids, intent->id);
 737                 kfree(intent);
 738         }
 739         spin_unlock_irqrestore(&channel->intent_lock, flags);
 740 }
 741
 742 /**
 743  * qcom_glink_handle_intent_req() - Receive a request for rx_intent
 744  *                                          from remote side
 745  * @glink:      Pointer to the transport interface
 746  * @cid:        Remote channel ID
 747  * @size:       size of the intent
 748  *
 749  * The function searches for the local channel to which the request for
 750  * rx_intent has arrived and allocates and notifies the remote back
 751  */
 752 static void qcom_glink_handle_intent_req(struct qcom_glink *glink,
 753                                          u32 cid, size_t size)
 754 {
 755         struct glink_core_rx_intent *intent;
 756         struct glink_channel *channel;
 757         unsigned long flags;
 758
 759         spin_lock_irqsave(&glink->idr_lock, flags);
 760         channel = idr_find(&glink->rcids, cid);
 761         spin_unlock_irqrestore(&glink->idr_lock, flags);
 762
 763         if (!channel) {
 764                 pr_err("%s channel not found for cid %d\n", __func__, cid);
 765                 return;
 766         }
 767
 768         intent = qcom_glink_alloc_intent(glink, channel, size, false);
 769         if (intent)
 770                 qcom_glink_advertise_intent(glink, channel, intent);
 771
 772         qcom_glink_send_intent_req_ack(glink, channel, !!intent);
 773 }
 774
 775 static int qcom_glink_rx_defer(struct qcom_glink *glink, size_t extra)
 776 {
 777         struct glink_defer_cmd *dcmd;
 778
 779         extra = ALIGN(extra, 8);
 780
 781         if (qcom_glink_rx_avail(glink) < sizeof(struct glink_msg) + extra) {
 782                 dev_dbg(glink->dev, "Insufficient data in rx fifo");
 783                 return -ENXIO;
 784         }
 785
 786         dcmd = kzalloc(struct_size(dcmd, data, extra), GFP_ATOMIC);
 787         if (!dcmd)
 788                 return -ENOMEM;
 789
 790         INIT_LIST_HEAD(&dcmd->node);
 791
 792         qcom_glink_rx_peak(glink, &dcmd->msg, 0, sizeof(dcmd->msg) + extra);
 793
 794         spin_lock(&glink->rx_lock);
 795         list_add_tail(&dcmd->node, &glink->rx_queue);
 796         spin_unlock(&glink->rx_lock);
 797
 798         schedule_work(&glink->rx_work);
 799         qcom_glink_rx_advance(glink, sizeof(dcmd->msg) + extra);
 800
 801         return 0;
 802 }
 803
 804 static int qcom_glink_rx_data(struct qcom_glink *glink, size_t avail)
 805 {
 806         struct glink_core_rx_intent *intent;
 807         struct glink_channel *channel;
 808         struct {
 809                 struct glink_msg msg;
 810                 __le32 chunk_size;
 811                 __le32 left_size;
 812         } __packed hdr;
 813         unsigned int chunk_size;
 814         unsigned int left_size;
 815         unsigned int rcid;
 816         unsigned int liid;
 817         int ret = 0;
 818         unsigned long flags;
 819
 820         if (avail < sizeof(hdr)) {
 821                 dev_dbg(glink->dev, "Not enough data in fifo\n");
 822                 return -EAGAIN;
 823         }
 824
 825         qcom_glink_rx_peak(glink, &hdr, 0, sizeof(hdr));
 826         chunk_size = le32_to_cpu(hdr.chunk_size);
 827         left_size = le32_to_cpu(hdr.left_size);
 828
 829         if (avail < sizeof(hdr) + chunk_size) {
 830                 dev_dbg(glink->dev, "Payload not yet in fifo\n");
 831                 return -EAGAIN;
 832         }
 833
 834         rcid = le16_to_cpu(hdr.msg.param1);
 835         spin_lock_irqsave(&glink->idr_lock, flags);
 836         channel = idr_find(&glink->rcids, rcid);
 837         spin_unlock_irqrestore(&glink->idr_lock, flags);
 838         if (!channel) {
 839                 dev_dbg(glink->dev, "Data on non-existing channel\n");
 840
 841                 /* Drop the message */
 842                 goto advance_rx;
 843         }
 844
 845         if (glink->intentless) {
 846                 /* Might have an ongoing, fragmented, message to append */
 847                 if (!channel->buf) {
 848                         intent = kzalloc(sizeof(*intent), GFP_ATOMIC);
 849                         if (!intent)
 850                                 return -ENOMEM;
 851
 852                         intent->data = kmalloc(chunk_size + left_size,
 853                                                GFP_ATOMIC);
 854                         if (!intent->data) {
 855                                 kfree(intent);
 856                                 return -ENOMEM;
 857                         }
 858
 859                         intent->id = 0xbabababa;
 860                         intent->size = chunk_size + left_size;
 861                         intent->offset = 0;
 862
 863                         channel->buf = intent;
 864                 } else {
 865                         intent = channel->buf;
 866                 }
 867         } else {
 868                 liid = le32_to_cpu(hdr.msg.param2);
 869
 870                 spin_lock_irqsave(&channel->intent_lock, flags);
 871                 intent = idr_find(&channel->liids, liid);
 872                 spin_unlock_irqrestore(&channel->intent_lock, flags);
 873
 874                 if (!intent) {
 875                         dev_err(glink->dev,
 876                                 "no intent found for channel %s intent %d",
 877                                 channel->name, liid);
 878                         ret = -ENOENT;
 879                         goto advance_rx;
 880                 }
 881         }
 882
 883         if (intent->size - intent->offset < chunk_size) {
 884                 dev_err(glink->dev, "Insufficient space in intent\n");
 885
 886                 /* The packet header lied, drop payload */
 887                 goto advance_rx;
 888         }
 889
 890         qcom_glink_rx_peak(glink, intent->data + intent->offset,
 891                            sizeof(hdr), chunk_size);
 892         intent->offset += chunk_size;
 893
 894         /* Handle message when no fragments remain to be received */
 895         if (!left_size) {
 896                 spin_lock(&channel->recv_lock);
 897                 if (channel->ept.cb) {
 898                         channel->ept.cb(channel->ept.rpdev,
 899                                         intent->data,
 900                                         intent->offset,
 901                                         channel->ept.priv,
 902                                         RPMSG_ADDR_ANY);
 903                 }
 904                 spin_unlock(&channel->recv_lock);
 905
 906                 intent->offset = 0;
 907                 channel->buf = NULL;
 908
 909                 qcom_glink_rx_done(glink, channel, intent);
 910         }
 911
 912 advance_rx:
 913         qcom_glink_rx_advance(glink, ALIGN(sizeof(hdr) + chunk_size, 8));
 914
 915         return ret;
 916 }
 917
 918 static void qcom_glink_handle_intent(struct qcom_glink *glink,
 919                                      unsigned int cid,
 920                                      unsigned int count,
 921                                      size_t avail)
 922 {
 923         struct glink_core_rx_intent *intent;
 924         struct glink_channel *channel;
 925         struct intent_pair {
 926                 __le32 size;
 927                 __le32 iid;
 928         };
 929
 930         struct {
 931                 struct glink_msg msg;
 932                 struct intent_pair intents[];
 933         } __packed * msg;
 934
 935         const size_t msglen = struct_size(msg, intents, count);
 936         int ret;
 937         int i;
 938         unsigned long flags;
 939
 940         if (avail < msglen) {
 941                 dev_dbg(glink->dev, "Not enough data in fifo\n");
 942                 return;
 943         }
 944
 945         spin_lock_irqsave(&glink->idr_lock, flags);
 946         channel = idr_find(&glink->rcids, cid);
 947         spin_unlock_irqrestore(&glink->idr_lock, flags);
 948         if (!channel) {
 949                 dev_err(glink->dev, "intents for non-existing channel\n");
 950                 return;
 951         }
 952
 953         msg = kmalloc(msglen, GFP_ATOMIC);
 954         if (!msg)
 955                 return;
 956
 957         qcom_glink_rx_peak(glink, msg, 0, msglen);
 958
 959         for (i = 0; i < count; ++i) {
 960                 intent = kzalloc(sizeof(*intent), GFP_ATOMIC);
 961                 if (!intent)
 962                         break;
 963
 964                 intent->id = le32_to_cpu(msg->intents[i].iid);
 965                 intent->size = le32_to_cpu(msg->intents[i].size);
 966
 967                 spin_lock_irqsave(&channel->intent_lock, flags);
 968                 ret = idr_alloc(&channel->riids, intent,
 969                                 intent->id, intent->id + 1, GFP_ATOMIC);
 970                 spin_unlock_irqrestore(&channel->intent_lock, flags);
 971
 972                 if (ret < 0)
 973                         dev_err(glink->dev, "failed to store remote intent\n");
 974         }
 975
 976         kfree(msg);
 977         qcom_glink_rx_advance(glink, ALIGN(msglen, 8));
 978 }
 979
 980 static int qcom_glink_rx_open_ack(struct qcom_glink *glink, unsigned int lcid)
 981 {
 982         struct glink_channel *channel;
 983
 984         spin_lock(&glink->idr_lock);
 985         channel = idr_find(&glink->lcids, lcid);
 986         spin_unlock(&glink->idr_lock);
 987         if (!channel) {
 988                 dev_err(glink->dev, "Invalid open ack packet\n");
 989                 return -EINVAL;
 990         }
 991
 992         complete_all(&channel->open_ack);
 993
 994         return 0;
 995 }
 996
 997 void qcom_glink_native_rx(struct qcom_glink *glink)
 998 {
 999         struct glink_msg msg;
1000         unsigned int param1;
1001         unsigned int param2;
1002         unsigned int avail;
1003         unsigned int cmd;
1004         int ret = 0;
1005
1006         /* To wakeup any blocking writers */
1007         wake_up_all(&glink->tx_avail_notify);
1008
1009         for (;;) {
1010                 avail = qcom_glink_rx_avail(glink);
1011                 if (avail < sizeof(msg))
1012                         break;
1013
1014                 qcom_glink_rx_peak(glink, &msg, 0, sizeof(msg));
1015
1016                 cmd = le16_to_cpu(msg.cmd);
1017                 param1 = le16_to_cpu(msg.param1);
1018                 param2 = le32_to_cpu(msg.param2);
1019
1020                 switch (cmd) {
1021                 case RPM_CMD_VERSION:
1022                 case RPM_CMD_VERSION_ACK:
1023                 case RPM_CMD_CLOSE:
1024                 case RPM_CMD_CLOSE_ACK:
1025                 case RPM_CMD_RX_INTENT_REQ:
1026                         ret = qcom_glink_rx_defer(glink, 0);
1027                         break;
1028                 case RPM_CMD_OPEN_ACK:
1029                         ret = qcom_glink_rx_open_ack(glink, param1);
1030                         qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
1031                         break;
1032                 case RPM_CMD_OPEN:
1033                         ret = qcom_glink_rx_defer(glink, param2);
1034                         break;
1035                 case RPM_CMD_TX_DATA:
1036                 case RPM_CMD_TX_DATA_CONT:
1037                         ret = qcom_glink_rx_data(glink, avail);
1038                         break;
1039                 case RPM_CMD_READ_NOTIF:
1040                         qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
1041                         qcom_glink_tx_kick(glink);
1042                         break;
1043                 case RPM_CMD_INTENT:
1044                         qcom_glink_handle_intent(glink, param1, param2, avail);
1045                         break;
1046                 case RPM_CMD_RX_DONE:
1047                         qcom_glink_handle_rx_done(glink, param1, param2, false);
1048                         qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
1049                         break;
1050                 case RPM_CMD_RX_DONE_W_REUSE:
1051                         qcom_glink_handle_rx_done(glink, param1, param2, true);
1052                         qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
1053                         break;
1054                 case RPM_CMD_RX_INTENT_REQ_ACK:
1055                         qcom_glink_handle_intent_req_ack(glink, param1, param2);
1056                         qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
1057                         break;
1058                 default:
1059                         dev_err(glink->dev, "unhandled rx cmd: %d\n", cmd);
1060                         ret = -EINVAL;
1061                         break;
1062                 }
1063
1064                 if (ret)
1065                         break;
1066         }
1067 }
1068 EXPORT_SYMBOL(qcom_glink_native_rx);
1069
1070 /* Locally initiated rpmsg_create_ept */
1071 static struct glink_channel *qcom_glink_create_local(struct qcom_glink *glink,
1072                                                      const char *name)
1073 {
1074         struct glink_channel *channel;
1075         int ret;
1076         unsigned long flags;
1077
1078         channel = qcom_glink_alloc_channel(glink, name);
1079         if (IS_ERR(channel))
1080                 return ERR_CAST(channel);
1081
1082         ret = qcom_glink_send_open_req(glink, channel);
1083         if (ret)
1084                 goto release_channel;
1085
1086         ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ);
1087         if (!ret)
1088                 goto err_timeout;
1089
1090         ret = wait_for_completion_timeout(&channel->open_req, 5 * HZ);
1091         if (!ret)
1092                 goto err_timeout;
1093
1094         qcom_glink_send_open_ack(glink, channel);
1095
1096         return channel;
1097
1098 err_timeout:
1099         /* qcom_glink_send_open_req() did register the channel in lcids*/
1100         spin_lock_irqsave(&glink->idr_lock, flags);
1101         idr_remove(&glink->lcids, channel->lcid);
1102         spin_unlock_irqrestore(&glink->idr_lock, flags);
1103
1104 release_channel:
1105         /* Release qcom_glink_send_open_req() reference */
1106         kref_put(&channel->refcount, qcom_glink_channel_release);
1107         /* Release qcom_glink_alloc_channel() reference */
1108         kref_put(&channel->refcount, qcom_glink_channel_release);
1109
1110         return ERR_PTR(-ETIMEDOUT);
1111 }
1112
1113 /* Remote initiated rpmsg_create_ept */
1114 static int qcom_glink_create_remote(struct qcom_glink *glink,
1115                                     struct glink_channel *channel)
1116 {
1117         int ret;
1118
1119         qcom_glink_send_open_ack(glink, channel);
1120
1121         ret = qcom_glink_send_open_req(glink, channel);
1122         if (ret)
1123                 goto close_link;
1124
1125         ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ);
1126         if (!ret) {
1127                 ret = -ETIMEDOUT;
1128                 goto close_link;
1129         }
1130
1131         return 0;
1132
1133 close_link:
1134         /*
1135          * Send a close request to "undo" our open-ack. The close-ack will
1136          * release qcom_glink_send_open_req() reference and the last reference
1137          * will be relesed after receiving remote_close or transport unregister
1138          * by calling qcom_glink_native_remove().
1139          */
1140         qcom_glink_send_close_req(glink, channel);
1141
1142         return ret;
1143 }
1144
1145 static struct rpmsg_endpoint *qcom_glink_create_ept(struct rpmsg_device *rpdev,
1146                                                     rpmsg_rx_cb_t cb,
1147                                                     void *priv,
1148                                                     struct rpmsg_channel_info
1149                                                                         chinfo)
1150 {
1151         struct glink_channel *parent = to_glink_channel(rpdev->ept);
1152         struct glink_channel *channel;
1153         struct qcom_glink *glink = parent->glink;
1154         struct rpmsg_endpoint *ept;
1155         const char *name = chinfo.name;
1156         int cid;
1157         int ret;
1158         unsigned long flags;
1159
1160         spin_lock_irqsave(&glink->idr_lock, flags);
1161         idr_for_each_entry(&glink->rcids, channel, cid) {
1162                 if (!strcmp(channel->name, name))
1163                         break;
1164         }
1165         spin_unlock_irqrestore(&glink->idr_lock, flags);
1166
1167         if (!channel) {
1168                 channel = qcom_glink_create_local(glink, name);
1169                 if (IS_ERR(channel))
1170                         return NULL;
1171         } else {
1172                 ret = qcom_glink_create_remote(glink, channel);
1173                 if (ret)
1174                         return NULL;
1175         }
1176
1177         ept = &channel->ept;
1178         ept->rpdev = rpdev;
1179         ept->cb = cb;
1180         ept->priv = priv;
1181         ept->ops = &glink_endpoint_ops;
1182
1183         return ept;
1184 }
1185
1186 static int qcom_glink_announce_create(struct rpmsg_device *rpdev)
1187 {
1188         struct glink_channel *channel = to_glink_channel(rpdev->ept);
1189         struct device_node *np = rpdev->dev.of_node;
1190         struct qcom_glink *glink = channel->glink;
1191         struct glink_core_rx_intent *intent;
1192         const struct property *prop = NULL;
1193         __be32 defaults[] = { cpu_to_be32(SZ_1K), cpu_to_be32(5) };
1194         int num_intents;
1195         int num_groups = 1;
1196         __be32 *val = defaults;
1197         int size;
1198
1199         if (glink->intentless || !completion_done(&channel->open_ack))
1200                 return 0;
1201
1202         prop = of_find_property(np, "qcom,intents", NULL);
1203         if (prop) {
1204                 val = prop->value;
1205                 num_groups = prop->length / sizeof(u32) / 2;
1206         }
1207
1208         /* Channel is now open, advertise base set of intents */
1209         while (num_groups--) {
1210                 size = be32_to_cpup(val++);
1211                 num_intents = be32_to_cpup(val++);
1212                 while (num_intents--) {
1213                         intent = qcom_glink_alloc_intent(glink, channel, size,
1214                                                          true);
1215                         if (!intent)
1216                                 break;
1217
1218                         qcom_glink_advertise_intent(glink, channel, intent);
1219                 }
1220         }
1221         return 0;
1222 }
1223
1224 static void qcom_glink_destroy_ept(struct rpmsg_endpoint *ept)
1225 {
1226         struct glink_channel *channel = to_glink_channel(ept);
1227         struct qcom_glink *glink = channel->glink;
1228         unsigned long flags;
1229
1230         spin_lock_irqsave(&channel->recv_lock, flags);
1231         channel->ept.cb = NULL;
1232         spin_unlock_irqrestore(&channel->recv_lock, flags);
1233
1234         /* Decouple the potential rpdev from the channel */
1235         channel->rpdev = NULL;
1236
1237         qcom_glink_send_close_req(glink, channel);
1238 }
1239
1240 static int qcom_glink_request_intent(struct qcom_glink *glink,
1241                                      struct glink_channel *channel,
1242                                      size_t size)
1243 {
1244         struct {
1245                 u16 id;
1246                 u16 cid;
1247                 u32 size;
1248         } __packed cmd;
1249
1250         int ret;
1251
1252         mutex_lock(&channel->intent_req_lock);
1253
1254         reinit_completion(&channel->intent_req_comp);
1255
1256         cmd.id = RPM_CMD_RX_INTENT_REQ;
1257         cmd.cid = channel->lcid;
1258         cmd.size = size;
1259
1260         ret = qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true);
1261         if (ret)
1262                 goto unlock;
1263
1264         ret = wait_for_completion_timeout(&channel->intent_req_comp, 10 * HZ);
1265         if (!ret) {
1266                 dev_err(glink->dev, "intent request timed out\n");
1267                 ret = -ETIMEDOUT;
1268         } else {
1269                 ret = channel->intent_req_result ? 0 : -ECANCELED;
1270         }
1271
1272 unlock:
1273         mutex_unlock(&channel->intent_req_lock);
1274         return ret;
1275 }
1276
1277 static int __qcom_glink_send(struct glink_channel *channel,
1278                              void *data, int len, bool wait)
1279 {
1280         struct qcom_glink *glink = channel->glink;
1281         struct glink_core_rx_intent *intent = NULL;
1282         struct glink_core_rx_intent *tmp;
1283         int iid = 0;
1284         struct {
1285                 struct glink_msg msg;
1286                 __le32 chunk_size;
1287                 __le32 left_size;
1288         } __packed req;
1289         int ret;
1290         unsigned long flags;
1291         int chunk_size = len;
1292         int left_size = 0;
1293
1294         if (!glink->intentless) {
1295                 while (!intent) {
1296                         spin_lock_irqsave(&channel->intent_lock, flags);
1297                         idr_for_each_entry(&channel->riids, tmp, iid) {
1298                                 if (tmp->size >= len && !tmp->in_use) {
1299                                         if (!intent)
1300                                                 intent = tmp;
1301                                         else if (intent->size > tmp->size)
1302                                                 intent = tmp;
1303                                         if (intent->size == len)
1304                                                 break;
1305                                 }
1306                         }
1307                         if (intent)
1308                                 intent->in_use = true;
1309                         spin_unlock_irqrestore(&channel->intent_lock, flags);
1310
1311                         /* We found an available intent */
1312                         if (intent)
1313                                 break;
1314
1315                         if (!wait)
1316                                 return -EBUSY;
1317
1318                         ret = qcom_glink_request_intent(glink, channel, len);
1319                         if (ret < 0)
1320                                 return ret;
1321                 }
1322
1323                 iid = intent->id;
1324         }
1325
1326         if (wait && chunk_size > SZ_8K) {
1327                 chunk_size = SZ_8K;
1328                 left_size = len - chunk_size;
1329         }
1330         req.msg.cmd = cpu_to_le16(RPM_CMD_TX_DATA);
1331         req.msg.param1 = cpu_to_le16(channel->lcid);
1332         req.msg.param2 = cpu_to_le32(iid);
1333         req.chunk_size = cpu_to_le32(chunk_size);
1334         req.left_size = cpu_to_le32(left_size);
1335
1336         ret = qcom_glink_tx(glink, &req, sizeof(req), data, chunk_size, wait);
1337
1338         /* Mark intent available if we failed */
1339         if (ret && intent) {
1340                 intent->in_use = false;
1341                 return ret;
1342         }
1343
1344         while (left_size > 0) {
1345                 data = (void *)((char *)data + chunk_size);
1346                 chunk_size = left_size;
1347                 if (chunk_size > SZ_8K)
1348                         chunk_size = SZ_8K;
1349                 left_size -= chunk_size;
1350
1351                 req.msg.cmd = cpu_to_le16(RPM_CMD_TX_DATA_CONT);
1352                 req.msg.param1 = cpu_to_le16(channel->lcid);
1353                 req.msg.param2 = cpu_to_le32(iid);
1354                 req.chunk_size = cpu_to_le32(chunk_size);
1355                 req.left_size = cpu_to_le32(left_size);
1356
1357                 ret = qcom_glink_tx(glink, &req, sizeof(req), data,
1358                                     chunk_size, wait);
1359
1360                 /* Mark intent available if we failed */
1361                 if (ret && intent) {
1362                         intent->in_use = false;
1363                         break;
1364                 }
1365         }
1366         return ret;
1367 }
1368
1369 static int qcom_glink_send(struct rpmsg_endpoint *ept, void *data, int len)
1370 {
1371         struct glink_channel *channel = to_glink_channel(ept);
1372
1373         return __qcom_glink_send(channel, data, len, true);
1374 }
1375
1376 static int qcom_glink_trysend(struct rpmsg_endpoint *ept, void *data, int len)
1377 {
1378         struct glink_channel *channel = to_glink_channel(ept);
1379
1380         return __qcom_glink_send(channel, data, len, false);
1381 }
1382
1383 static int qcom_glink_sendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst)
1384 {
1385         struct glink_channel *channel = to_glink_channel(ept);
1386
1387         return __qcom_glink_send(channel, data, len, true);
1388 }
1389
1390 static int qcom_glink_trysendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst)
1391 {
1392         struct glink_channel *channel = to_glink_channel(ept);
1393
1394         return __qcom_glink_send(channel, data, len, false);
1395 }
1396
1397 /*
1398  * Finds the device_node for the glink child interested in this channel.
1399  */
1400 static struct device_node *qcom_glink_match_channel(struct device_node *node,
1401                                                     const char *channel)
1402 {
1403         struct device_node *child;
1404         const char *name;
1405         const char *key;
1406         int ret;
1407
1408         for_each_available_child_of_node(node, child) {
1409                 key = "qcom,glink-channels";
1410                 ret = of_property_read_string(child, key, &name);
1411                 if (ret)
1412                         continue;
1413
1414                 if (strcmp(name, channel) == 0)
1415                         return child;
1416         }
1417
1418         return NULL;
1419 }
1420
1421 static const struct rpmsg_device_ops glink_device_ops = {
1422         .create_ept = qcom_glink_create_ept,
1423         .announce_create = qcom_glink_announce_create,
1424 };
1425
1426 static const struct rpmsg_endpoint_ops glink_endpoint_ops = {
1427         .destroy_ept = qcom_glink_destroy_ept,
1428         .send = qcom_glink_send,
1429         .sendto = qcom_glink_sendto,
1430         .trysend = qcom_glink_trysend,
1431         .trysendto = qcom_glink_trysendto,
1432 };
1433
1434 static void qcom_glink_rpdev_release(struct device *dev)
1435 {
1436         struct rpmsg_device *rpdev = to_rpmsg_device(dev);
1437
1438         kfree(rpdev);
1439 }
1440
1441 static int qcom_glink_rx_open(struct qcom_glink *glink, unsigned int rcid,
1442                               char *name)
1443 {
1444         struct glink_channel *channel;
1445         struct rpmsg_device *rpdev;
1446         bool create_device = false;
1447         struct device_node *node;
1448         int lcid;
1449         int ret;
1450         unsigned long flags;
1451
1452         spin_lock_irqsave(&glink->idr_lock, flags);
1453         idr_for_each_entry(&glink->lcids, channel, lcid) {
1454                 if (!strcmp(channel->name, name))
1455                         break;
1456         }
1457         spin_unlock_irqrestore(&glink->idr_lock, flags);
1458
1459         if (!channel) {
1460                 channel = qcom_glink_alloc_channel(glink, name);
1461                 if (IS_ERR(channel))
1462                         return PTR_ERR(channel);
1463
1464                 /* The opening dance was initiated by the remote */
1465                 create_device = true;
1466         }
1467
1468         spin_lock_irqsave(&glink->idr_lock, flags);
1469         ret = idr_alloc(&glink->rcids, channel, rcid, rcid + 1, GFP_ATOMIC);
1470         if (ret < 0) {
1471                 dev_err(glink->dev, "Unable to insert channel into rcid list\n");
1472                 spin_unlock_irqrestore(&glink->idr_lock, flags);
1473                 goto free_channel;
1474         }
1475         channel->rcid = ret;
1476         spin_unlock_irqrestore(&glink->idr_lock, flags);
1477
1478         complete_all(&channel->open_req);
1479
1480         if (create_device) {
1481                 rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL);
1482                 if (!rpdev) {
1483                         ret = -ENOMEM;
1484                         goto rcid_remove;
1485                 }
1486
1487                 rpdev->ept = &channel->ept;
1488                 strscpy_pad(rpdev->id.name, name, RPMSG_NAME_SIZE);
1489                 rpdev->src = RPMSG_ADDR_ANY;
1490                 rpdev->dst = RPMSG_ADDR_ANY;
1491                 rpdev->ops = &glink_device_ops;
1492
1493                 node = qcom_glink_match_channel(glink->dev->of_node, name);
1494                 rpdev->dev.of_node = node;
1495                 rpdev->dev.parent = glink->dev;
1496                 rpdev->dev.release = qcom_glink_rpdev_release;
1497
1498                 ret = rpmsg_register_device(rpdev);
1499                 if (ret)
1500                         goto rcid_remove;
1501
1502                 channel->rpdev = rpdev;
1503         }
1504
1505         return 0;
1506
1507 rcid_remove:
1508         spin_lock_irqsave(&glink->idr_lock, flags);
1509         idr_remove(&glink->rcids, channel->rcid);
1510         channel->rcid = 0;
1511         spin_unlock_irqrestore(&glink->idr_lock, flags);
1512 free_channel:
1513         /* Release the reference, iff we took it */
1514         if (create_device)
1515                 kref_put(&channel->refcount, qcom_glink_channel_release);
1516
1517         return ret;
1518 }
1519
1520 static void qcom_glink_rx_close(struct qcom_glink *glink, unsigned int rcid)
1521 {
1522         struct rpmsg_channel_info chinfo;
1523         struct glink_channel *channel;
1524         unsigned long flags;
1525
1526         spin_lock_irqsave(&glink->idr_lock, flags);
1527         channel = idr_find(&glink->rcids, rcid);
1528         spin_unlock_irqrestore(&glink->idr_lock, flags);
1529         if (WARN(!channel, "close request on unknown channel\n"))
1530                 return;
1531
1532         /* cancel pending rx_done work */
1533         cancel_work_sync(&channel->intent_work);
1534
1535         if (channel->rpdev) {
1536                 strscpy_pad(chinfo.name, channel->name, sizeof(chinfo.name));
1537                 chinfo.src = RPMSG_ADDR_ANY;
1538                 chinfo.dst = RPMSG_ADDR_ANY;
1539
1540                 rpmsg_unregister_device(glink->dev, &chinfo);
1541         }
1542         channel->rpdev = NULL;
1543
1544         qcom_glink_send_close_ack(glink, channel->rcid);
1545
1546         spin_lock_irqsave(&glink->idr_lock, flags);
1547         idr_remove(&glink->rcids, channel->rcid);
1548         channel->rcid = 0;
1549         spin_unlock_irqrestore(&glink->idr_lock, flags);
1550
1551         kref_put(&channel->refcount, qcom_glink_channel_release);
1552 }
1553
1554 static void qcom_glink_rx_close_ack(struct qcom_glink *glink, unsigned int lcid)
1555 {
1556         struct rpmsg_channel_info chinfo;
1557         struct glink_channel *channel;
1558         unsigned long flags;
1559
1560         /* To wakeup any blocking writers */
1561         wake_up_all(&glink->tx_avail_notify);
1562
1563         spin_lock_irqsave(&glink->idr_lock, flags);
1564         channel = idr_find(&glink->lcids, lcid);
1565         if (WARN(!channel, "close ack on unknown channel\n")) {
1566                 spin_unlock_irqrestore(&glink->idr_lock, flags);
1567                 return;
1568         }
1569
1570         idr_remove(&glink->lcids, channel->lcid);
1571         channel->lcid = 0;
1572         spin_unlock_irqrestore(&glink->idr_lock, flags);
1573
1574         /* Decouple the potential rpdev from the channel */
1575         if (channel->rpdev) {
1576                 strscpy(chinfo.name, channel->name, sizeof(chinfo.name));
1577                 chinfo.src = RPMSG_ADDR_ANY;
1578                 chinfo.dst = RPMSG_ADDR_ANY;
1579
1580                 rpmsg_unregister_device(glink->dev, &chinfo);
1581         }
1582         channel->rpdev = NULL;
1583
1584         kref_put(&channel->refcount, qcom_glink_channel_release);
1585 }
1586
1587 static void qcom_glink_work(struct work_struct *work)
1588 {
1589         struct qcom_glink *glink = container_of(work, struct qcom_glink,
1590                                                 rx_work);
1591         struct glink_defer_cmd *dcmd;
1592         struct glink_msg *msg;
1593         unsigned long flags;
1594         unsigned int param1;
1595         unsigned int param2;
1596         unsigned int cmd;
1597
1598         for (;;) {
1599                 spin_lock_irqsave(&glink->rx_lock, flags);
1600                 if (list_empty(&glink->rx_queue)) {
1601                         spin_unlock_irqrestore(&glink->rx_lock, flags);
1602                         break;
1603                 }
1604                 dcmd = list_first_entry(&glink->rx_queue,
1605                                         struct glink_defer_cmd, node);
1606                 list_del(&dcmd->node);
1607                 spin_unlock_irqrestore(&glink->rx_lock, flags);
1608
1609                 msg = &dcmd->msg;
1610                 cmd = le16_to_cpu(msg->cmd);
1611                 param1 = le16_to_cpu(msg->param1);
1612                 param2 = le32_to_cpu(msg->param2);
1613
1614                 switch (cmd) {
1615                 case RPM_CMD_VERSION:
1616                         qcom_glink_receive_version(glink, param1, param2);
1617                         break;
1618                 case RPM_CMD_VERSION_ACK:
1619                         qcom_glink_receive_version_ack(glink, param1, param2);
1620                         break;
1621                 case RPM_CMD_OPEN:
1622                         qcom_glink_rx_open(glink, param1, msg->data);
1623                         break;
1624                 case RPM_CMD_CLOSE:
1625                         qcom_glink_rx_close(glink, param1);
1626                         break;
1627                 case RPM_CMD_CLOSE_ACK:
1628                         qcom_glink_rx_close_ack(glink, param1);
1629                         break;
1630                 case RPM_CMD_RX_INTENT_REQ:
1631                         qcom_glink_handle_intent_req(glink, param1, param2);
1632                         break;
1633                 default:
1634                         WARN(1, "Unknown defer object %d\n", cmd);
1635                         break;
1636                 }
1637
1638                 kfree(dcmd);
1639         }
1640 }
1641
1642 static void qcom_glink_cancel_rx_work(struct qcom_glink *glink)
1643 {
1644         struct glink_defer_cmd *dcmd;
1645         struct glink_defer_cmd *tmp;
1646
1647         /* cancel any pending deferred rx_work */
1648         cancel_work_sync(&glink->rx_work);
1649
1650         list_for_each_entry_safe(dcmd, tmp, &glink->rx_queue, node)
1651                 kfree(dcmd);
1652 }
1653
1654 static ssize_t rpmsg_name_show(struct device *dev,
1655                                struct device_attribute *attr, char *buf)
1656 {
1657         int ret = 0;
1658         const char *name;
1659
1660         ret = of_property_read_string(dev->of_node, "label", &name);
1661         if (ret < 0)
1662                 name = dev->of_node->name;
1663
1664         return sysfs_emit(buf, "%s\n", name);
1665 }
1666 static DEVICE_ATTR_RO(rpmsg_name);
1667
1668 static struct attribute *qcom_glink_attrs[] = {
1669         &dev_attr_rpmsg_name.attr,
1670         NULL
1671 };
1672 ATTRIBUTE_GROUPS(qcom_glink);
1673
1674 static void qcom_glink_device_release(struct device *dev)
1675 {
1676         struct rpmsg_device *rpdev = to_rpmsg_device(dev);
1677         struct glink_channel *channel = to_glink_channel(rpdev->ept);
1678
1679         /* Release qcom_glink_alloc_channel() reference */
1680         kref_put(&channel->refcount, qcom_glink_channel_release);
1681         kfree(rpdev);
1682 }
1683
1684 static int qcom_glink_create_chrdev(struct qcom_glink *glink)
1685 {
1686         struct rpmsg_device *rpdev;
1687         struct glink_channel *channel;
1688
1689         rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL);
1690         if (!rpdev)
1691                 return -ENOMEM;
1692
1693         channel = qcom_glink_alloc_channel(glink, "rpmsg_chrdev");
1694         if (IS_ERR(channel)) {
1695                 kfree(rpdev);
1696                 return PTR_ERR(channel);
1697         }
1698         channel->rpdev = rpdev;
1699
1700         rpdev->ept = &channel->ept;
1701         rpdev->ops = &glink_device_ops;
1702         rpdev->dev.parent = glink->dev;
1703         rpdev->dev.release = qcom_glink_device_release;
1704
1705         return rpmsg_ctrldev_register_device(rpdev);
1706 }
1707
1708 struct qcom_glink *qcom_glink_native_probe(struct device *dev,
1709                                            unsigned long features,
1710                                            struct qcom_glink_pipe *rx,
1711                                            struct qcom_glink_pipe *tx,
1712                                            bool intentless)
1713 {
1714         int ret;
1715         struct qcom_glink *glink;
1716
1717         glink = devm_kzalloc(dev, sizeof(*glink), GFP_KERNEL);
1718         if (!glink)
1719                 return ERR_PTR(-ENOMEM);
1720
1721         glink->dev = dev;
1722         glink->tx_pipe = tx;
1723         glink->rx_pipe = rx;
1724
1725         glink->features = features;
1726         glink->intentless = intentless;
1727
1728         spin_lock_init(&glink->tx_lock);
1729         spin_lock_init(&glink->rx_lock);
1730         INIT_LIST_HEAD(&glink->rx_queue);
1731         INIT_WORK(&glink->rx_work, qcom_glink_work);
1732         init_waitqueue_head(&glink->tx_avail_notify);
1733
1734         spin_lock_init(&glink->idr_lock);
1735         idr_init(&glink->lcids);
1736         idr_init(&glink->rcids);
1737
1738         glink->dev->groups = qcom_glink_groups;
1739
1740         ret = device_add_groups(dev, qcom_glink_groups);
1741         if (ret)
1742                 dev_err(dev, "failed to add groups\n");
1743
1744         ret = qcom_glink_send_version(glink);
1745         if (ret)
1746                 return ERR_PTR(ret);
1747
1748         ret = qcom_glink_create_chrdev(glink);
1749         if (ret)
1750                 dev_err(glink->dev, "failed to register chrdev\n");
1751
1752         return glink;
1753 }
1754 EXPORT_SYMBOL_GPL(qcom_glink_native_probe);
1755
1756 static int qcom_glink_remove_device(struct device *dev, void *data)
1757 {
1758         device_unregister(dev);
1759
1760         return 0;
1761 }
1762
1763 void qcom_glink_native_remove(struct qcom_glink *glink)
1764 {
1765         struct glink_channel *channel;
1766         int cid;
1767         int ret;
1768
1769         qcom_glink_cancel_rx_work(glink);
1770
1771         ret = device_for_each_child(glink->dev, NULL, qcom_glink_remove_device);
1772         if (ret)
1773                 dev_warn(glink->dev, "Can't remove GLINK devices: %d\n", ret);
1774
1775         /* Release any defunct local channels, waiting for close-ack */
1776         idr_for_each_entry(&glink->lcids, channel, cid)
1777                 kref_put(&channel->refcount, qcom_glink_channel_release);
1778
1779         /* Release any defunct local channels, waiting for close-req */
1780         idr_for_each_entry(&glink->rcids, channel, cid)
1781                 kref_put(&channel->refcount, qcom_glink_channel_release);
1782
1783         idr_destroy(&glink->lcids);
1784         idr_destroy(&glink->rcids);
1785 }
1786 EXPORT_SYMBOL_GPL(qcom_glink_native_remove);
1787
1788 MODULE_DESCRIPTION("Qualcomm GLINK driver");
1789 MODULE_LICENSE("GPL v2");