net/tls/tls_sw.c

   1 /*
   2  * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved.
   3  * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved.
   4  * Copyright (c) 2016-2017, Lance Chao <lancerchao@fb.com>. All rights reserved.
   5  * Copyright (c) 2016, Fridolin Pokorny <fridolin.pokorny@gmail.com>. All rights reserved.
   6  * Copyright (c) 2016, Nikos Mavrogiannopoulos <nmav@gnutls.org>. All rights reserved.
   7  *
   8  * This software is available to you under a choice of one of two
   9  * licenses.  You may choose to be licensed under the terms of the GNU
  10  * General Public License (GPL) Version 2, available from the file
  11  * COPYING in the main directory of this source tree, or the
  12  * OpenIB.org BSD license below:
  13  *
  14  *     Redistribution and use in source and binary forms, with or
  15  *     without modification, are permitted provided that the following
  16  *     conditions are met:
  17  *
  18  *      - Redistributions of source code must retain the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer.
  21  *
  22  *      - Redistributions in binary form must reproduce the above
  23  *        copyright notice, this list of conditions and the following
  24  *        disclaimer in the documentation and/or other materials
  25  *        provided with the distribution.
  26  *
  27  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  28  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  29  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  30  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  31  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  32  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  33  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  34  * SOFTWARE.
  35  */
  36
  37 #include <linux/sched/signal.h>
  38 #include <linux/module.h>
  39 #include <crypto/aead.h>
  40
  41 #include <net/strparser.h>
  42 #include <net/tls.h>
  43
  44 #define MAX_IV_SIZE     TLS_CIPHER_AES_GCM_128_IV_SIZE
  45
  46 static int tls_do_decryption(struct sock *sk,
  47                              struct scatterlist *sgin,
  48                              struct scatterlist *sgout,
  49                              char *iv_recv,
  50                              size_t data_len,
  51                              struct sk_buff *skb,
  52                              gfp_t flags)
  53 {
  54         struct tls_context *tls_ctx = tls_get_ctx(sk);
  55         struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
  56         struct strp_msg *rxm = strp_msg(skb);
  57         struct aead_request *aead_req;
  58
  59         int ret;
  60         unsigned int req_size = sizeof(struct aead_request) +
  61                 crypto_aead_reqsize(ctx->aead_recv);
  62
  63         aead_req = kzalloc(req_size, flags);
  64         if (!aead_req)
  65                 return -ENOMEM;
  66
  67         aead_request_set_tfm(aead_req, ctx->aead_recv);
  68         aead_request_set_ad(aead_req, TLS_AAD_SPACE_SIZE);
  69         aead_request_set_crypt(aead_req, sgin, sgout,
  70                                data_len + tls_ctx->rx.tag_size,
  71                                (u8 *)iv_recv);
  72         aead_request_set_callback(aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
  73                                   crypto_req_done, &ctx->async_wait);
  74
  75         ret = crypto_wait_req(crypto_aead_decrypt(aead_req), &ctx->async_wait);
  76
  77         if (ret < 0)
  78                 goto out;
  79
  80         rxm->offset += tls_ctx->rx.prepend_size;
  81         rxm->full_len -= tls_ctx->rx.overhead_size;
  82         tls_advance_record_sn(sk, &tls_ctx->rx);
  83
  84         ctx->decrypted = true;
  85
  86         ctx->saved_data_ready(sk);
  87
  88 out:
  89         kfree(aead_req);
  90         return ret;
  91 }
  92
  93 static void trim_sg(struct sock *sk, struct scatterlist *sg,
  94                     int *sg_num_elem, unsigned int *sg_size, int target_size)
  95 {
  96         int i = *sg_num_elem - 1;
  97         int trim = *sg_size - target_size;
  98
  99         if (trim <= 0) {
 100                 WARN_ON(trim < 0);
 101                 return;
 102         }
 103
 104         *sg_size = target_size;
 105         while (trim >= sg[i].length) {
 106                 trim -= sg[i].length;
 107                 sk_mem_uncharge(sk, sg[i].length);
 108                 put_page(sg_page(&sg[i]));
 109                 i--;
 110
 111                 if (i < 0)
 112                         goto out;
 113         }
 114
 115         sg[i].length -= trim;
 116         sk_mem_uncharge(sk, trim);
 117
 118 out:
 119         *sg_num_elem = i + 1;
 120 }
 121
 122 static void trim_both_sgl(struct sock *sk, int target_size)
 123 {
 124         struct tls_context *tls_ctx = tls_get_ctx(sk);
 125         struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
 126
 127         trim_sg(sk, ctx->sg_plaintext_data,
 128                 &ctx->sg_plaintext_num_elem,
 129                 &ctx->sg_plaintext_size,
 130                 target_size);
 131
 132         if (target_size > 0)
 133                 target_size += tls_ctx->tx.overhead_size;
 134
 135         trim_sg(sk, ctx->sg_encrypted_data,
 136                 &ctx->sg_encrypted_num_elem,
 137                 &ctx->sg_encrypted_size,
 138                 target_size);
 139 }
 140
 141 static int alloc_encrypted_sg(struct sock *sk, int len)
 142 {
 143         struct tls_context *tls_ctx = tls_get_ctx(sk);
 144         struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
 145         int rc = 0;
 146
 147         rc = sk_alloc_sg(sk, len,
 148                          ctx->sg_encrypted_data, 0,
 149                          &ctx->sg_encrypted_num_elem,
 150                          &ctx->sg_encrypted_size, 0);
 151
 152         return rc;
 153 }
 154
 155 static int alloc_plaintext_sg(struct sock *sk, int len)
 156 {
 157         struct tls_context *tls_ctx = tls_get_ctx(sk);
 158         struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
 159         int rc = 0;
 160
 161         rc = sk_alloc_sg(sk, len, ctx->sg_plaintext_data, 0,
 162                          &ctx->sg_plaintext_num_elem, &ctx->sg_plaintext_size,
 163                          tls_ctx->pending_open_record_frags);
 164
 165         return rc;
 166 }
 167
 168 static void free_sg(struct sock *sk, struct scatterlist *sg,
 169                     int *sg_num_elem, unsigned int *sg_size)
 170 {
 171         int i, n = *sg_num_elem;
 172
 173         for (i = 0; i < n; ++i) {
 174                 sk_mem_uncharge(sk, sg[i].length);
 175                 put_page(sg_page(&sg[i]));
 176         }
 177         *sg_num_elem = 0;
 178         *sg_size = 0;
 179 }
 180
 181 static void tls_free_both_sg(struct sock *sk)
 182 {
 183         struct tls_context *tls_ctx = tls_get_ctx(sk);
 184         struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
 185
 186         free_sg(sk, ctx->sg_encrypted_data, &ctx->sg_encrypted_num_elem,
 187                 &ctx->sg_encrypted_size);
 188
 189         free_sg(sk, ctx->sg_plaintext_data, &ctx->sg_plaintext_num_elem,
 190                 &ctx->sg_plaintext_size);
 191 }
 192
 193 static int tls_do_encryption(struct tls_context *tls_ctx,
 194                              struct tls_sw_context_tx *ctx,
 195                              struct aead_request *aead_req,
 196                              size_t data_len)
 197 {
 198         int rc;
 199
 200         ctx->sg_encrypted_data[0].offset += tls_ctx->tx.prepend_size;
 201         ctx->sg_encrypted_data[0].length -= tls_ctx->tx.prepend_size;
 202
 203         aead_request_set_tfm(aead_req, ctx->aead_send);
 204         aead_request_set_ad(aead_req, TLS_AAD_SPACE_SIZE);
 205         aead_request_set_crypt(aead_req, ctx->sg_aead_in, ctx->sg_aead_out,
 206                                data_len, tls_ctx->tx.iv);
 207
 208         aead_request_set_callback(aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
 209                                   crypto_req_done, &ctx->async_wait);
 210
 211         rc = crypto_wait_req(crypto_aead_encrypt(aead_req), &ctx->async_wait);
 212
 213         ctx->sg_encrypted_data[0].offset -= tls_ctx->tx.prepend_size;
 214         ctx->sg_encrypted_data[0].length += tls_ctx->tx.prepend_size;
 215
 216         return rc;
 217 }
 218
 219 static int tls_push_record(struct sock *sk, int flags,
 220                            unsigned char record_type)
 221 {
 222         struct tls_context *tls_ctx = tls_get_ctx(sk);
 223         struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
 224         struct aead_request *req;
 225         int rc;
 226
 227         req = kzalloc(sizeof(struct aead_request) +
 228                       crypto_aead_reqsize(ctx->aead_send), sk->sk_allocation);
 229         if (!req)
 230                 return -ENOMEM;
 231
 232         sg_mark_end(ctx->sg_plaintext_data + ctx->sg_plaintext_num_elem - 1);
 233         sg_mark_end(ctx->sg_encrypted_data + ctx->sg_encrypted_num_elem - 1);
 234
 235         tls_make_aad(ctx->aad_space, ctx->sg_plaintext_size,
 236                      tls_ctx->tx.rec_seq, tls_ctx->tx.rec_seq_size,
 237                      record_type);
 238
 239         tls_fill_prepend(tls_ctx,
 240                          page_address(sg_page(&ctx->sg_encrypted_data[0])) +
 241                          ctx->sg_encrypted_data[0].offset,
 242                          ctx->sg_plaintext_size, record_type);
 243
 244         tls_ctx->pending_open_record_frags = 0;
 245         set_bit(TLS_PENDING_CLOSED_RECORD, &tls_ctx->flags);
 246
 247         rc = tls_do_encryption(tls_ctx, ctx, req, ctx->sg_plaintext_size);
 248         if (rc < 0) {
 249                 /* If we are called from write_space and
 250                  * we fail, we need to set this SOCK_NOSPACE
 251                  * to trigger another write_space in the future.
 252                  */
 253                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 254                 goto out_req;
 255         }
 256
 257         free_sg(sk, ctx->sg_plaintext_data, &ctx->sg_plaintext_num_elem,
 258                 &ctx->sg_plaintext_size);
 259
 260         ctx->sg_encrypted_num_elem = 0;
 261         ctx->sg_encrypted_size = 0;
 262
 263         /* Only pass through MSG_DONTWAIT and MSG_NOSIGNAL flags */
 264         rc = tls_push_sg(sk, tls_ctx, ctx->sg_encrypted_data, 0, flags);
 265         if (rc < 0 && rc != -EAGAIN)
 266                 tls_err_abort(sk, EBADMSG);
 267
 268         tls_advance_record_sn(sk, &tls_ctx->tx);
 269 out_req:
 270         kfree(req);
 271         return rc;
 272 }
 273
 274 static int tls_sw_push_pending_record(struct sock *sk, int flags)
 275 {
 276         return tls_push_record(sk, flags, TLS_RECORD_TYPE_DATA);
 277 }
 278
 279 static int zerocopy_from_iter(struct sock *sk, struct iov_iter *from,
 280                               int length, int *pages_used,
 281                               unsigned int *size_used,
 282                               struct scatterlist *to, int to_max_pages,
 283                               bool charge)
 284 {
 285         struct page *pages[MAX_SKB_FRAGS];
 286
 287         size_t offset;
 288         ssize_t copied, use;
 289         int i = 0;
 290         unsigned int size = *size_used;
 291         int num_elem = *pages_used;
 292         int rc = 0;
 293         int maxpages;
 294
 295         while (length > 0) {
 296                 i = 0;
 297                 maxpages = to_max_pages - num_elem;
 298                 if (maxpages == 0) {
 299                         rc = -EFAULT;
 300                         goto out;
 301                 }
 302                 copied = iov_iter_get_pages(from, pages,
 303                                             length,
 304                                             maxpages, &offset);
 305                 if (copied <= 0) {
 306                         rc = -EFAULT;
 307                         goto out;
 308                 }
 309
 310                 iov_iter_advance(from, copied);
 311
 312                 length -= copied;
 313                 size += copied;
 314                 while (copied) {
 315                         use = min_t(int, copied, PAGE_SIZE - offset);
 316
 317                         sg_set_page(&to[num_elem],
 318                                     pages[i], use, offset);
 319                         sg_unmark_end(&to[num_elem]);
 320                         if (charge)
 321                                 sk_mem_charge(sk, use);
 322
 323                         offset = 0;
 324                         copied -= use;
 325
 326                         ++i;
 327                         ++num_elem;
 328                 }
 329         }
 330
 331 out:
 332         *size_used = size;
 333         *pages_used = num_elem;
 334
 335         return rc;
 336 }
 337
 338 static int memcopy_from_iter(struct sock *sk, struct iov_iter *from,
 339                              int bytes)
 340 {
 341         struct tls_context *tls_ctx = tls_get_ctx(sk);
 342         struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
 343         struct scatterlist *sg = ctx->sg_plaintext_data;
 344         int copy, i, rc = 0;
 345
 346         for (i = tls_ctx->pending_open_record_frags;
 347              i < ctx->sg_plaintext_num_elem; ++i) {
 348                 copy = sg[i].length;
 349                 if (copy_from_iter(
 350                                 page_address(sg_page(&sg[i])) + sg[i].offset,
 351                                 copy, from) != copy) {
 352                         rc = -EFAULT;
 353                         goto out;
 354                 }
 355                 bytes -= copy;
 356
 357                 ++tls_ctx->pending_open_record_frags;
 358
 359                 if (!bytes)
 360                         break;
 361         }
 362
 363 out:
 364         return rc;
 365 }
 366
 367 int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
 368 {
 369         struct tls_context *tls_ctx = tls_get_ctx(sk);
 370         struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
 371         int ret = 0;
 372         int required_size;
 373         long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
 374         bool eor = !(msg->msg_flags & MSG_MORE);
 375         size_t try_to_copy, copied = 0;
 376         unsigned char record_type = TLS_RECORD_TYPE_DATA;
 377         int record_room;
 378         bool full_record;
 379         int orig_size;
 380
 381         if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL))
 382                 return -ENOTSUPP;
 383
 384         lock_sock(sk);
 385
 386         if (tls_complete_pending_work(sk, tls_ctx, msg->msg_flags, &timeo))
 387                 goto send_end;
 388
 389         if (unlikely(msg->msg_controllen)) {
 390                 ret = tls_proccess_cmsg(sk, msg, &record_type);
 391                 if (ret)
 392                         goto send_end;
 393         }
 394
 395         while (msg_data_left(msg)) {
 396                 if (sk->sk_err) {
 397                         ret = -sk->sk_err;
 398                         goto send_end;
 399                 }
 400
 401                 orig_size = ctx->sg_plaintext_size;
 402                 full_record = false;
 403                 try_to_copy = msg_data_left(msg);
 404                 record_room = TLS_MAX_PAYLOAD_SIZE - ctx->sg_plaintext_size;
 405                 if (try_to_copy >= record_room) {
 406                         try_to_copy = record_room;
 407                         full_record = true;
 408                 }
 409
 410                 required_size = ctx->sg_plaintext_size + try_to_copy +
 411                                 tls_ctx->tx.overhead_size;
 412
 413                 if (!sk_stream_memory_free(sk))
 414                         goto wait_for_sndbuf;
 415 alloc_encrypted:
 416                 ret = alloc_encrypted_sg(sk, required_size);
 417                 if (ret) {
 418                         if (ret != -ENOSPC)
 419                                 goto wait_for_memory;
 420
 421                         /* Adjust try_to_copy according to the amount that was
 422                          * actually allocated. The difference is due
 423                          * to max sg elements limit
 424                          */
 425                         try_to_copy -= required_size - ctx->sg_encrypted_size;
 426                         full_record = true;
 427                 }
 428
 429                 if (full_record || eor) {
 430                         ret = zerocopy_from_iter(sk, &msg->msg_iter,
 431                                 try_to_copy, &ctx->sg_plaintext_num_elem,
 432                                 &ctx->sg_plaintext_size,
 433                                 ctx->sg_plaintext_data,
 434                                 ARRAY_SIZE(ctx->sg_plaintext_data),
 435                                 true);
 436                         if (ret)
 437                                 goto fallback_to_reg_send;
 438
 439                         copied += try_to_copy;
 440                         ret = tls_push_record(sk, msg->msg_flags, record_type);
 441                         if (!ret)
 442                                 continue;
 443                         if (ret < 0)
 444                                 goto send_end;
 445
 446                         copied -= try_to_copy;
 447 fallback_to_reg_send:
 448                         iov_iter_revert(&msg->msg_iter,
 449                                         ctx->sg_plaintext_size - orig_size);
 450                         trim_sg(sk, ctx->sg_plaintext_data,
 451                                 &ctx->sg_plaintext_num_elem,
 452                                 &ctx->sg_plaintext_size,
 453                                 orig_size);
 454                 }
 455
 456                 required_size = ctx->sg_plaintext_size + try_to_copy;
 457 alloc_plaintext:
 458                 ret = alloc_plaintext_sg(sk, required_size);
 459                 if (ret) {
 460                         if (ret != -ENOSPC)
 461                                 goto wait_for_memory;
 462
 463                         /* Adjust try_to_copy according to the amount that was
 464                          * actually allocated. The difference is due
 465                          * to max sg elements limit
 466                          */
 467                         try_to_copy -= required_size - ctx->sg_plaintext_size;
 468                         full_record = true;
 469
 470                         trim_sg(sk, ctx->sg_encrypted_data,
 471                                 &ctx->sg_encrypted_num_elem,
 472                                 &ctx->sg_encrypted_size,
 473                                 ctx->sg_plaintext_size +
 474                                 tls_ctx->tx.overhead_size);
 475                 }
 476
 477                 ret = memcopy_from_iter(sk, &msg->msg_iter, try_to_copy);
 478                 if (ret)
 479                         goto trim_sgl;
 480
 481                 copied += try_to_copy;
 482                 if (full_record || eor) {
 483 push_record:
 484                         ret = tls_push_record(sk, msg->msg_flags, record_type);
 485                         if (ret) {
 486                                 if (ret == -ENOMEM)
 487                                         goto wait_for_memory;
 488
 489                                 goto send_end;
 490                         }
 491                 }
 492
 493                 continue;
 494
 495 wait_for_sndbuf:
 496                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 497 wait_for_memory:
 498                 ret = sk_stream_wait_memory(sk, &timeo);
 499                 if (ret) {
 500 trim_sgl:
 501                         trim_both_sgl(sk, orig_size);
 502                         goto send_end;
 503                 }
 504
 505                 if (tls_is_pending_closed_record(tls_ctx))
 506                         goto push_record;
 507
 508                 if (ctx->sg_encrypted_size < required_size)
 509                         goto alloc_encrypted;
 510
 511                 goto alloc_plaintext;
 512         }
 513
 514 send_end:
 515         ret = sk_stream_error(sk, msg->msg_flags, ret);
 516
 517         release_sock(sk);
 518         return copied ? copied : ret;
 519 }
 520
 521 int tls_sw_sendpage(struct sock *sk, struct page *page,
 522                     int offset, size_t size, int flags)
 523 {
 524         struct tls_context *tls_ctx = tls_get_ctx(sk);
 525         struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
 526         int ret = 0;
 527         long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
 528         bool eor;
 529         size_t orig_size = size;
 530         unsigned char record_type = TLS_RECORD_TYPE_DATA;
 531         struct scatterlist *sg;
 532         bool full_record;
 533         int record_room;
 534
 535         if (flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL |
 536                       MSG_SENDPAGE_NOTLAST))
 537                 return -ENOTSUPP;
 538
 539         /* No MSG_EOR from splice, only look at MSG_MORE */
 540         eor = !(flags & (MSG_MORE | MSG_SENDPAGE_NOTLAST));
 541
 542         lock_sock(sk);
 543
 544         sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
 545
 546         if (tls_complete_pending_work(sk, tls_ctx, flags, &timeo))
 547                 goto sendpage_end;
 548
 549         /* Call the sk_stream functions to manage the sndbuf mem. */
 550         while (size > 0) {
 551                 size_t copy, required_size;
 552
 553                 if (sk->sk_err) {
 554                         ret = -sk->sk_err;
 555                         goto sendpage_end;
 556                 }
 557
 558                 full_record = false;
 559                 record_room = TLS_MAX_PAYLOAD_SIZE - ctx->sg_plaintext_size;
 560                 copy = size;
 561                 if (copy >= record_room) {
 562                         copy = record_room;
 563                         full_record = true;
 564                 }
 565                 required_size = ctx->sg_plaintext_size + copy +
 566                               tls_ctx->tx.overhead_size;
 567
 568                 if (!sk_stream_memory_free(sk))
 569                         goto wait_for_sndbuf;
 570 alloc_payload:
 571                 ret = alloc_encrypted_sg(sk, required_size);
 572                 if (ret) {
 573                         if (ret != -ENOSPC)
 574                                 goto wait_for_memory;
 575
 576                         /* Adjust copy according to the amount that was
 577                          * actually allocated. The difference is due
 578                          * to max sg elements limit
 579                          */
 580                         copy -= required_size - ctx->sg_plaintext_size;
 581                         full_record = true;
 582                 }
 583
 584                 get_page(page);
 585                 sg = ctx->sg_plaintext_data + ctx->sg_plaintext_num_elem;
 586                 sg_set_page(sg, page, copy, offset);
 587                 sg_unmark_end(sg);
 588
 589                 ctx->sg_plaintext_num_elem++;
 590
 591                 sk_mem_charge(sk, copy);
 592                 offset += copy;
 593                 size -= copy;
 594                 ctx->sg_plaintext_size += copy;
 595                 tls_ctx->pending_open_record_frags = ctx->sg_plaintext_num_elem;
 596
 597                 if (full_record || eor ||
 598                     ctx->sg_plaintext_num_elem ==
 599                     ARRAY_SIZE(ctx->sg_plaintext_data)) {
 600 push_record:
 601                         ret = tls_push_record(sk, flags, record_type);
 602                         if (ret) {
 603                                 if (ret == -ENOMEM)
 604                                         goto wait_for_memory;
 605
 606                                 goto sendpage_end;
 607                         }
 608                 }
 609                 continue;
 610 wait_for_sndbuf:
 611                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 612 wait_for_memory:
 613                 ret = sk_stream_wait_memory(sk, &timeo);
 614                 if (ret) {
 615                         trim_both_sgl(sk, ctx->sg_plaintext_size);
 616                         goto sendpage_end;
 617                 }
 618
 619                 if (tls_is_pending_closed_record(tls_ctx))
 620                         goto push_record;
 621
 622                 goto alloc_payload;
 623         }
 624
 625 sendpage_end:
 626         if (orig_size > size)
 627                 ret = orig_size - size;
 628         else
 629                 ret = sk_stream_error(sk, flags, ret);
 630
 631         release_sock(sk);
 632         return ret;
 633 }
 634
 635 static struct sk_buff *tls_wait_data(struct sock *sk, int flags,
 636                                      long timeo, int *err)
 637 {
 638         struct tls_context *tls_ctx = tls_get_ctx(sk);
 639         struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
 640         struct sk_buff *skb;
 641         DEFINE_WAIT_FUNC(wait, woken_wake_function);
 642
 643         while (!(skb = ctx->recv_pkt)) {
 644                 if (sk->sk_err) {
 645                         *err = sock_error(sk);
 646                         return NULL;
 647                 }
 648
 649                 if (sk->sk_shutdown & RCV_SHUTDOWN)
 650                         return NULL;
 651
 652                 if (sock_flag(sk, SOCK_DONE))
 653                         return NULL;
 654
 655                 if ((flags & MSG_DONTWAIT) || !timeo) {
 656                         *err = -EAGAIN;
 657                         return NULL;
 658                 }
 659
 660                 add_wait_queue(sk_sleep(sk), &wait);
 661                 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
 662                 sk_wait_event(sk, &timeo, ctx->recv_pkt != skb, &wait);
 663                 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
 664                 remove_wait_queue(sk_sleep(sk), &wait);
 665
 666                 /* Handle signals */
 667                 if (signal_pending(current)) {
 668                         *err = sock_intr_errno(timeo);
 669                         return NULL;
 670                 }
 671         }
 672
 673         return skb;
 674 }
 675
 676 static int decrypt_skb(struct sock *sk, struct sk_buff *skb,
 677                        struct scatterlist *sgout)
 678 {
 679         struct tls_context *tls_ctx = tls_get_ctx(sk);
 680         struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
 681         char iv[TLS_CIPHER_AES_GCM_128_SALT_SIZE + MAX_IV_SIZE];
 682         struct scatterlist sgin_arr[MAX_SKB_FRAGS + 2];
 683         struct scatterlist *sgin = &sgin_arr[0];
 684         struct strp_msg *rxm = strp_msg(skb);
 685         int ret, nsg = ARRAY_SIZE(sgin_arr);
 686         struct sk_buff *unused;
 687
 688         ret = skb_copy_bits(skb, rxm->offset + TLS_HEADER_SIZE,
 689                             iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE,
 690                             tls_ctx->rx.iv_size);
 691         if (ret < 0)
 692                 return ret;
 693
 694         memcpy(iv, tls_ctx->rx.iv, TLS_CIPHER_AES_GCM_128_SALT_SIZE);
 695         if (!sgout) {
 696                 nsg = skb_cow_data(skb, 0, &unused) + 1;
 697                 sgin = kmalloc_array(nsg, sizeof(*sgin), sk->sk_allocation);
 698                 sgout = sgin;
 699         }
 700
 701         sg_init_table(sgin, nsg);
 702         sg_set_buf(&sgin[0], ctx->rx_aad_ciphertext, TLS_AAD_SPACE_SIZE);
 703
 704         nsg = skb_to_sgvec(skb, &sgin[1],
 705                            rxm->offset + tls_ctx->rx.prepend_size,
 706                            rxm->full_len - tls_ctx->rx.prepend_size);
 707         if (nsg < 0) {
 708                 ret = nsg;
 709                 goto out;
 710         }
 711
 712         tls_make_aad(ctx->rx_aad_ciphertext,
 713                      rxm->full_len - tls_ctx->rx.overhead_size,
 714                      tls_ctx->rx.rec_seq,
 715                      tls_ctx->rx.rec_seq_size,
 716                      ctx->control);
 717
 718         ret = tls_do_decryption(sk, sgin, sgout, iv,
 719                                 rxm->full_len - tls_ctx->rx.overhead_size,
 720                                 skb, sk->sk_allocation);
 721
 722 out:
 723         if (sgin != &sgin_arr[0])
 724                 kfree(sgin);
 725
 726         return ret;
 727 }
 728
 729 static bool tls_sw_advance_skb(struct sock *sk, struct sk_buff *skb,
 730                                unsigned int len)
 731 {
 732         struct tls_context *tls_ctx = tls_get_ctx(sk);
 733         struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
 734         struct strp_msg *rxm = strp_msg(skb);
 735
 736         if (len < rxm->full_len) {
 737                 rxm->offset += len;
 738                 rxm->full_len -= len;
 739
 740                 return false;
 741         }
 742
 743         /* Finished with message */
 744         ctx->recv_pkt = NULL;
 745         kfree_skb(skb);
 746         __strp_unpause(&ctx->strp);
 747
 748         return true;
 749 }
 750
 751 int tls_sw_recvmsg(struct sock *sk,
 752                    struct msghdr *msg,
 753                    size_t len,
 754                    int nonblock,
 755                    int flags,
 756                    int *addr_len)
 757 {
 758         struct tls_context *tls_ctx = tls_get_ctx(sk);
 759         struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
 760         unsigned char control;
 761         struct strp_msg *rxm;
 762         struct sk_buff *skb;
 763         ssize_t copied = 0;
 764         bool cmsg = false;
 765         int target, err = 0;
 766         long timeo;
 767
 768         flags |= nonblock;
 769
 770         if (unlikely(flags & MSG_ERRQUEUE))
 771                 return sock_recv_errqueue(sk, msg, len, SOL_IP, IP_RECVERR);
 772
 773         lock_sock(sk);
 774
 775         target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
 776         timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
 777         do {
 778                 bool zc = false;
 779                 int chunk = 0;
 780
 781                 skb = tls_wait_data(sk, flags, timeo, &err);
 782                 if (!skb)
 783                         goto recv_end;
 784
 785                 rxm = strp_msg(skb);
 786                 if (!cmsg) {
 787                         int cerr;
 788
 789                         cerr = put_cmsg(msg, SOL_TLS, TLS_GET_RECORD_TYPE,
 790                                         sizeof(ctx->control), &ctx->control);
 791                         cmsg = true;
 792                         control = ctx->control;
 793                         if (ctx->control != TLS_RECORD_TYPE_DATA) {
 794                                 if (cerr || msg->msg_flags & MSG_CTRUNC) {
 795                                         err = -EIO;
 796                                         goto recv_end;
 797                                 }
 798                         }
 799                 } else if (control != ctx->control) {
 800                         goto recv_end;
 801                 }
 802
 803                 if (!ctx->decrypted) {
 804                         int page_count;
 805                         int to_copy;
 806
 807                         page_count = iov_iter_npages(&msg->msg_iter,
 808                                                      MAX_SKB_FRAGS);
 809                         to_copy = rxm->full_len - tls_ctx->rx.overhead_size;
 810                         if (to_copy <= len && page_count < MAX_SKB_FRAGS &&
 811                             likely(!(flags & MSG_PEEK)))  {
 812                                 struct scatterlist sgin[MAX_SKB_FRAGS + 1];
 813                                 int pages = 0;
 814
 815                                 zc = true;
 816                                 sg_init_table(sgin, MAX_SKB_FRAGS + 1);
 817                                 sg_set_buf(&sgin[0], ctx->rx_aad_plaintext,
 818                                            TLS_AAD_SPACE_SIZE);
 819
 820                                 err = zerocopy_from_iter(sk, &msg->msg_iter,
 821                                                          to_copy, &pages,
 822                                                          &chunk, &sgin[1],
 823                                                          MAX_SKB_FRAGS, false);
 824                                 if (err < 0)
 825                                         goto fallback_to_reg_recv;
 826
 827                                 err = decrypt_skb(sk, skb, sgin);
 828                                 for (; pages > 0; pages--)
 829                                         put_page(sg_page(&sgin[pages]));
 830                                 if (err < 0) {
 831                                         tls_err_abort(sk, EBADMSG);
 832                                         goto recv_end;
 833                                 }
 834                         } else {
 835 fallback_to_reg_recv:
 836                                 err = decrypt_skb(sk, skb, NULL);
 837                                 if (err < 0) {
 838                                         tls_err_abort(sk, EBADMSG);
 839                                         goto recv_end;
 840                                 }
 841                         }
 842                         ctx->decrypted = true;
 843                 }
 844
 845                 if (!zc) {
 846                         chunk = min_t(unsigned int, rxm->full_len, len);
 847                         err = skb_copy_datagram_msg(skb, rxm->offset, msg,
 848                                                     chunk);
 849                         if (err < 0)
 850                                 goto recv_end;
 851                 }
 852
 853                 copied += chunk;
 854                 len -= chunk;
 855                 if (likely(!(flags & MSG_PEEK))) {
 856                         u8 control = ctx->control;
 857
 858                         if (tls_sw_advance_skb(sk, skb, chunk)) {
 859                                 /* Return full control message to
 860                                  * userspace before trying to parse
 861                                  * another message type
 862                                  */
 863                                 msg->msg_flags |= MSG_EOR;
 864                                 if (control != TLS_RECORD_TYPE_DATA)
 865                                         goto recv_end;
 866                         }
 867                 }
 868                 /* If we have a new message from strparser, continue now. */
 869                 if (copied >= target && !ctx->recv_pkt)
 870                         break;
 871         } while (len);
 872
 873 recv_end:
 874         release_sock(sk);
 875         return copied ? : err;
 876 }
 877
 878 ssize_t tls_sw_splice_read(struct socket *sock,  loff_t *ppos,
 879                            struct pipe_inode_info *pipe,
 880                            size_t len, unsigned int flags)
 881 {
 882         struct tls_context *tls_ctx = tls_get_ctx(sock->sk);
 883         struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
 884         struct strp_msg *rxm = NULL;
 885         struct sock *sk = sock->sk;
 886         struct sk_buff *skb;
 887         ssize_t copied = 0;
 888         int err = 0;
 889         long timeo;
 890         int chunk;
 891
 892         lock_sock(sk);
 893
 894         timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
 895
 896         skb = tls_wait_data(sk, flags, timeo, &err);
 897         if (!skb)
 898                 goto splice_read_end;
 899
 900         /* splice does not support reading control messages */
 901         if (ctx->control != TLS_RECORD_TYPE_DATA) {
 902                 err = -ENOTSUPP;
 903                 goto splice_read_end;
 904         }
 905
 906         if (!ctx->decrypted) {
 907                 err = decrypt_skb(sk, skb, NULL);
 908
 909                 if (err < 0) {
 910                         tls_err_abort(sk, EBADMSG);
 911                         goto splice_read_end;
 912                 }
 913                 ctx->decrypted = true;
 914         }
 915         rxm = strp_msg(skb);
 916
 917         chunk = min_t(unsigned int, rxm->full_len, len);
 918         copied = skb_splice_bits(skb, sk, rxm->offset, pipe, chunk, flags);
 919         if (copied < 0)
 920                 goto splice_read_end;
 921
 922         if (likely(!(flags & MSG_PEEK)))
 923                 tls_sw_advance_skb(sk, skb, copied);
 924
 925 splice_read_end:
 926         release_sock(sk);
 927         return copied ? : err;
 928 }
 929
 930 unsigned int tls_sw_poll(struct file *file, struct socket *sock,
 931                          struct poll_table_struct *wait)
 932 {
 933         unsigned int ret;
 934         struct sock *sk = sock->sk;
 935         struct tls_context *tls_ctx = tls_get_ctx(sk);
 936         struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
 937
 938         /* Grab POLLOUT and POLLHUP from the underlying socket */
 939         ret = ctx->sk_poll(file, sock, wait);
 940
 941         /* Clear POLLIN bits, and set based on recv_pkt */
 942         ret &= ~(POLLIN | POLLRDNORM);
 943         if (ctx->recv_pkt)
 944                 ret |= POLLIN | POLLRDNORM;
 945
 946         return ret;
 947 }
 948
 949 static int tls_read_size(struct strparser *strp, struct sk_buff *skb)
 950 {
 951         struct tls_context *tls_ctx = tls_get_ctx(strp->sk);
 952         struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
 953         char header[tls_ctx->rx.prepend_size];
 954         struct strp_msg *rxm = strp_msg(skb);
 955         size_t cipher_overhead;
 956         size_t data_len = 0;
 957         int ret;
 958
 959         /* Verify that we have a full TLS header, or wait for more data */
 960         if (rxm->offset + tls_ctx->rx.prepend_size > skb->len)
 961                 return 0;
 962
 963         /* Linearize header to local buffer */
 964         ret = skb_copy_bits(skb, rxm->offset, header, tls_ctx->rx.prepend_size);
 965
 966         if (ret < 0)
 967                 goto read_failure;
 968
 969         ctx->control = header[0];
 970
 971         data_len = ((header[4] & 0xFF) | (header[3] << 8));
 972
 973         cipher_overhead = tls_ctx->rx.tag_size + tls_ctx->rx.iv_size;
 974
 975         if (data_len > TLS_MAX_PAYLOAD_SIZE + cipher_overhead) {
 976                 ret = -EMSGSIZE;
 977                 goto read_failure;
 978         }
 979         if (data_len < cipher_overhead) {
 980                 ret = -EBADMSG;
 981                 goto read_failure;
 982         }
 983
 984         if (header[1] != TLS_VERSION_MINOR(tls_ctx->crypto_recv.version) ||
 985             header[2] != TLS_VERSION_MAJOR(tls_ctx->crypto_recv.version)) {
 986                 ret = -EINVAL;
 987                 goto read_failure;
 988         }
 989
 990         return data_len + TLS_HEADER_SIZE;
 991
 992 read_failure:
 993         tls_err_abort(strp->sk, ret);
 994
 995         return ret;
 996 }
 997
 998 static void tls_queue(struct strparser *strp, struct sk_buff *skb)
 999 {
1000         struct tls_context *tls_ctx = tls_get_ctx(strp->sk);
1001         struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1002         struct strp_msg *rxm;
1003
1004         rxm = strp_msg(skb);
1005
1006         ctx->decrypted = false;
1007
1008         ctx->recv_pkt = skb;
1009         strp_pause(strp);
1010
1011         strp->sk->sk_state_change(strp->sk);
1012 }
1013
1014 static void tls_data_ready(struct sock *sk)
1015 {
1016         struct tls_context *tls_ctx = tls_get_ctx(sk);
1017         struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1018
1019         strp_data_ready(&ctx->strp);
1020 }
1021
1022 void tls_sw_free_resources_tx(struct sock *sk)
1023 {
1024         struct tls_context *tls_ctx = tls_get_ctx(sk);
1025         struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
1026
1027         if (ctx->aead_send)
1028                 crypto_free_aead(ctx->aead_send);
1029         tls_free_both_sg(sk);
1030
1031         kfree(ctx);
1032 }
1033
1034 void tls_sw_free_resources_rx(struct sock *sk)
1035 {
1036         struct tls_context *tls_ctx = tls_get_ctx(sk);
1037         struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1038
1039         if (ctx->aead_recv) {
1040                 if (ctx->recv_pkt) {
1041                         kfree_skb(ctx->recv_pkt);
1042                         ctx->recv_pkt = NULL;
1043                 }
1044                 crypto_free_aead(ctx->aead_recv);
1045                 strp_stop(&ctx->strp);
1046                 write_lock_bh(&sk->sk_callback_lock);
1047                 sk->sk_data_ready = ctx->saved_data_ready;
1048                 write_unlock_bh(&sk->sk_callback_lock);
1049                 release_sock(sk);
1050                 strp_done(&ctx->strp);
1051                 lock_sock(sk);
1052         }
1053
1054         kfree(ctx);
1055 }
1056
1057 int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx)
1058 {
1059         char keyval[TLS_CIPHER_AES_GCM_128_KEY_SIZE];
1060         struct tls_crypto_info *crypto_info;
1061         struct tls12_crypto_info_aes_gcm_128 *gcm_128_info;
1062         struct tls_sw_context_tx *sw_ctx_tx = NULL;
1063         struct tls_sw_context_rx *sw_ctx_rx = NULL;
1064         struct cipher_context *cctx;
1065         struct crypto_aead **aead;
1066         struct strp_callbacks cb;
1067         u16 nonce_size, tag_size, iv_size, rec_seq_size;
1068         char *iv, *rec_seq;
1069         int rc = 0;
1070
1071         if (!ctx) {
1072                 rc = -EINVAL;
1073                 goto out;
1074         }
1075
1076         if (tx) {
1077                 sw_ctx_tx = kzalloc(sizeof(*sw_ctx_tx), GFP_KERNEL);
1078                 if (!sw_ctx_tx) {
1079                         rc = -ENOMEM;
1080                         goto out;
1081                 }
1082                 crypto_init_wait(&sw_ctx_tx->async_wait);
1083                 ctx->priv_ctx_tx = sw_ctx_tx;
1084         } else {
1085                 sw_ctx_rx = kzalloc(sizeof(*sw_ctx_rx), GFP_KERNEL);
1086                 if (!sw_ctx_rx) {
1087                         rc = -ENOMEM;
1088                         goto out;
1089                 }
1090                 crypto_init_wait(&sw_ctx_rx->async_wait);
1091                 ctx->priv_ctx_rx = sw_ctx_rx;
1092         }
1093
1094         if (tx) {
1095                 crypto_info = &ctx->crypto_send;
1096                 cctx = &ctx->tx;
1097                 aead = &sw_ctx_tx->aead_send;
1098         } else {
1099                 crypto_info = &ctx->crypto_recv;
1100                 cctx = &ctx->rx;
1101                 aead = &sw_ctx_rx->aead_recv;
1102         }
1103
1104         switch (crypto_info->cipher_type) {
1105         case TLS_CIPHER_AES_GCM_128: {
1106                 nonce_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
1107                 tag_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE;
1108                 iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
1109                 iv = ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->iv;
1110                 rec_seq_size = TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE;
1111                 rec_seq =
1112                  ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->rec_seq;
1113                 gcm_128_info =
1114                         (struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
1115                 break;
1116         }
1117         default:
1118                 rc = -EINVAL;
1119                 goto free_priv;
1120         }
1121
1122         /* Sanity-check the IV size for stack allocations. */
1123         if (iv_size > MAX_IV_SIZE) {
1124                 rc = -EINVAL;
1125                 goto free_priv;
1126         }
1127
1128         cctx->prepend_size = TLS_HEADER_SIZE + nonce_size;
1129         cctx->tag_size = tag_size;
1130         cctx->overhead_size = cctx->prepend_size + cctx->tag_size;
1131         cctx->iv_size = iv_size;
1132         cctx->iv = kmalloc(iv_size + TLS_CIPHER_AES_GCM_128_SALT_SIZE,
1133                            GFP_KERNEL);
1134         if (!cctx->iv) {
1135                 rc = -ENOMEM;
1136                 goto free_priv;
1137         }
1138         memcpy(cctx->iv, gcm_128_info->salt, TLS_CIPHER_AES_GCM_128_SALT_SIZE);
1139         memcpy(cctx->iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE, iv, iv_size);
1140         cctx->rec_seq_size = rec_seq_size;
1141         cctx->rec_seq = kmalloc(rec_seq_size, GFP_KERNEL);
1142         if (!cctx->rec_seq) {
1143                 rc = -ENOMEM;
1144                 goto free_iv;
1145         }
1146         memcpy(cctx->rec_seq, rec_seq, rec_seq_size);
1147
1148         if (sw_ctx_tx) {
1149                 sg_init_table(sw_ctx_tx->sg_encrypted_data,
1150                               ARRAY_SIZE(sw_ctx_tx->sg_encrypted_data));
1151                 sg_init_table(sw_ctx_tx->sg_plaintext_data,
1152                               ARRAY_SIZE(sw_ctx_tx->sg_plaintext_data));
1153
1154                 sg_init_table(sw_ctx_tx->sg_aead_in, 2);
1155                 sg_set_buf(&sw_ctx_tx->sg_aead_in[0], sw_ctx_tx->aad_space,
1156                            sizeof(sw_ctx_tx->aad_space));
1157                 sg_unmark_end(&sw_ctx_tx->sg_aead_in[1]);
1158                 sg_chain(sw_ctx_tx->sg_aead_in, 2,
1159                          sw_ctx_tx->sg_plaintext_data);
1160                 sg_init_table(sw_ctx_tx->sg_aead_out, 2);
1161                 sg_set_buf(&sw_ctx_tx->sg_aead_out[0], sw_ctx_tx->aad_space,
1162                            sizeof(sw_ctx_tx->aad_space));
1163                 sg_unmark_end(&sw_ctx_tx->sg_aead_out[1]);
1164                 sg_chain(sw_ctx_tx->sg_aead_out, 2,
1165                          sw_ctx_tx->sg_encrypted_data);
1166         }
1167
1168         if (!*aead) {
1169                 *aead = crypto_alloc_aead("gcm(aes)", 0, 0);
1170                 if (IS_ERR(*aead)) {
1171                         rc = PTR_ERR(*aead);
1172                         *aead = NULL;
1173                         goto free_rec_seq;
1174                 }
1175         }
1176
1177         ctx->push_pending_record = tls_sw_push_pending_record;
1178
1179         memcpy(keyval, gcm_128_info->key, TLS_CIPHER_AES_GCM_128_KEY_SIZE);
1180
1181         rc = crypto_aead_setkey(*aead, keyval,
1182                                 TLS_CIPHER_AES_GCM_128_KEY_SIZE);
1183         if (rc)
1184                 goto free_aead;
1185
1186         rc = crypto_aead_setauthsize(*aead, cctx->tag_size);
1187         if (rc)
1188                 goto free_aead;
1189
1190         if (sw_ctx_rx) {
1191                 /* Set up strparser */
1192                 memset(&cb, 0, sizeof(cb));
1193                 cb.rcv_msg = tls_queue;
1194                 cb.parse_msg = tls_read_size;
1195
1196                 strp_init(&sw_ctx_rx->strp, sk, &cb);
1197
1198                 write_lock_bh(&sk->sk_callback_lock);
1199                 sw_ctx_rx->saved_data_ready = sk->sk_data_ready;
1200                 sk->sk_data_ready = tls_data_ready;
1201                 write_unlock_bh(&sk->sk_callback_lock);
1202
1203                 sw_ctx_rx->sk_poll = sk->sk_socket->ops->poll;
1204
1205                 strp_check_rcv(&sw_ctx_rx->strp);
1206         }
1207
1208         goto out;
1209
1210 free_aead:
1211         crypto_free_aead(*aead);
1212         *aead = NULL;
1213 free_rec_seq:
1214         kfree(cctx->rec_seq);
1215         cctx->rec_seq = NULL;
1216 free_iv:
1217         kfree(cctx->iv);
1218         cctx->iv = NULL;
1219 free_priv:
1220         if (tx) {
1221                 kfree(ctx->priv_ctx_tx);
1222                 ctx->priv_ctx_tx = NULL;
1223         } else {
1224                 kfree(ctx->priv_ctx_rx);
1225                 ctx->priv_ctx_rx = NULL;
1226         }
1227 out:
1228         return rc;
1229 }