fs/ceph/mds_client.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <linux/ceph/ceph_debug.h>
   3
   4 #include <linux/fs.h>
   5 #include <linux/wait.h>
   6 #include <linux/slab.h>
   7 #include <linux/gfp.h>
   8 #include <linux/sched.h>
   9 #include <linux/debugfs.h>
  10 #include <linux/seq_file.h>
  11 #include <linux/ratelimit.h>
  12 #include <linux/bits.h>
  13 #include <linux/ktime.h>
  14 #include <linux/bitmap.h>
  15 #include <linux/mnt_idmapping.h>
  16
  17 #include "super.h"
  18 #include "mds_client.h"
  19 #include "crypto.h"
  20
  21 #include <linux/ceph/ceph_features.h>
  22 #include <linux/ceph/messenger.h>
  23 #include <linux/ceph/decode.h>
  24 #include <linux/ceph/pagelist.h>
  25 #include <linux/ceph/auth.h>
  26 #include <linux/ceph/debugfs.h>
  27
  28 #define RECONNECT_MAX_SIZE (INT_MAX - PAGE_SIZE)
  29
  30 /*
  31  * A cluster of MDS (metadata server) daemons is responsible for
  32  * managing the file system namespace (the directory hierarchy and
  33  * inodes) and for coordinating shared access to storage.  Metadata is
  34  * partitioning hierarchically across a number of servers, and that
  35  * partition varies over time as the cluster adjusts the distribution
  36  * in order to balance load.
  37  *
  38  * The MDS client is primarily responsible to managing synchronous
  39  * metadata requests for operations like open, unlink, and so forth.
  40  * If there is a MDS failure, we find out about it when we (possibly
  41  * request and) receive a new MDS map, and can resubmit affected
  42  * requests.
  43  *
  44  * For the most part, though, we take advantage of a lossless
  45  * communications channel to the MDS, and do not need to worry about
  46  * timing out or resubmitting requests.
  47  *
  48  * We maintain a stateful "session" with each MDS we interact with.
  49  * Within each session, we sent periodic heartbeat messages to ensure
  50  * any capabilities or leases we have been issues remain valid.  If
  51  * the session times out and goes stale, our leases and capabilities
  52  * are no longer valid.
  53  */
  54
  55 struct ceph_reconnect_state {
  56         struct ceph_mds_session *session;
  57         int nr_caps, nr_realms;
  58         struct ceph_pagelist *pagelist;
  59         unsigned msg_version;
  60         bool allow_multi;
  61 };
  62
  63 static void __wake_requests(struct ceph_mds_client *mdsc,
  64                             struct list_head *head);
  65 static void ceph_cap_release_work(struct work_struct *work);
  66 static void ceph_cap_reclaim_work(struct work_struct *work);
  67
  68 static const struct ceph_connection_operations mds_con_ops;
  69
  70
  71 /*
  72  * mds reply parsing
  73  */
  74
  75 static int parse_reply_info_quota(void **p, void *end,
  76                                   struct ceph_mds_reply_info_in *info)
  77 {
  78         u8 struct_v, struct_compat;
  79         u32 struct_len;
  80
  81         ceph_decode_8_safe(p, end, struct_v, bad);
  82         ceph_decode_8_safe(p, end, struct_compat, bad);
  83         /* struct_v is expected to be >= 1. we only
  84          * understand encoding with struct_compat == 1. */
  85         if (!struct_v || struct_compat != 1)
  86                 goto bad;
  87         ceph_decode_32_safe(p, end, struct_len, bad);
  88         ceph_decode_need(p, end, struct_len, bad);
  89         end = *p + struct_len;
  90         ceph_decode_64_safe(p, end, info->max_bytes, bad);
  91         ceph_decode_64_safe(p, end, info->max_files, bad);
  92         *p = end;
  93         return 0;
  94 bad:
  95         return -EIO;
  96 }
  97
  98 /*
  99  * parse individual inode info
 100  */
 101 static int parse_reply_info_in(void **p, void *end,
 102                                struct ceph_mds_reply_info_in *info,
 103                                u64 features)
 104 {
 105         int err = 0;
 106         u8 struct_v = 0;
 107
 108         if (features == (u64)-1) {
 109                 u32 struct_len;
 110                 u8 struct_compat;
 111                 ceph_decode_8_safe(p, end, struct_v, bad);
 112                 ceph_decode_8_safe(p, end, struct_compat, bad);
 113                 /* struct_v is expected to be >= 1. we only understand
 114                  * encoding with struct_compat == 1. */
 115                 if (!struct_v || struct_compat != 1)
 116                         goto bad;
 117                 ceph_decode_32_safe(p, end, struct_len, bad);
 118                 ceph_decode_need(p, end, struct_len, bad);
 119                 end = *p + struct_len;
 120         }
 121
 122         ceph_decode_need(p, end, sizeof(struct ceph_mds_reply_inode), bad);
 123         info->in = *p;
 124         *p += sizeof(struct ceph_mds_reply_inode) +
 125                 sizeof(*info->in->fragtree.splits) *
 126                 le32_to_cpu(info->in->fragtree.nsplits);
 127
 128         ceph_decode_32_safe(p, end, info->symlink_len, bad);
 129         ceph_decode_need(p, end, info->symlink_len, bad);
 130         info->symlink = *p;
 131         *p += info->symlink_len;
 132
 133         ceph_decode_copy_safe(p, end, &info->dir_layout,
 134                               sizeof(info->dir_layout), bad);
 135         ceph_decode_32_safe(p, end, info->xattr_len, bad);
 136         ceph_decode_need(p, end, info->xattr_len, bad);
 137         info->xattr_data = *p;
 138         *p += info->xattr_len;
 139
 140         if (features == (u64)-1) {
 141                 /* inline data */
 142                 ceph_decode_64_safe(p, end, info->inline_version, bad);
 143                 ceph_decode_32_safe(p, end, info->inline_len, bad);
 144                 ceph_decode_need(p, end, info->inline_len, bad);
 145                 info->inline_data = *p;
 146                 *p += info->inline_len;
 147                 /* quota */
 148                 err = parse_reply_info_quota(p, end, info);
 149                 if (err < 0)
 150                         goto out_bad;
 151                 /* pool namespace */
 152                 ceph_decode_32_safe(p, end, info->pool_ns_len, bad);
 153                 if (info->pool_ns_len > 0) {
 154                         ceph_decode_need(p, end, info->pool_ns_len, bad);
 155                         info->pool_ns_data = *p;
 156                         *p += info->pool_ns_len;
 157                 }
 158
 159                 /* btime */
 160                 ceph_decode_need(p, end, sizeof(info->btime), bad);
 161                 ceph_decode_copy(p, &info->btime, sizeof(info->btime));
 162
 163                 /* change attribute */
 164                 ceph_decode_64_safe(p, end, info->change_attr, bad);
 165
 166                 /* dir pin */
 167                 if (struct_v >= 2) {
 168                         ceph_decode_32_safe(p, end, info->dir_pin, bad);
 169                 } else {
 170                         info->dir_pin = -ENODATA;
 171                 }
 172
 173                 /* snapshot birth time, remains zero for v<=2 */
 174                 if (struct_v >= 3) {
 175                         ceph_decode_need(p, end, sizeof(info->snap_btime), bad);
 176                         ceph_decode_copy(p, &info->snap_btime,
 177                                          sizeof(info->snap_btime));
 178                 } else {
 179                         memset(&info->snap_btime, 0, sizeof(info->snap_btime));
 180                 }
 181
 182                 /* snapshot count, remains zero for v<=3 */
 183                 if (struct_v >= 4) {
 184                         ceph_decode_64_safe(p, end, info->rsnaps, bad);
 185                 } else {
 186                         info->rsnaps = 0;
 187                 }
 188
 189                 if (struct_v >= 5) {
 190                         u32 alen;
 191
 192                         ceph_decode_32_safe(p, end, alen, bad);
 193
 194                         while (alen--) {
 195                                 u32 len;
 196
 197                                 /* key */
 198                                 ceph_decode_32_safe(p, end, len, bad);
 199                                 ceph_decode_skip_n(p, end, len, bad);
 200                                 /* value */
 201                                 ceph_decode_32_safe(p, end, len, bad);
 202                                 ceph_decode_skip_n(p, end, len, bad);
 203                         }
 204                 }
 205
 206                 /* fscrypt flag -- ignore */
 207                 if (struct_v >= 6)
 208                         ceph_decode_skip_8(p, end, bad);
 209
 210                 info->fscrypt_auth = NULL;
 211                 info->fscrypt_auth_len = 0;
 212                 info->fscrypt_file = NULL;
 213                 info->fscrypt_file_len = 0;
 214                 if (struct_v >= 7) {
 215                         ceph_decode_32_safe(p, end, info->fscrypt_auth_len, bad);
 216                         if (info->fscrypt_auth_len) {
 217                                 info->fscrypt_auth = kmalloc(info->fscrypt_auth_len,
 218                                                              GFP_KERNEL);
 219                                 if (!info->fscrypt_auth)
 220                                         return -ENOMEM;
 221                                 ceph_decode_copy_safe(p, end, info->fscrypt_auth,
 222                                                       info->fscrypt_auth_len, bad);
 223                         }
 224                         ceph_decode_32_safe(p, end, info->fscrypt_file_len, bad);
 225                         if (info->fscrypt_file_len) {
 226                                 info->fscrypt_file = kmalloc(info->fscrypt_file_len,
 227                                                              GFP_KERNEL);
 228                                 if (!info->fscrypt_file)
 229                                         return -ENOMEM;
 230                                 ceph_decode_copy_safe(p, end, info->fscrypt_file,
 231                                                       info->fscrypt_file_len, bad);
 232                         }
 233                 }
 234                 *p = end;
 235         } else {
 236                 /* legacy (unversioned) struct */
 237                 if (features & CEPH_FEATURE_MDS_INLINE_DATA) {
 238                         ceph_decode_64_safe(p, end, info->inline_version, bad);
 239                         ceph_decode_32_safe(p, end, info->inline_len, bad);
 240                         ceph_decode_need(p, end, info->inline_len, bad);
 241                         info->inline_data = *p;
 242                         *p += info->inline_len;
 243                 } else
 244                         info->inline_version = CEPH_INLINE_NONE;
 245
 246                 if (features & CEPH_FEATURE_MDS_QUOTA) {
 247                         err = parse_reply_info_quota(p, end, info);
 248                         if (err < 0)
 249                                 goto out_bad;
 250                 } else {
 251                         info->max_bytes = 0;
 252                         info->max_files = 0;
 253                 }
 254
 255                 info->pool_ns_len = 0;
 256                 info->pool_ns_data = NULL;
 257                 if (features & CEPH_FEATURE_FS_FILE_LAYOUT_V2) {
 258                         ceph_decode_32_safe(p, end, info->pool_ns_len, bad);
 259                         if (info->pool_ns_len > 0) {
 260                                 ceph_decode_need(p, end, info->pool_ns_len, bad);
 261                                 info->pool_ns_data = *p;
 262                                 *p += info->pool_ns_len;
 263                         }
 264                 }
 265
 266                 if (features & CEPH_FEATURE_FS_BTIME) {
 267                         ceph_decode_need(p, end, sizeof(info->btime), bad);
 268                         ceph_decode_copy(p, &info->btime, sizeof(info->btime));
 269                         ceph_decode_64_safe(p, end, info->change_attr, bad);
 270                 }
 271
 272                 info->dir_pin = -ENODATA;
 273                 /* info->snap_btime and info->rsnaps remain zero */
 274         }
 275         return 0;
 276 bad:
 277         err = -EIO;
 278 out_bad:
 279         return err;
 280 }
 281
 282 static int parse_reply_info_dir(void **p, void *end,
 283                                 struct ceph_mds_reply_dirfrag **dirfrag,
 284                                 u64 features)
 285 {
 286         if (features == (u64)-1) {
 287                 u8 struct_v, struct_compat;
 288                 u32 struct_len;
 289                 ceph_decode_8_safe(p, end, struct_v, bad);
 290                 ceph_decode_8_safe(p, end, struct_compat, bad);
 291                 /* struct_v is expected to be >= 1. we only understand
 292                  * encoding whose struct_compat == 1. */
 293                 if (!struct_v || struct_compat != 1)
 294                         goto bad;
 295                 ceph_decode_32_safe(p, end, struct_len, bad);
 296                 ceph_decode_need(p, end, struct_len, bad);
 297                 end = *p + struct_len;
 298         }
 299
 300         ceph_decode_need(p, end, sizeof(**dirfrag), bad);
 301         *dirfrag = *p;
 302         *p += sizeof(**dirfrag) + sizeof(u32) * le32_to_cpu((*dirfrag)->ndist);
 303         if (unlikely(*p > end))
 304                 goto bad;
 305         if (features == (u64)-1)
 306                 *p = end;
 307         return 0;
 308 bad:
 309         return -EIO;
 310 }
 311
 312 static int parse_reply_info_lease(void **p, void *end,
 313                                   struct ceph_mds_reply_lease **lease,
 314                                   u64 features, u32 *altname_len, u8 **altname)
 315 {
 316         u8 struct_v;
 317         u32 struct_len;
 318         void *lend;
 319
 320         if (features == (u64)-1) {
 321                 u8 struct_compat;
 322
 323                 ceph_decode_8_safe(p, end, struct_v, bad);
 324                 ceph_decode_8_safe(p, end, struct_compat, bad);
 325
 326                 /* struct_v is expected to be >= 1. we only understand
 327                  * encoding whose struct_compat == 1. */
 328                 if (!struct_v || struct_compat != 1)
 329                         goto bad;
 330
 331                 ceph_decode_32_safe(p, end, struct_len, bad);
 332         } else {
 333                 struct_len = sizeof(**lease);
 334                 *altname_len = 0;
 335                 *altname = NULL;
 336         }
 337
 338         lend = *p + struct_len;
 339         ceph_decode_need(p, end, struct_len, bad);
 340         *lease = *p;
 341         *p += sizeof(**lease);
 342
 343         if (features == (u64)-1) {
 344                 if (struct_v >= 2) {
 345                         ceph_decode_32_safe(p, end, *altname_len, bad);
 346                         ceph_decode_need(p, end, *altname_len, bad);
 347                         *altname = *p;
 348                         *p += *altname_len;
 349                 } else {
 350                         *altname = NULL;
 351                         *altname_len = 0;
 352                 }
 353         }
 354         *p = lend;
 355         return 0;
 356 bad:
 357         return -EIO;
 358 }
 359
 360 /*
 361  * parse a normal reply, which may contain a (dir+)dentry and/or a
 362  * target inode.
 363  */
 364 static int parse_reply_info_trace(void **p, void *end,
 365                                   struct ceph_mds_reply_info_parsed *info,
 366                                   u64 features)
 367 {
 368         int err;
 369
 370         if (info->head->is_dentry) {
 371                 err = parse_reply_info_in(p, end, &info->diri, features);
 372                 if (err < 0)
 373                         goto out_bad;
 374
 375                 err = parse_reply_info_dir(p, end, &info->dirfrag, features);
 376                 if (err < 0)
 377                         goto out_bad;
 378
 379                 ceph_decode_32_safe(p, end, info->dname_len, bad);
 380                 ceph_decode_need(p, end, info->dname_len, bad);
 381                 info->dname = *p;
 382                 *p += info->dname_len;
 383
 384                 err = parse_reply_info_lease(p, end, &info->dlease, features,
 385                                              &info->altname_len, &info->altname);
 386                 if (err < 0)
 387                         goto out_bad;
 388         }
 389
 390         if (info->head->is_target) {
 391                 err = parse_reply_info_in(p, end, &info->targeti, features);
 392                 if (err < 0)
 393                         goto out_bad;
 394         }
 395
 396         if (unlikely(*p != end))
 397                 goto bad;
 398         return 0;
 399
 400 bad:
 401         err = -EIO;
 402 out_bad:
 403         pr_err("problem parsing mds trace %d\n", err);
 404         return err;
 405 }
 406
 407 /*
 408  * parse readdir results
 409  */
 410 static int parse_reply_info_readdir(void **p, void *end,
 411                                     struct ceph_mds_request *req,
 412                                     u64 features)
 413 {
 414         struct ceph_mds_reply_info_parsed *info = &req->r_reply_info;
 415         struct ceph_client *cl = req->r_mdsc->fsc->client;
 416         u32 num, i = 0;
 417         int err;
 418
 419         err = parse_reply_info_dir(p, end, &info->dir_dir, features);
 420         if (err < 0)
 421                 goto out_bad;
 422
 423         ceph_decode_need(p, end, sizeof(num) + 2, bad);
 424         num = ceph_decode_32(p);
 425         {
 426                 u16 flags = ceph_decode_16(p);
 427                 info->dir_end = !!(flags & CEPH_READDIR_FRAG_END);
 428                 info->dir_complete = !!(flags & CEPH_READDIR_FRAG_COMPLETE);
 429                 info->hash_order = !!(flags & CEPH_READDIR_HASH_ORDER);
 430                 info->offset_hash = !!(flags & CEPH_READDIR_OFFSET_HASH);
 431         }
 432         if (num == 0)
 433                 goto done;
 434
 435         BUG_ON(!info->dir_entries);
 436         if ((unsigned long)(info->dir_entries + num) >
 437             (unsigned long)info->dir_entries + info->dir_buf_size) {
 438                 pr_err_client(cl, "dir contents are larger than expected\n");
 439                 WARN_ON(1);
 440                 goto bad;
 441         }
 442
 443         info->dir_nr = num;
 444         while (num) {
 445                 struct inode *inode = d_inode(req->r_dentry);
 446                 struct ceph_inode_info *ci = ceph_inode(inode);
 447                 struct ceph_mds_reply_dir_entry *rde = info->dir_entries + i;
 448                 struct fscrypt_str tname = FSTR_INIT(NULL, 0);
 449                 struct fscrypt_str oname = FSTR_INIT(NULL, 0);
 450                 struct ceph_fname fname;
 451                 u32 altname_len, _name_len;
 452                 u8 *altname, *_name;
 453
 454                 /* dentry */
 455                 ceph_decode_32_safe(p, end, _name_len, bad);
 456                 ceph_decode_need(p, end, _name_len, bad);
 457                 _name = *p;
 458                 *p += _name_len;
 459                 doutc(cl, "parsed dir dname '%.*s'\n", _name_len, _name);
 460
 461                 if (info->hash_order)
 462                         rde->raw_hash = ceph_str_hash(ci->i_dir_layout.dl_dir_hash,
 463                                                       _name, _name_len);
 464
 465                 /* dentry lease */
 466                 err = parse_reply_info_lease(p, end, &rde->lease, features,
 467                                              &altname_len, &altname);
 468                 if (err)
 469                         goto out_bad;
 470
 471                 /*
 472                  * Try to dencrypt the dentry names and update them
 473                  * in the ceph_mds_reply_dir_entry struct.
 474                  */
 475                 fname.dir = inode;
 476                 fname.name = _name;
 477                 fname.name_len = _name_len;
 478                 fname.ctext = altname;
 479                 fname.ctext_len = altname_len;
 480                 /*
 481                  * The _name_len maybe larger than altname_len, such as
 482                  * when the human readable name length is in range of
 483                  * (CEPH_NOHASH_NAME_MAX, CEPH_NOHASH_NAME_MAX + SHA256_DIGEST_SIZE),
 484                  * then the copy in ceph_fname_to_usr will corrupt the
 485                  * data if there has no encryption key.
 486                  *
 487                  * Just set the no_copy flag and then if there has no
 488                  * encryption key the oname.name will be assigned to
 489                  * _name always.
 490                  */
 491                 fname.no_copy = true;
 492                 if (altname_len == 0) {
 493                         /*
 494                          * Set tname to _name, and this will be used
 495                          * to do the base64_decode in-place. It's
 496                          * safe because the decoded string should
 497                          * always be shorter, which is 3/4 of origin
 498                          * string.
 499                          */
 500                         tname.name = _name;
 501
 502                         /*
 503                          * Set oname to _name too, and this will be
 504                          * used to do the dencryption in-place.
 505                          */
 506                         oname.name = _name;
 507                         oname.len = _name_len;
 508                 } else {
 509                         /*
 510                          * This will do the decryption only in-place
 511                          * from altname cryptext directly.
 512                          */
 513                         oname.name = altname;
 514                         oname.len = altname_len;
 515                 }
 516                 rde->is_nokey = false;
 517                 err = ceph_fname_to_usr(&fname, &tname, &oname, &rde->is_nokey);
 518                 if (err) {
 519                         pr_err_client(cl, "unable to decode %.*s, got %d\n",
 520                                       _name_len, _name, err);
 521                         goto out_bad;
 522                 }
 523                 rde->name = oname.name;
 524                 rde->name_len = oname.len;
 525
 526                 /* inode */
 527                 err = parse_reply_info_in(p, end, &rde->inode, features);
 528                 if (err < 0)
 529                         goto out_bad;
 530                 /* ceph_readdir_prepopulate() will update it */
 531                 rde->offset = 0;
 532                 i++;
 533                 num--;
 534         }
 535
 536 done:
 537         /* Skip over any unrecognized fields */
 538         *p = end;
 539         return 0;
 540
 541 bad:
 542         err = -EIO;
 543 out_bad:
 544         pr_err_client(cl, "problem parsing dir contents %d\n", err);
 545         return err;
 546 }
 547
 548 /*
 549  * parse fcntl F_GETLK results
 550  */
 551 static int parse_reply_info_filelock(void **p, void *end,
 552                                      struct ceph_mds_reply_info_parsed *info,
 553                                      u64 features)
 554 {
 555         if (*p + sizeof(*info->filelock_reply) > end)
 556                 goto bad;
 557
 558         info->filelock_reply = *p;
 559
 560         /* Skip over any unrecognized fields */
 561         *p = end;
 562         return 0;
 563 bad:
 564         return -EIO;
 565 }
 566
 567
 568 #if BITS_PER_LONG == 64
 569
 570 #define DELEGATED_INO_AVAILABLE         xa_mk_value(1)
 571
 572 static int ceph_parse_deleg_inos(void **p, void *end,
 573                                  struct ceph_mds_session *s)
 574 {
 575         struct ceph_client *cl = s->s_mdsc->fsc->client;
 576         u32 sets;
 577
 578         ceph_decode_32_safe(p, end, sets, bad);
 579         doutc(cl, "got %u sets of delegated inodes\n", sets);
 580         while (sets--) {
 581                 u64 start, len;
 582
 583                 ceph_decode_64_safe(p, end, start, bad);
 584                 ceph_decode_64_safe(p, end, len, bad);
 585
 586                 /* Don't accept a delegation of system inodes */
 587                 if (start < CEPH_INO_SYSTEM_BASE) {
 588                         pr_warn_ratelimited_client(cl,
 589                                 "ignoring reserved inode range delegation (start=0x%llx len=0x%llx)\n",
 590                                 start, len);
 591                         continue;
 592                 }
 593                 while (len--) {
 594                         int err = xa_insert(&s->s_delegated_inos, start++,
 595                                             DELEGATED_INO_AVAILABLE,
 596                                             GFP_KERNEL);
 597                         if (!err) {
 598                                 doutc(cl, "added delegated inode 0x%llx\n", start - 1);
 599                         } else if (err == -EBUSY) {
 600                                 pr_warn_client(cl,
 601                                         "MDS delegated inode 0x%llx more than once.\n",
 602                                         start - 1);
 603                         } else {
 604                                 return err;
 605                         }
 606                 }
 607         }
 608         return 0;
 609 bad:
 610         return -EIO;
 611 }
 612
 613 u64 ceph_get_deleg_ino(struct ceph_mds_session *s)
 614 {
 615         unsigned long ino;
 616         void *val;
 617
 618         xa_for_each(&s->s_delegated_inos, ino, val) {
 619                 val = xa_erase(&s->s_delegated_inos, ino);
 620                 if (val == DELEGATED_INO_AVAILABLE)
 621                         return ino;
 622         }
 623         return 0;
 624 }
 625
 626 int ceph_restore_deleg_ino(struct ceph_mds_session *s, u64 ino)
 627 {
 628         return xa_insert(&s->s_delegated_inos, ino, DELEGATED_INO_AVAILABLE,
 629                          GFP_KERNEL);
 630 }
 631 #else /* BITS_PER_LONG == 64 */
 632 /*
 633  * FIXME: xarrays can't handle 64-bit indexes on a 32-bit arch. For now, just
 634  * ignore delegated_inos on 32 bit arch. Maybe eventually add xarrays for top
 635  * and bottom words?
 636  */
 637 static int ceph_parse_deleg_inos(void **p, void *end,
 638                                  struct ceph_mds_session *s)
 639 {
 640         u32 sets;
 641
 642         ceph_decode_32_safe(p, end, sets, bad);
 643         if (sets)
 644                 ceph_decode_skip_n(p, end, sets * 2 * sizeof(__le64), bad);
 645         return 0;
 646 bad:
 647         return -EIO;
 648 }
 649
 650 u64 ceph_get_deleg_ino(struct ceph_mds_session *s)
 651 {
 652         return 0;
 653 }
 654
 655 int ceph_restore_deleg_ino(struct ceph_mds_session *s, u64 ino)
 656 {
 657         return 0;
 658 }
 659 #endif /* BITS_PER_LONG == 64 */
 660
 661 /*
 662  * parse create results
 663  */
 664 static int parse_reply_info_create(void **p, void *end,
 665                                   struct ceph_mds_reply_info_parsed *info,
 666                                   u64 features, struct ceph_mds_session *s)
 667 {
 668         int ret;
 669
 670         if (features == (u64)-1 ||
 671             (features & CEPH_FEATURE_REPLY_CREATE_INODE)) {
 672                 if (*p == end) {
 673                         /* Malformed reply? */
 674                         info->has_create_ino = false;
 675                 } else if (test_bit(CEPHFS_FEATURE_DELEG_INO, &s->s_features)) {
 676                         info->has_create_ino = true;
 677                         /* struct_v, struct_compat, and len */
 678                         ceph_decode_skip_n(p, end, 2 + sizeof(u32), bad);
 679                         ceph_decode_64_safe(p, end, info->ino, bad);
 680                         ret = ceph_parse_deleg_inos(p, end, s);
 681                         if (ret)
 682                                 return ret;
 683                 } else {
 684                         /* legacy */
 685                         ceph_decode_64_safe(p, end, info->ino, bad);
 686                         info->has_create_ino = true;
 687                 }
 688         } else {
 689                 if (*p != end)
 690                         goto bad;
 691         }
 692
 693         /* Skip over any unrecognized fields */
 694         *p = end;
 695         return 0;
 696 bad:
 697         return -EIO;
 698 }
 699
 700 static int parse_reply_info_getvxattr(void **p, void *end,
 701                                       struct ceph_mds_reply_info_parsed *info,
 702                                       u64 features)
 703 {
 704         u32 value_len;
 705
 706         ceph_decode_skip_8(p, end, bad); /* skip current version: 1 */
 707         ceph_decode_skip_8(p, end, bad); /* skip first version: 1 */
 708         ceph_decode_skip_32(p, end, bad); /* skip payload length */
 709
 710         ceph_decode_32_safe(p, end, value_len, bad);
 711
 712         if (value_len == end - *p) {
 713           info->xattr_info.xattr_value = *p;
 714           info->xattr_info.xattr_value_len = value_len;
 715           *p = end;
 716           return value_len;
 717         }
 718 bad:
 719         return -EIO;
 720 }
 721
 722 /*
 723  * parse extra results
 724  */
 725 static int parse_reply_info_extra(void **p, void *end,
 726                                   struct ceph_mds_request *req,
 727                                   u64 features, struct ceph_mds_session *s)
 728 {
 729         struct ceph_mds_reply_info_parsed *info = &req->r_reply_info;
 730         u32 op = le32_to_cpu(info->head->op);
 731
 732         if (op == CEPH_MDS_OP_GETFILELOCK)
 733                 return parse_reply_info_filelock(p, end, info, features);
 734         else if (op == CEPH_MDS_OP_READDIR || op == CEPH_MDS_OP_LSSNAP)
 735                 return parse_reply_info_readdir(p, end, req, features);
 736         else if (op == CEPH_MDS_OP_CREATE)
 737                 return parse_reply_info_create(p, end, info, features, s);
 738         else if (op == CEPH_MDS_OP_GETVXATTR)
 739                 return parse_reply_info_getvxattr(p, end, info, features);
 740         else
 741                 return -EIO;
 742 }
 743
 744 /*
 745  * parse entire mds reply
 746  */
 747 static int parse_reply_info(struct ceph_mds_session *s, struct ceph_msg *msg,
 748                             struct ceph_mds_request *req, u64 features)
 749 {
 750         struct ceph_mds_reply_info_parsed *info = &req->r_reply_info;
 751         struct ceph_client *cl = s->s_mdsc->fsc->client;
 752         void *p, *end;
 753         u32 len;
 754         int err;
 755
 756         info->head = msg->front.iov_base;
 757         p = msg->front.iov_base + sizeof(struct ceph_mds_reply_head);
 758         end = p + msg->front.iov_len - sizeof(struct ceph_mds_reply_head);
 759
 760         /* trace */
 761         ceph_decode_32_safe(&p, end, len, bad);
 762         if (len > 0) {
 763                 ceph_decode_need(&p, end, len, bad);
 764                 err = parse_reply_info_trace(&p, p+len, info, features);
 765                 if (err < 0)
 766                         goto out_bad;
 767         }
 768
 769         /* extra */
 770         ceph_decode_32_safe(&p, end, len, bad);
 771         if (len > 0) {
 772                 ceph_decode_need(&p, end, len, bad);
 773                 err = parse_reply_info_extra(&p, p+len, req, features, s);
 774                 if (err < 0)
 775                         goto out_bad;
 776         }
 777
 778         /* snap blob */
 779         ceph_decode_32_safe(&p, end, len, bad);
 780         info->snapblob_len = len;
 781         info->snapblob = p;
 782         p += len;
 783
 784         if (p != end)
 785                 goto bad;
 786         return 0;
 787
 788 bad:
 789         err = -EIO;
 790 out_bad:
 791         pr_err_client(cl, "mds parse_reply err %d\n", err);
 792         ceph_msg_dump(msg);
 793         return err;
 794 }
 795
 796 static void destroy_reply_info(struct ceph_mds_reply_info_parsed *info)
 797 {
 798         int i;
 799
 800         kfree(info->diri.fscrypt_auth);
 801         kfree(info->diri.fscrypt_file);
 802         kfree(info->targeti.fscrypt_auth);
 803         kfree(info->targeti.fscrypt_file);
 804         if (!info->dir_entries)
 805                 return;
 806
 807         for (i = 0; i < info->dir_nr; i++) {
 808                 struct ceph_mds_reply_dir_entry *rde = info->dir_entries + i;
 809
 810                 kfree(rde->inode.fscrypt_auth);
 811                 kfree(rde->inode.fscrypt_file);
 812         }
 813         free_pages((unsigned long)info->dir_entries, get_order(info->dir_buf_size));
 814 }
 815
 816 /*
 817  * In async unlink case the kclient won't wait for the first reply
 818  * from MDS and just drop all the links and unhash the dentry and then
 819  * succeeds immediately.
 820  *
 821  * For any new create/link/rename,etc requests followed by using the
 822  * same file names we must wait for the first reply of the inflight
 823  * unlink request, or the MDS possibly will fail these following
 824  * requests with -EEXIST if the inflight async unlink request was
 825  * delayed for some reasons.
 826  *
 827  * And the worst case is that for the none async openc request it will
 828  * successfully open the file if the CDentry hasn't been unlinked yet,
 829  * but later the previous delayed async unlink request will remove the
 830  * CDenty. That means the just created file is possiblly deleted later
 831  * by accident.
 832  *
 833  * We need to wait for the inflight async unlink requests to finish
 834  * when creating new files/directories by using the same file names.
 835  */
 836 int ceph_wait_on_conflict_unlink(struct dentry *dentry)
 837 {
 838         struct ceph_fs_client *fsc = ceph_sb_to_fs_client(dentry->d_sb);
 839         struct ceph_client *cl = fsc->client;
 840         struct dentry *pdentry = dentry->d_parent;
 841         struct dentry *udentry, *found = NULL;
 842         struct ceph_dentry_info *di;
 843         struct qstr dname;
 844         u32 hash = dentry->d_name.hash;
 845         int err;
 846
 847         dname.name = dentry->d_name.name;
 848         dname.len = dentry->d_name.len;
 849
 850         rcu_read_lock();
 851         hash_for_each_possible_rcu(fsc->async_unlink_conflict, di,
 852                                    hnode, hash) {
 853                 udentry = di->dentry;
 854
 855                 spin_lock(&udentry->d_lock);
 856                 if (udentry->d_name.hash != hash)
 857                         goto next;
 858                 if (unlikely(udentry->d_parent != pdentry))
 859                         goto next;
 860                 if (!hash_hashed(&di->hnode))
 861                         goto next;
 862
 863                 if (!test_bit(CEPH_DENTRY_ASYNC_UNLINK_BIT, &di->flags))
 864                         pr_warn_client(cl, "dentry %p:%pd async unlink bit is not set\n",
 865                                        dentry, dentry);
 866
 867                 if (!d_same_name(udentry, pdentry, &dname))
 868                         goto next;
 869
 870                 found = dget_dlock(udentry);
 871                 spin_unlock(&udentry->d_lock);
 872                 break;
 873 next:
 874                 spin_unlock(&udentry->d_lock);
 875         }
 876         rcu_read_unlock();
 877
 878         if (likely(!found))
 879                 return 0;
 880
 881         doutc(cl, "dentry %p:%pd conflict with old %p:%pd\n", dentry, dentry,
 882               found, found);
 883
 884         err = wait_on_bit(&di->flags, CEPH_DENTRY_ASYNC_UNLINK_BIT,
 885                           TASK_KILLABLE);
 886         dput(found);
 887         return err;
 888 }
 889
 890
 891 /*
 892  * sessions
 893  */
 894 const char *ceph_session_state_name(int s)
 895 {
 896         switch (s) {
 897         case CEPH_MDS_SESSION_NEW: return "new";
 898         case CEPH_MDS_SESSION_OPENING: return "opening";
 899         case CEPH_MDS_SESSION_OPEN: return "open";
 900         case CEPH_MDS_SESSION_HUNG: return "hung";
 901         case CEPH_MDS_SESSION_CLOSING: return "closing";
 902         case CEPH_MDS_SESSION_CLOSED: return "closed";
 903         case CEPH_MDS_SESSION_RESTARTING: return "restarting";
 904         case CEPH_MDS_SESSION_RECONNECTING: return "reconnecting";
 905         case CEPH_MDS_SESSION_REJECTED: return "rejected";
 906         default: return "???";
 907         }
 908 }
 909
 910 struct ceph_mds_session *ceph_get_mds_session(struct ceph_mds_session *s)
 911 {
 912         if (refcount_inc_not_zero(&s->s_ref))
 913                 return s;
 914         return NULL;
 915 }
 916
 917 void ceph_put_mds_session(struct ceph_mds_session *s)
 918 {
 919         if (IS_ERR_OR_NULL(s))
 920                 return;
 921
 922         if (refcount_dec_and_test(&s->s_ref)) {
 923                 if (s->s_auth.authorizer)
 924                         ceph_auth_destroy_authorizer(s->s_auth.authorizer);
 925                 WARN_ON(mutex_is_locked(&s->s_mutex));
 926                 xa_destroy(&s->s_delegated_inos);
 927                 kfree(s);
 928         }
 929 }
 930
 931 /*
 932  * called under mdsc->mutex
 933  */
 934 struct ceph_mds_session *__ceph_lookup_mds_session(struct ceph_mds_client *mdsc,
 935                                                    int mds)
 936 {
 937         if (mds >= mdsc->max_sessions || !mdsc->sessions[mds])
 938                 return NULL;
 939         return ceph_get_mds_session(mdsc->sessions[mds]);
 940 }
 941
 942 static bool __have_session(struct ceph_mds_client *mdsc, int mds)
 943 {
 944         if (mds >= mdsc->max_sessions || !mdsc->sessions[mds])
 945                 return false;
 946         else
 947                 return true;
 948 }
 949
 950 static int __verify_registered_session(struct ceph_mds_client *mdsc,
 951                                        struct ceph_mds_session *s)
 952 {
 953         if (s->s_mds >= mdsc->max_sessions ||
 954             mdsc->sessions[s->s_mds] != s)
 955                 return -ENOENT;
 956         return 0;
 957 }
 958
 959 /*
 960  * create+register a new session for given mds.
 961  * called under mdsc->mutex.
 962  */
 963 static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc,
 964                                                  int mds)
 965 {
 966         struct ceph_client *cl = mdsc->fsc->client;
 967         struct ceph_mds_session *s;
 968
 969         if (READ_ONCE(mdsc->fsc->mount_state) == CEPH_MOUNT_FENCE_IO)
 970                 return ERR_PTR(-EIO);
 971
 972         if (mds >= mdsc->mdsmap->possible_max_rank)
 973                 return ERR_PTR(-EINVAL);
 974
 975         s = kzalloc(sizeof(*s), GFP_NOFS);
 976         if (!s)
 977                 return ERR_PTR(-ENOMEM);
 978
 979         if (mds >= mdsc->max_sessions) {
 980                 int newmax = 1 << get_count_order(mds + 1);
 981                 struct ceph_mds_session **sa;
 982
 983                 doutc(cl, "realloc to %d\n", newmax);
 984                 sa = kcalloc(newmax, sizeof(void *), GFP_NOFS);
 985                 if (!sa)
 986                         goto fail_realloc;
 987                 if (mdsc->sessions) {
 988                         memcpy(sa, mdsc->sessions,
 989                                mdsc->max_sessions * sizeof(void *));
 990                         kfree(mdsc->sessions);
 991                 }
 992                 mdsc->sessions = sa;
 993                 mdsc->max_sessions = newmax;
 994         }
 995
 996         doutc(cl, "mds%d\n", mds);
 997         s->s_mdsc = mdsc;
 998         s->s_mds = mds;
 999         s->s_state = CEPH_MDS_SESSION_NEW;
1000         mutex_init(&s->s_mutex);
1001
1002         ceph_con_init(&s->s_con, s, &mds_con_ops, &mdsc->fsc->client->msgr);
1003
1004         atomic_set(&s->s_cap_gen, 1);
1005         s->s_cap_ttl = jiffies - 1;
1006
1007         spin_lock_init(&s->s_cap_lock);
1008         INIT_LIST_HEAD(&s->s_caps);
1009         refcount_set(&s->s_ref, 1);
1010         INIT_LIST_HEAD(&s->s_waiting);
1011         INIT_LIST_HEAD(&s->s_unsafe);
1012         xa_init(&s->s_delegated_inos);
1013         INIT_LIST_HEAD(&s->s_cap_releases);
1014         INIT_WORK(&s->s_cap_release_work, ceph_cap_release_work);
1015
1016         INIT_LIST_HEAD(&s->s_cap_dirty);
1017         INIT_LIST_HEAD(&s->s_cap_flushing);
1018
1019         mdsc->sessions[mds] = s;
1020         atomic_inc(&mdsc->num_sessions);
1021         refcount_inc(&s->s_ref);  /* one ref to sessions[], one to caller */
1022
1023         ceph_con_open(&s->s_con, CEPH_ENTITY_TYPE_MDS, mds,
1024                       ceph_mdsmap_get_addr(mdsc->mdsmap, mds));
1025
1026         return s;
1027
1028 fail_realloc:
1029         kfree(s);
1030         return ERR_PTR(-ENOMEM);
1031 }
1032
1033 /*
1034  * called under mdsc->mutex
1035  */
1036 static void __unregister_session(struct ceph_mds_client *mdsc,
1037                                struct ceph_mds_session *s)
1038 {
1039         doutc(mdsc->fsc->client, "mds%d %p\n", s->s_mds, s);
1040         BUG_ON(mdsc->sessions[s->s_mds] != s);
1041         mdsc->sessions[s->s_mds] = NULL;
1042         ceph_con_close(&s->s_con);
1043         ceph_put_mds_session(s);
1044         atomic_dec(&mdsc->num_sessions);
1045 }
1046
1047 /*
1048  * drop session refs in request.
1049  *
1050  * should be last request ref, or hold mdsc->mutex
1051  */
1052 static void put_request_session(struct ceph_mds_request *req)
1053 {
1054         if (req->r_session) {
1055                 ceph_put_mds_session(req->r_session);
1056                 req->r_session = NULL;
1057         }
1058 }
1059
1060 void ceph_mdsc_iterate_sessions(struct ceph_mds_client *mdsc,
1061                                 void (*cb)(struct ceph_mds_session *),
1062                                 bool check_state)
1063 {
1064         int mds;
1065
1066         mutex_lock(&mdsc->mutex);
1067         for (mds = 0; mds < mdsc->max_sessions; ++mds) {
1068                 struct ceph_mds_session *s;
1069
1070                 s = __ceph_lookup_mds_session(mdsc, mds);
1071                 if (!s)
1072                         continue;
1073
1074                 if (check_state && !check_session_state(s)) {
1075                         ceph_put_mds_session(s);
1076                         continue;
1077                 }
1078
1079                 mutex_unlock(&mdsc->mutex);
1080                 cb(s);
1081                 ceph_put_mds_session(s);
1082                 mutex_lock(&mdsc->mutex);
1083         }
1084         mutex_unlock(&mdsc->mutex);
1085 }
1086
1087 void ceph_mdsc_release_request(struct kref *kref)
1088 {
1089         struct ceph_mds_request *req = container_of(kref,
1090                                                     struct ceph_mds_request,
1091                                                     r_kref);
1092         ceph_mdsc_release_dir_caps_no_check(req);
1093         destroy_reply_info(&req->r_reply_info);
1094         if (req->r_request)
1095                 ceph_msg_put(req->r_request);
1096         if (req->r_reply)
1097                 ceph_msg_put(req->r_reply);
1098         if (req->r_inode) {
1099                 ceph_put_cap_refs(ceph_inode(req->r_inode), CEPH_CAP_PIN);
1100                 iput(req->r_inode);
1101         }
1102         if (req->r_parent) {
1103                 ceph_put_cap_refs(ceph_inode(req->r_parent), CEPH_CAP_PIN);
1104                 iput(req->r_parent);
1105         }
1106         iput(req->r_target_inode);
1107         iput(req->r_new_inode);
1108         if (req->r_dentry)
1109                 dput(req->r_dentry);
1110         if (req->r_old_dentry)
1111                 dput(req->r_old_dentry);
1112         if (req->r_old_dentry_dir) {
1113                 /*
1114                  * track (and drop pins for) r_old_dentry_dir
1115                  * separately, since r_old_dentry's d_parent may have
1116                  * changed between the dir mutex being dropped and
1117                  * this request being freed.
1118                  */
1119                 ceph_put_cap_refs(ceph_inode(req->r_old_dentry_dir),
1120                                   CEPH_CAP_PIN);
1121                 iput(req->r_old_dentry_dir);
1122         }
1123         kfree(req->r_path1);
1124         kfree(req->r_path2);
1125         put_cred(req->r_cred);
1126         if (req->r_mnt_idmap)
1127                 mnt_idmap_put(req->r_mnt_idmap);
1128         if (req->r_pagelist)
1129                 ceph_pagelist_release(req->r_pagelist);
1130         kfree(req->r_fscrypt_auth);
1131         kfree(req->r_altname);
1132         put_request_session(req);
1133         ceph_unreserve_caps(req->r_mdsc, &req->r_caps_reservation);
1134         WARN_ON_ONCE(!list_empty(&req->r_wait));
1135         kmem_cache_free(ceph_mds_request_cachep, req);
1136 }
1137
1138 DEFINE_RB_FUNCS(request, struct ceph_mds_request, r_tid, r_node)
1139
1140 /*
1141  * lookup session, bump ref if found.
1142  *
1143  * called under mdsc->mutex.
1144  */
1145 static struct ceph_mds_request *
1146 lookup_get_request(struct ceph_mds_client *mdsc, u64 tid)
1147 {
1148         struct ceph_mds_request *req;
1149
1150         req = lookup_request(&mdsc->request_tree, tid);
1151         if (req)
1152                 ceph_mdsc_get_request(req);
1153
1154         return req;
1155 }
1156
1157 /*
1158  * Register an in-flight request, and assign a tid.  Link to directory
1159  * are modifying (if any).
1160  *
1161  * Called under mdsc->mutex.
1162  */
1163 static void __register_request(struct ceph_mds_client *mdsc,
1164                                struct ceph_mds_request *req,
1165                                struct inode *dir)
1166 {
1167         struct ceph_client *cl = mdsc->fsc->client;
1168         int ret = 0;
1169
1170         req->r_tid = ++mdsc->last_tid;
1171         if (req->r_num_caps) {
1172                 ret = ceph_reserve_caps(mdsc, &req->r_caps_reservation,
1173                                         req->r_num_caps);
1174                 if (ret < 0) {
1175                         pr_err_client(cl, "%p failed to reserve caps: %d\n",
1176                                       req, ret);
1177                         /* set req->r_err to fail early from __do_request */
1178                         req->r_err = ret;
1179                         return;
1180                 }
1181         }
1182         doutc(cl, "%p tid %lld\n", req, req->r_tid);
1183         ceph_mdsc_get_request(req);
1184         insert_request(&mdsc->request_tree, req);
1185
1186         req->r_cred = get_current_cred();
1187         if (!req->r_mnt_idmap)
1188                 req->r_mnt_idmap = &nop_mnt_idmap;
1189
1190         if (mdsc->oldest_tid == 0 && req->r_op != CEPH_MDS_OP_SETFILELOCK)
1191                 mdsc->oldest_tid = req->r_tid;
1192
1193         if (dir) {
1194                 struct ceph_inode_info *ci = ceph_inode(dir);
1195
1196                 ihold(dir);
1197                 req->r_unsafe_dir = dir;
1198                 spin_lock(&ci->i_unsafe_lock);
1199                 list_add_tail(&req->r_unsafe_dir_item, &ci->i_unsafe_dirops);
1200                 spin_unlock(&ci->i_unsafe_lock);
1201         }
1202 }
1203
1204 static void __unregister_request(struct ceph_mds_client *mdsc,
1205                                  struct ceph_mds_request *req)
1206 {
1207         doutc(mdsc->fsc->client, "%p tid %lld\n", req, req->r_tid);
1208
1209         /* Never leave an unregistered request on an unsafe list! */
1210         list_del_init(&req->r_unsafe_item);
1211
1212         if (req->r_tid == mdsc->oldest_tid) {
1213                 struct rb_node *p = rb_next(&req->r_node);
1214                 mdsc->oldest_tid = 0;
1215                 while (p) {
1216                         struct ceph_mds_request *next_req =
1217                                 rb_entry(p, struct ceph_mds_request, r_node);
1218                         if (next_req->r_op != CEPH_MDS_OP_SETFILELOCK) {
1219                                 mdsc->oldest_tid = next_req->r_tid;
1220                                 break;
1221                         }
1222                         p = rb_next(p);
1223                 }
1224         }
1225
1226         erase_request(&mdsc->request_tree, req);
1227
1228         if (req->r_unsafe_dir) {
1229                 struct ceph_inode_info *ci = ceph_inode(req->r_unsafe_dir);
1230                 spin_lock(&ci->i_unsafe_lock);
1231                 list_del_init(&req->r_unsafe_dir_item);
1232                 spin_unlock(&ci->i_unsafe_lock);
1233         }
1234         if (req->r_target_inode &&
1235             test_bit(CEPH_MDS_R_GOT_UNSAFE, &req->r_req_flags)) {
1236                 struct ceph_inode_info *ci = ceph_inode(req->r_target_inode);
1237                 spin_lock(&ci->i_unsafe_lock);
1238                 list_del_init(&req->r_unsafe_target_item);
1239                 spin_unlock(&ci->i_unsafe_lock);
1240         }
1241
1242         if (req->r_unsafe_dir) {
1243                 iput(req->r_unsafe_dir);
1244                 req->r_unsafe_dir = NULL;
1245         }
1246
1247         complete_all(&req->r_safe_completion);
1248
1249         ceph_mdsc_put_request(req);
1250 }
1251
1252 /*
1253  * Walk back up the dentry tree until we hit a dentry representing a
1254  * non-snapshot inode. We do this using the rcu_read_lock (which must be held
1255  * when calling this) to ensure that the objects won't disappear while we're
1256  * working with them. Once we hit a candidate dentry, we attempt to take a
1257  * reference to it, and return that as the result.
1258  */
1259 static struct inode *get_nonsnap_parent(struct dentry *dentry)
1260 {
1261         struct inode *inode = NULL;
1262
1263         while (dentry && !IS_ROOT(dentry)) {
1264                 inode = d_inode_rcu(dentry);
1265                 if (!inode || ceph_snap(inode) == CEPH_NOSNAP)
1266                         break;
1267                 dentry = dentry->d_parent;
1268         }
1269         if (inode)
1270                 inode = igrab(inode);
1271         return inode;
1272 }
1273
1274 /*
1275  * Choose mds to send request to next.  If there is a hint set in the
1276  * request (e.g., due to a prior forward hint from the mds), use that.
1277  * Otherwise, consult frag tree and/or caps to identify the
1278  * appropriate mds.  If all else fails, choose randomly.
1279  *
1280  * Called under mdsc->mutex.
1281  */
1282 static int __choose_mds(struct ceph_mds_client *mdsc,
1283                         struct ceph_mds_request *req,
1284                         bool *random)
1285 {
1286         struct inode *inode;
1287         struct ceph_inode_info *ci;
1288         struct ceph_cap *cap;
1289         int mode = req->r_direct_mode;
1290         int mds = -1;
1291         u32 hash = req->r_direct_hash;
1292         bool is_hash = test_bit(CEPH_MDS_R_DIRECT_IS_HASH, &req->r_req_flags);
1293         struct ceph_client *cl = mdsc->fsc->client;
1294
1295         if (random)
1296                 *random = false;
1297
1298         /*
1299          * is there a specific mds we should try?  ignore hint if we have
1300          * no session and the mds is not up (active or recovering).
1301          */
1302         if (req->r_resend_mds >= 0 &&
1303             (__have_session(mdsc, req->r_resend_mds) ||
1304              ceph_mdsmap_get_state(mdsc->mdsmap, req->r_resend_mds) > 0)) {
1305                 doutc(cl, "using resend_mds mds%d\n", req->r_resend_mds);
1306                 return req->r_resend_mds;
1307         }
1308
1309         if (mode == USE_RANDOM_MDS)
1310                 goto random;
1311
1312         inode = NULL;
1313         if (req->r_inode) {
1314                 if (ceph_snap(req->r_inode) != CEPH_SNAPDIR) {
1315                         inode = req->r_inode;
1316                         ihold(inode);
1317                 } else {
1318                         /* req->r_dentry is non-null for LSSNAP request */
1319                         rcu_read_lock();
1320                         inode = get_nonsnap_parent(req->r_dentry);
1321                         rcu_read_unlock();
1322                         doutc(cl, "using snapdir's parent %p %llx.%llx\n",
1323                               inode, ceph_vinop(inode));
1324                 }
1325         } else if (req->r_dentry) {
1326                 /* ignore race with rename; old or new d_parent is okay */
1327                 struct dentry *parent;
1328                 struct inode *dir;
1329
1330                 rcu_read_lock();
1331                 parent = READ_ONCE(req->r_dentry->d_parent);
1332                 dir = req->r_parent ? : d_inode_rcu(parent);
1333
1334                 if (!dir || dir->i_sb != mdsc->fsc->sb) {
1335                         /*  not this fs or parent went negative */
1336                         inode = d_inode(req->r_dentry);
1337                         if (inode)
1338                                 ihold(inode);
1339                 } else if (ceph_snap(dir) != CEPH_NOSNAP) {
1340                         /* direct snapped/virtual snapdir requests
1341                          * based on parent dir inode */
1342                         inode = get_nonsnap_parent(parent);
1343                         doutc(cl, "using nonsnap parent %p %llx.%llx\n",
1344                               inode, ceph_vinop(inode));
1345                 } else {
1346                         /* dentry target */
1347                         inode = d_inode(req->r_dentry);
1348                         if (!inode || mode == USE_AUTH_MDS) {
1349                                 /* dir + name */
1350                                 inode = igrab(dir);
1351                                 hash = ceph_dentry_hash(dir, req->r_dentry);
1352                                 is_hash = true;
1353                         } else {
1354                                 ihold(inode);
1355                         }
1356                 }
1357                 rcu_read_unlock();
1358         }
1359
1360         if (!inode)
1361                 goto random;
1362
1363         doutc(cl, "%p %llx.%llx is_hash=%d (0x%x) mode %d\n", inode,
1364               ceph_vinop(inode), (int)is_hash, hash, mode);
1365         ci = ceph_inode(inode);
1366
1367         if (is_hash && S_ISDIR(inode->i_mode)) {
1368                 struct ceph_inode_frag frag;
1369                 int found;
1370
1371                 ceph_choose_frag(ci, hash, &frag, &found);
1372                 if (found) {
1373                         if (mode == USE_ANY_MDS && frag.ndist > 0) {
1374                                 u8 r;
1375
1376                                 /* choose a random replica */
1377                                 get_random_bytes(&r, 1);
1378                                 r %= frag.ndist;
1379                                 mds = frag.dist[r];
1380                                 doutc(cl, "%p %llx.%llx frag %u mds%d (%d/%d)\n",
1381                                       inode, ceph_vinop(inode), frag.frag,
1382                                       mds, (int)r, frag.ndist);
1383                                 if (ceph_mdsmap_get_state(mdsc->mdsmap, mds) >=
1384                                     CEPH_MDS_STATE_ACTIVE &&
1385                                     !ceph_mdsmap_is_laggy(mdsc->mdsmap, mds))
1386                                         goto out;
1387                         }
1388
1389                         /* since this file/dir wasn't known to be
1390                          * replicated, then we want to look for the
1391                          * authoritative mds. */
1392                         if (frag.mds >= 0) {
1393                                 /* choose auth mds */
1394                                 mds = frag.mds;
1395                                 doutc(cl, "%p %llx.%llx frag %u mds%d (auth)\n",
1396                                       inode, ceph_vinop(inode), frag.frag, mds);
1397                                 if (ceph_mdsmap_get_state(mdsc->mdsmap, mds) >=
1398                                     CEPH_MDS_STATE_ACTIVE) {
1399                                         if (!ceph_mdsmap_is_laggy(mdsc->mdsmap,
1400                                                                   mds))
1401                                                 goto out;
1402                                 }
1403                         }
1404                         mode = USE_AUTH_MDS;
1405                 }
1406         }
1407
1408         spin_lock(&ci->i_ceph_lock);
1409         cap = NULL;
1410         if (mode == USE_AUTH_MDS)
1411                 cap = ci->i_auth_cap;
1412         if (!cap && !RB_EMPTY_ROOT(&ci->i_caps))
1413                 cap = rb_entry(rb_first(&ci->i_caps), struct ceph_cap, ci_node);
1414         if (!cap) {
1415                 spin_unlock(&ci->i_ceph_lock);
1416                 iput(inode);
1417                 goto random;
1418         }
1419         mds = cap->session->s_mds;
1420         doutc(cl, "%p %llx.%llx mds%d (%scap %p)\n", inode,
1421               ceph_vinop(inode), mds,
1422               cap == ci->i_auth_cap ? "auth " : "", cap);
1423         spin_unlock(&ci->i_ceph_lock);
1424 out:
1425         iput(inode);
1426         return mds;
1427
1428 random:
1429         if (random)
1430                 *random = true;
1431
1432         mds = ceph_mdsmap_get_random_mds(mdsc->mdsmap);
1433         doutc(cl, "chose random mds%d\n", mds);
1434         return mds;
1435 }
1436
1437
1438 /*
1439  * session messages
1440  */
1441 struct ceph_msg *ceph_create_session_msg(u32 op, u64 seq)
1442 {
1443         struct ceph_msg *msg;
1444         struct ceph_mds_session_head *h;
1445
1446         msg = ceph_msg_new(CEPH_MSG_CLIENT_SESSION, sizeof(*h), GFP_NOFS,
1447                            false);
1448         if (!msg) {
1449                 pr_err("ENOMEM creating session %s msg\n",
1450                        ceph_session_op_name(op));
1451                 return NULL;
1452         }
1453         h = msg->front.iov_base;
1454         h->op = cpu_to_le32(op);
1455         h->seq = cpu_to_le64(seq);
1456
1457         return msg;
1458 }
1459
1460 static const unsigned char feature_bits[] = CEPHFS_FEATURES_CLIENT_SUPPORTED;
1461 #define FEATURE_BYTES(c) (DIV_ROUND_UP((size_t)feature_bits[c - 1] + 1, 64) * 8)
1462 static int encode_supported_features(void **p, void *end)
1463 {
1464         static const size_t count = ARRAY_SIZE(feature_bits);
1465
1466         if (count > 0) {
1467                 size_t i;
1468                 size_t size = FEATURE_BYTES(count);
1469                 unsigned long bit;
1470
1471                 if (WARN_ON_ONCE(*p + 4 + size > end))
1472                         return -ERANGE;
1473
1474                 ceph_encode_32(p, size);
1475                 memset(*p, 0, size);
1476                 for (i = 0; i < count; i++) {
1477                         bit = feature_bits[i];
1478                         ((unsigned char *)(*p))[bit / 8] |= BIT(bit % 8);
1479                 }
1480                 *p += size;
1481         } else {
1482                 if (WARN_ON_ONCE(*p + 4 > end))
1483                         return -ERANGE;
1484
1485                 ceph_encode_32(p, 0);
1486         }
1487
1488         return 0;
1489 }
1490
1491 static const unsigned char metric_bits[] = CEPHFS_METRIC_SPEC_CLIENT_SUPPORTED;
1492 #define METRIC_BYTES(cnt) (DIV_ROUND_UP((size_t)metric_bits[cnt - 1] + 1, 64) * 8)
1493 static int encode_metric_spec(void **p, void *end)
1494 {
1495         static const size_t count = ARRAY_SIZE(metric_bits);
1496
1497         /* header */
1498         if (WARN_ON_ONCE(*p + 2 > end))
1499                 return -ERANGE;
1500
1501         ceph_encode_8(p, 1); /* version */
1502         ceph_encode_8(p, 1); /* compat */
1503
1504         if (count > 0) {
1505                 size_t i;
1506                 size_t size = METRIC_BYTES(count);
1507
1508                 if (WARN_ON_ONCE(*p + 4 + 4 + size > end))
1509                         return -ERANGE;
1510
1511                 /* metric spec info length */
1512                 ceph_encode_32(p, 4 + size);
1513
1514                 /* metric spec */
1515                 ceph_encode_32(p, size);
1516                 memset(*p, 0, size);
1517                 for (i = 0; i < count; i++)
1518                         ((unsigned char *)(*p))[i / 8] |= BIT(metric_bits[i] % 8);
1519                 *p += size;
1520         } else {
1521                 if (WARN_ON_ONCE(*p + 4 + 4 > end))
1522                         return -ERANGE;
1523
1524                 /* metric spec info length */
1525                 ceph_encode_32(p, 4);
1526                 /* metric spec */
1527                 ceph_encode_32(p, 0);
1528         }
1529
1530         return 0;
1531 }
1532
1533 /*
1534  * session message, specialization for CEPH_SESSION_REQUEST_OPEN
1535  * to include additional client metadata fields.
1536  */
1537 static struct ceph_msg *create_session_open_msg(struct ceph_mds_client *mdsc, u64 seq)
1538 {
1539         struct ceph_msg *msg;
1540         struct ceph_mds_session_head *h;
1541         int i;
1542         int extra_bytes = 0;
1543         int metadata_key_count = 0;
1544         struct ceph_options *opt = mdsc->fsc->client->options;
1545         struct ceph_mount_options *fsopt = mdsc->fsc->mount_options;
1546         struct ceph_client *cl = mdsc->fsc->client;
1547         size_t size, count;
1548         void *p, *end;
1549         int ret;
1550
1551         const char* metadata[][2] = {
1552                 {"hostname", mdsc->nodename},
1553                 {"kernel_version", init_utsname()->release},
1554                 {"entity_id", opt->name ? : ""},
1555                 {"root", fsopt->server_path ? : "/"},
1556                 {NULL, NULL}
1557         };
1558
1559         /* Calculate serialized length of metadata */
1560         extra_bytes = 4;  /* map length */
1561         for (i = 0; metadata[i][0]; ++i) {
1562                 extra_bytes += 8 + strlen(metadata[i][0]) +
1563                         strlen(metadata[i][1]);
1564                 metadata_key_count++;
1565         }
1566
1567         /* supported feature */
1568         size = 0;
1569         count = ARRAY_SIZE(feature_bits);
1570         if (count > 0)
1571                 size = FEATURE_BYTES(count);
1572         extra_bytes += 4 + size;
1573
1574         /* metric spec */
1575         size = 0;
1576         count = ARRAY_SIZE(metric_bits);
1577         if (count > 0)
1578                 size = METRIC_BYTES(count);
1579         extra_bytes += 2 + 4 + 4 + size;
1580
1581         /* Allocate the message */
1582         msg = ceph_msg_new(CEPH_MSG_CLIENT_SESSION, sizeof(*h) + extra_bytes,
1583                            GFP_NOFS, false);
1584         if (!msg) {
1585                 pr_err_client(cl, "ENOMEM creating session open msg\n");
1586                 return ERR_PTR(-ENOMEM);
1587         }
1588         p = msg->front.iov_base;
1589         end = p + msg->front.iov_len;
1590
1591         h = p;
1592         h->op = cpu_to_le32(CEPH_SESSION_REQUEST_OPEN);
1593         h->seq = cpu_to_le64(seq);
1594
1595         /*
1596          * Serialize client metadata into waiting buffer space, using
1597          * the format that userspace expects for map<string, string>
1598          *
1599          * ClientSession messages with metadata are v4
1600          */
1601         msg->hdr.version = cpu_to_le16(4);
1602         msg->hdr.compat_version = cpu_to_le16(1);
1603
1604         /* The write pointer, following the session_head structure */
1605         p += sizeof(*h);
1606
1607         /* Number of entries in the map */
1608         ceph_encode_32(&p, metadata_key_count);
1609
1610         /* Two length-prefixed strings for each entry in the map */
1611         for (i = 0; metadata[i][0]; ++i) {
1612                 size_t const key_len = strlen(metadata[i][0]);
1613                 size_t const val_len = strlen(metadata[i][1]);
1614
1615                 ceph_encode_32(&p, key_len);
1616                 memcpy(p, metadata[i][0], key_len);
1617                 p += key_len;
1618                 ceph_encode_32(&p, val_len);
1619                 memcpy(p, metadata[i][1], val_len);
1620                 p += val_len;
1621         }
1622
1623         ret = encode_supported_features(&p, end);
1624         if (ret) {
1625                 pr_err_client(cl, "encode_supported_features failed!\n");
1626                 ceph_msg_put(msg);
1627                 return ERR_PTR(ret);
1628         }
1629
1630         ret = encode_metric_spec(&p, end);
1631         if (ret) {
1632                 pr_err_client(cl, "encode_metric_spec failed!\n");
1633                 ceph_msg_put(msg);
1634                 return ERR_PTR(ret);
1635         }
1636
1637         msg->front.iov_len = p - msg->front.iov_base;
1638         msg->hdr.front_len = cpu_to_le32(msg->front.iov_len);
1639
1640         return msg;
1641 }
1642
1643 /*
1644  * send session open request.
1645  *
1646  * called under mdsc->mutex
1647  */
1648 static int __open_session(struct ceph_mds_client *mdsc,
1649                           struct ceph_mds_session *session)
1650 {
1651         struct ceph_msg *msg;
1652         int mstate;
1653         int mds = session->s_mds;
1654
1655         if (READ_ONCE(mdsc->fsc->mount_state) == CEPH_MOUNT_FENCE_IO)
1656                 return -EIO;
1657
1658         /* wait for mds to go active? */
1659         mstate = ceph_mdsmap_get_state(mdsc->mdsmap, mds);
1660         doutc(mdsc->fsc->client, "open_session to mds%d (%s)\n", mds,
1661               ceph_mds_state_name(mstate));
1662         session->s_state = CEPH_MDS_SESSION_OPENING;
1663         session->s_renew_requested = jiffies;
1664
1665         /* send connect message */
1666         msg = create_session_open_msg(mdsc, session->s_seq);
1667         if (IS_ERR(msg))
1668                 return PTR_ERR(msg);
1669         ceph_con_send(&session->s_con, msg);
1670         return 0;
1671 }
1672
1673 /*
1674  * open sessions for any export targets for the given mds
1675  *
1676  * called under mdsc->mutex
1677  */
1678 static struct ceph_mds_session *
1679 __open_export_target_session(struct ceph_mds_client *mdsc, int target)
1680 {
1681         struct ceph_mds_session *session;
1682         int ret;
1683
1684         session = __ceph_lookup_mds_session(mdsc, target);
1685         if (!session) {
1686                 session = register_session(mdsc, target);
1687                 if (IS_ERR(session))
1688                         return session;
1689         }
1690         if (session->s_state == CEPH_MDS_SESSION_NEW ||
1691             session->s_state == CEPH_MDS_SESSION_CLOSING) {
1692                 ret = __open_session(mdsc, session);
1693                 if (ret)
1694                         return ERR_PTR(ret);
1695         }
1696
1697         return session;
1698 }
1699
1700 struct ceph_mds_session *
1701 ceph_mdsc_open_export_target_session(struct ceph_mds_client *mdsc, int target)
1702 {
1703         struct ceph_mds_session *session;
1704         struct ceph_client *cl = mdsc->fsc->client;
1705
1706         doutc(cl, "to mds%d\n", target);
1707
1708         mutex_lock(&mdsc->mutex);
1709         session = __open_export_target_session(mdsc, target);
1710         mutex_unlock(&mdsc->mutex);
1711
1712         return session;
1713 }
1714
1715 static void __open_export_target_sessions(struct ceph_mds_client *mdsc,
1716                                           struct ceph_mds_session *session)
1717 {
1718         struct ceph_mds_info *mi;
1719         struct ceph_mds_session *ts;
1720         int i, mds = session->s_mds;
1721         struct ceph_client *cl = mdsc->fsc->client;
1722
1723         if (mds >= mdsc->mdsmap->possible_max_rank)
1724                 return;
1725
1726         mi = &mdsc->mdsmap->m_info[mds];
1727         doutc(cl, "for mds%d (%d targets)\n", session->s_mds,
1728               mi->num_export_targets);
1729
1730         for (i = 0; i < mi->num_export_targets; i++) {
1731                 ts = __open_export_target_session(mdsc, mi->export_targets[i]);
1732                 ceph_put_mds_session(ts);
1733         }
1734 }
1735
1736 void ceph_mdsc_open_export_target_sessions(struct ceph_mds_client *mdsc,
1737                                            struct ceph_mds_session *session)
1738 {
1739         mutex_lock(&mdsc->mutex);
1740         __open_export_target_sessions(mdsc, session);
1741         mutex_unlock(&mdsc->mutex);
1742 }
1743
1744 /*
1745  * session caps
1746  */
1747
1748 static void detach_cap_releases(struct ceph_mds_session *session,
1749                                 struct list_head *target)
1750 {
1751         struct ceph_client *cl = session->s_mdsc->fsc->client;
1752
1753         lockdep_assert_held(&session->s_cap_lock);
1754
1755         list_splice_init(&session->s_cap_releases, target);
1756         session->s_num_cap_releases = 0;
1757         doutc(cl, "mds%d\n", session->s_mds);
1758 }
1759
1760 static void dispose_cap_releases(struct ceph_mds_client *mdsc,
1761                                  struct list_head *dispose)
1762 {
1763         while (!list_empty(dispose)) {
1764                 struct ceph_cap *cap;
1765                 /* zero out the in-progress message */
1766                 cap = list_first_entry(dispose, struct ceph_cap, session_caps);
1767                 list_del(&cap->session_caps);
1768                 ceph_put_cap(mdsc, cap);
1769         }
1770 }
1771
1772 static void cleanup_session_requests(struct ceph_mds_client *mdsc,
1773                                      struct ceph_mds_session *session)
1774 {
1775         struct ceph_client *cl = mdsc->fsc->client;
1776         struct ceph_mds_request *req;
1777         struct rb_node *p;
1778
1779         doutc(cl, "mds%d\n", session->s_mds);
1780         mutex_lock(&mdsc->mutex);
1781         while (!list_empty(&session->s_unsafe)) {
1782                 req = list_first_entry(&session->s_unsafe,
1783                                        struct ceph_mds_request, r_unsafe_item);
1784                 pr_warn_ratelimited_client(cl, " dropping unsafe request %llu\n",
1785                                            req->r_tid);
1786                 if (req->r_target_inode)
1787                         mapping_set_error(req->r_target_inode->i_mapping, -EIO);
1788                 if (req->r_unsafe_dir)
1789                         mapping_set_error(req->r_unsafe_dir->i_mapping, -EIO);
1790                 __unregister_request(mdsc, req);
1791         }
1792         /* zero r_attempts, so kick_requests() will re-send requests */
1793         p = rb_first(&mdsc->request_tree);
1794         while (p) {
1795                 req = rb_entry(p, struct ceph_mds_request, r_node);
1796                 p = rb_next(p);
1797                 if (req->r_session &&
1798                     req->r_session->s_mds == session->s_mds)
1799                         req->r_attempts = 0;
1800         }
1801         mutex_unlock(&mdsc->mutex);
1802 }
1803
1804 /*
1805  * Helper to safely iterate over all caps associated with a session, with
1806  * special care taken to handle a racing __ceph_remove_cap().
1807  *
1808  * Caller must hold session s_mutex.
1809  */
1810 int ceph_iterate_session_caps(struct ceph_mds_session *session,
1811                               int (*cb)(struct inode *, int mds, void *),
1812                               void *arg)
1813 {
1814         struct ceph_client *cl = session->s_mdsc->fsc->client;
1815         struct list_head *p;
1816         struct ceph_cap *cap;
1817         struct inode *inode, *last_inode = NULL;
1818         struct ceph_cap *old_cap = NULL;
1819         int ret;
1820
1821         doutc(cl, "%p mds%d\n", session, session->s_mds);
1822         spin_lock(&session->s_cap_lock);
1823         p = session->s_caps.next;
1824         while (p != &session->s_caps) {
1825                 int mds;
1826
1827                 cap = list_entry(p, struct ceph_cap, session_caps);
1828                 inode = igrab(&cap->ci->netfs.inode);
1829                 if (!inode) {
1830                         p = p->next;
1831                         continue;
1832                 }
1833                 session->s_cap_iterator = cap;
1834                 mds = cap->mds;
1835                 spin_unlock(&session->s_cap_lock);
1836
1837                 if (last_inode) {
1838                         iput(last_inode);
1839                         last_inode = NULL;
1840                 }
1841                 if (old_cap) {
1842                         ceph_put_cap(session->s_mdsc, old_cap);
1843                         old_cap = NULL;
1844                 }
1845
1846                 ret = cb(inode, mds, arg);
1847                 last_inode = inode;
1848
1849                 spin_lock(&session->s_cap_lock);
1850                 p = p->next;
1851                 if (!cap->ci) {
1852                         doutc(cl, "finishing cap %p removal\n", cap);
1853                         BUG_ON(cap->session != session);
1854                         cap->session = NULL;
1855                         list_del_init(&cap->session_caps);
1856                         session->s_nr_caps--;
1857                         atomic64_dec(&session->s_mdsc->metric.total_caps);
1858                         if (cap->queue_release)
1859                                 __ceph_queue_cap_release(session, cap);
1860                         else
1861                                 old_cap = cap;  /* put_cap it w/o locks held */
1862                 }
1863                 if (ret < 0)
1864                         goto out;
1865         }
1866         ret = 0;
1867 out:
1868         session->s_cap_iterator = NULL;
1869         spin_unlock(&session->s_cap_lock);
1870
1871         iput(last_inode);
1872         if (old_cap)
1873                 ceph_put_cap(session->s_mdsc, old_cap);
1874
1875         return ret;
1876 }
1877
1878 static int remove_session_caps_cb(struct inode *inode, int mds, void *arg)
1879 {
1880         struct ceph_inode_info *ci = ceph_inode(inode);
1881         struct ceph_client *cl = ceph_inode_to_client(inode);
1882         bool invalidate = false;
1883         struct ceph_cap *cap;
1884         int iputs = 0;
1885
1886         spin_lock(&ci->i_ceph_lock);
1887         cap = __get_cap_for_mds(ci, mds);
1888         if (cap) {
1889                 doutc(cl, " removing cap %p, ci is %p, inode is %p\n",
1890                       cap, ci, &ci->netfs.inode);
1891
1892                 iputs = ceph_purge_inode_cap(inode, cap, &invalidate);
1893         }
1894         spin_unlock(&ci->i_ceph_lock);
1895
1896         if (cap)
1897                 wake_up_all(&ci->i_cap_wq);
1898         if (invalidate)
1899                 ceph_queue_invalidate(inode);
1900         while (iputs--)
1901                 iput(inode);
1902         return 0;
1903 }
1904
1905 /*
1906  * caller must hold session s_mutex
1907  */
1908 static void remove_session_caps(struct ceph_mds_session *session)
1909 {
1910         struct ceph_fs_client *fsc = session->s_mdsc->fsc;
1911         struct super_block *sb = fsc->sb;
1912         LIST_HEAD(dispose);
1913
1914         doutc(fsc->client, "on %p\n", session);
1915         ceph_iterate_session_caps(session, remove_session_caps_cb, fsc);
1916
1917         wake_up_all(&fsc->mdsc->cap_flushing_wq);
1918
1919         spin_lock(&session->s_cap_lock);
1920         if (session->s_nr_caps > 0) {
1921                 struct inode *inode;
1922                 struct ceph_cap *cap, *prev = NULL;
1923                 struct ceph_vino vino;
1924                 /*
1925                  * iterate_session_caps() skips inodes that are being
1926                  * deleted, we need to wait until deletions are complete.
1927                  * __wait_on_freeing_inode() is designed for the job,
1928                  * but it is not exported, so use lookup inode function
1929                  * to access it.
1930                  */
1931                 while (!list_empty(&session->s_caps)) {
1932                         cap = list_entry(session->s_caps.next,
1933                                          struct ceph_cap, session_caps);
1934                         if (cap == prev)
1935                                 break;
1936                         prev = cap;
1937                         vino = cap->ci->i_vino;
1938                         spin_unlock(&session->s_cap_lock);
1939
1940                         inode = ceph_find_inode(sb, vino);
1941                         iput(inode);
1942
1943                         spin_lock(&session->s_cap_lock);
1944                 }
1945         }
1946
1947         // drop cap expires and unlock s_cap_lock
1948         detach_cap_releases(session, &dispose);
1949
1950         BUG_ON(session->s_nr_caps > 0);
1951         BUG_ON(!list_empty(&session->s_cap_flushing));
1952         spin_unlock(&session->s_cap_lock);
1953         dispose_cap_releases(session->s_mdsc, &dispose);
1954 }
1955
1956 enum {
1957         RECONNECT,
1958         RENEWCAPS,
1959         FORCE_RO,
1960 };
1961
1962 /*
1963  * wake up any threads waiting on this session's caps.  if the cap is
1964  * old (didn't get renewed on the client reconnect), remove it now.
1965  *
1966  * caller must hold s_mutex.
1967  */
1968 static int wake_up_session_cb(struct inode *inode, int mds, void *arg)
1969 {
1970         struct ceph_inode_info *ci = ceph_inode(inode);
1971         unsigned long ev = (unsigned long)arg;
1972
1973         if (ev == RECONNECT) {
1974                 spin_lock(&ci->i_ceph_lock);
1975                 ci->i_wanted_max_size = 0;
1976                 ci->i_requested_max_size = 0;
1977                 spin_unlock(&ci->i_ceph_lock);
1978         } else if (ev == RENEWCAPS) {
1979                 struct ceph_cap *cap;
1980
1981                 spin_lock(&ci->i_ceph_lock);
1982                 cap = __get_cap_for_mds(ci, mds);
1983                 /* mds did not re-issue stale cap */
1984                 if (cap && cap->cap_gen < atomic_read(&cap->session->s_cap_gen))
1985                         cap->issued = cap->implemented = CEPH_CAP_PIN;
1986                 spin_unlock(&ci->i_ceph_lock);
1987         } else if (ev == FORCE_RO) {
1988         }
1989         wake_up_all(&ci->i_cap_wq);
1990         return 0;
1991 }
1992
1993 static void wake_up_session_caps(struct ceph_mds_session *session, int ev)
1994 {
1995         struct ceph_client *cl = session->s_mdsc->fsc->client;
1996
1997         doutc(cl, "session %p mds%d\n", session, session->s_mds);
1998         ceph_iterate_session_caps(session, wake_up_session_cb,
1999                                   (void *)(unsigned long)ev);
2000 }
2001
2002 /*
2003  * Send periodic message to MDS renewing all currently held caps.  The
2004  * ack will reset the expiration for all caps from this session.
2005  *
2006  * caller holds s_mutex
2007  */
2008 static int send_renew_caps(struct ceph_mds_client *mdsc,
2009                            struct ceph_mds_session *session)
2010 {
2011         struct ceph_client *cl = mdsc->fsc->client;
2012         struct ceph_msg *msg;
2013         int state;
2014
2015         if (time_after_eq(jiffies, session->s_cap_ttl) &&
2016             time_after_eq(session->s_cap_ttl, session->s_renew_requested))
2017                 pr_info_client(cl, "mds%d caps stale\n", session->s_mds);
2018         session->s_renew_requested = jiffies;
2019
2020         /* do not try to renew caps until a recovering mds has reconnected
2021          * with its clients. */
2022         state = ceph_mdsmap_get_state(mdsc->mdsmap, session->s_mds);
2023         if (state < CEPH_MDS_STATE_RECONNECT) {
2024                 doutc(cl, "ignoring mds%d (%s)\n", session->s_mds,
2025                       ceph_mds_state_name(state));
2026                 return 0;
2027         }
2028
2029         doutc(cl, "to mds%d (%s)\n", session->s_mds,
2030               ceph_mds_state_name(state));
2031         msg = ceph_create_session_msg(CEPH_SESSION_REQUEST_RENEWCAPS,
2032                                       ++session->s_renew_seq);
2033         if (!msg)
2034                 return -ENOMEM;
2035         ceph_con_send(&session->s_con, msg);
2036         return 0;
2037 }
2038
2039 static int send_flushmsg_ack(struct ceph_mds_client *mdsc,
2040                              struct ceph_mds_session *session, u64 seq)
2041 {
2042         struct ceph_client *cl = mdsc->fsc->client;
2043         struct ceph_msg *msg;
2044
2045         doutc(cl, "to mds%d (%s)s seq %lld\n", session->s_mds,
2046               ceph_session_state_name(session->s_state), seq);
2047         msg = ceph_create_session_msg(CEPH_SESSION_FLUSHMSG_ACK, seq);
2048         if (!msg)
2049                 return -ENOMEM;
2050         ceph_con_send(&session->s_con, msg);
2051         return 0;
2052 }
2053
2054
2055 /*
2056  * Note new cap ttl, and any transition from stale -> not stale (fresh?).
2057  *
2058  * Called under session->s_mutex
2059  */
2060 static void renewed_caps(struct ceph_mds_client *mdsc,
2061                          struct ceph_mds_session *session, int is_renew)
2062 {
2063         struct ceph_client *cl = mdsc->fsc->client;
2064         int was_stale;
2065         int wake = 0;
2066
2067         spin_lock(&session->s_cap_lock);
2068         was_stale = is_renew && time_after_eq(jiffies, session->s_cap_ttl);
2069
2070         session->s_cap_ttl = session->s_renew_requested +
2071                 mdsc->mdsmap->m_session_timeout*HZ;
2072
2073         if (was_stale) {
2074                 if (time_before(jiffies, session->s_cap_ttl)) {
2075                         pr_info_client(cl, "mds%d caps renewed\n",
2076                                        session->s_mds);
2077                         wake = 1;
2078                 } else {
2079                         pr_info_client(cl, "mds%d caps still stale\n",
2080                                        session->s_mds);
2081                 }
2082         }
2083         doutc(cl, "mds%d ttl now %lu, was %s, now %s\n", session->s_mds,
2084               session->s_cap_ttl, was_stale ? "stale" : "fresh",
2085               time_before(jiffies, session->s_cap_ttl) ? "stale" : "fresh");
2086         spin_unlock(&session->s_cap_lock);
2087
2088         if (wake)
2089                 wake_up_session_caps(session, RENEWCAPS);
2090 }
2091
2092 /*
2093  * send a session close request
2094  */
2095 static int request_close_session(struct ceph_mds_session *session)
2096 {
2097         struct ceph_client *cl = session->s_mdsc->fsc->client;
2098         struct ceph_msg *msg;
2099
2100         doutc(cl, "mds%d state %s seq %lld\n", session->s_mds,
2101               ceph_session_state_name(session->s_state), session->s_seq);
2102         msg = ceph_create_session_msg(CEPH_SESSION_REQUEST_CLOSE,
2103                                       session->s_seq);
2104         if (!msg)
2105                 return -ENOMEM;
2106         ceph_con_send(&session->s_con, msg);
2107         return 1;
2108 }
2109
2110 /*
2111  * Called with s_mutex held.
2112  */
2113 static int __close_session(struct ceph_mds_client *mdsc,
2114                          struct ceph_mds_session *session)
2115 {
2116         if (session->s_state >= CEPH_MDS_SESSION_CLOSING)
2117                 return 0;
2118         session->s_state = CEPH_MDS_SESSION_CLOSING;
2119         return request_close_session(session);
2120 }
2121
2122 static bool drop_negative_children(struct dentry *dentry)
2123 {
2124         struct dentry *child;
2125         bool all_negative = true;
2126
2127         if (!d_is_dir(dentry))
2128                 goto out;
2129
2130         spin_lock(&dentry->d_lock);
2131         list_for_each_entry(child, &dentry->d_subdirs, d_child) {
2132                 if (d_really_is_positive(child)) {
2133                         all_negative = false;
2134                         break;
2135                 }
2136         }
2137         spin_unlock(&dentry->d_lock);
2138
2139         if (all_negative)
2140                 shrink_dcache_parent(dentry);
2141 out:
2142         return all_negative;
2143 }
2144
2145 /*
2146  * Trim old(er) caps.
2147  *
2148  * Because we can't cache an inode without one or more caps, we do
2149  * this indirectly: if a cap is unused, we prune its aliases, at which
2150  * point the inode will hopefully get dropped to.
2151  *
2152  * Yes, this is a bit sloppy.  Our only real goal here is to respond to
2153  * memory pressure from the MDS, though, so it needn't be perfect.
2154  */
2155 static int trim_caps_cb(struct inode *inode, int mds, void *arg)
2156 {
2157         struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb);
2158         struct ceph_client *cl = mdsc->fsc->client;
2159         int *remaining = arg;
2160         struct ceph_inode_info *ci = ceph_inode(inode);
2161         int used, wanted, oissued, mine;
2162         struct ceph_cap *cap;
2163
2164         if (*remaining <= 0)
2165                 return -1;
2166
2167         spin_lock(&ci->i_ceph_lock);
2168         cap = __get_cap_for_mds(ci, mds);
2169         if (!cap) {
2170                 spin_unlock(&ci->i_ceph_lock);
2171                 return 0;
2172         }
2173         mine = cap->issued | cap->implemented;
2174         used = __ceph_caps_used(ci);
2175         wanted = __ceph_caps_file_wanted(ci);
2176         oissued = __ceph_caps_issued_other(ci, cap);
2177
2178         doutc(cl, "%p %llx.%llx cap %p mine %s oissued %s used %s wanted %s\n",
2179               inode, ceph_vinop(inode), cap, ceph_cap_string(mine),
2180               ceph_cap_string(oissued), ceph_cap_string(used),
2181               ceph_cap_string(wanted));
2182         if (cap == ci->i_auth_cap) {
2183                 if (ci->i_dirty_caps || ci->i_flushing_caps ||
2184                     !list_empty(&ci->i_cap_snaps))
2185                         goto out;
2186                 if ((used | wanted) & CEPH_CAP_ANY_WR)
2187                         goto out;
2188                 /* Note: it's possible that i_filelock_ref becomes non-zero
2189                  * after dropping auth caps. It doesn't hurt because reply
2190                  * of lock mds request will re-add auth caps. */
2191                 if (atomic_read(&ci->i_filelock_ref) > 0)
2192                         goto out;
2193         }
2194         /* The inode has cached pages, but it's no longer used.
2195          * we can safely drop it */
2196         if (S_ISREG(inode->i_mode) &&
2197             wanted == 0 && used == CEPH_CAP_FILE_CACHE &&
2198             !(oissued & CEPH_CAP_FILE_CACHE)) {
2199           used = 0;
2200           oissued = 0;
2201         }
2202         if ((used | wanted) & ~oissued & mine)
2203                 goto out;   /* we need these caps */
2204
2205         if (oissued) {
2206                 /* we aren't the only cap.. just remove us */
2207                 ceph_remove_cap(mdsc, cap, true);
2208                 (*remaining)--;
2209         } else {
2210                 struct dentry *dentry;
2211                 /* try dropping referring dentries */
2212                 spin_unlock(&ci->i_ceph_lock);
2213                 dentry = d_find_any_alias(inode);
2214                 if (dentry && drop_negative_children(dentry)) {
2215                         int count;
2216                         dput(dentry);
2217                         d_prune_aliases(inode);
2218                         count = atomic_read(&inode->i_count);
2219                         if (count == 1)
2220                                 (*remaining)--;
2221                         doutc(cl, "%p %llx.%llx cap %p pruned, count now %d\n",
2222                               inode, ceph_vinop(inode), cap, count);
2223                 } else {
2224                         dput(dentry);
2225                 }
2226                 return 0;
2227         }
2228
2229 out:
2230         spin_unlock(&ci->i_ceph_lock);
2231         return 0;
2232 }
2233
2234 /*
2235  * Trim session cap count down to some max number.
2236  */
2237 int ceph_trim_caps(struct ceph_mds_client *mdsc,
2238                    struct ceph_mds_session *session,
2239                    int max_caps)
2240 {
2241         struct ceph_client *cl = mdsc->fsc->client;
2242         int trim_caps = session->s_nr_caps - max_caps;
2243
2244         doutc(cl, "mds%d start: %d / %d, trim %d\n", session->s_mds,
2245               session->s_nr_caps, max_caps, trim_caps);
2246         if (trim_caps > 0) {
2247                 int remaining = trim_caps;
2248
2249                 ceph_iterate_session_caps(session, trim_caps_cb, &remaining);
2250                 doutc(cl, "mds%d done: %d / %d, trimmed %d\n",
2251                       session->s_mds, session->s_nr_caps, max_caps,
2252                       trim_caps - remaining);
2253         }
2254
2255         ceph_flush_cap_releases(mdsc, session);
2256         return 0;
2257 }
2258
2259 static int check_caps_flush(struct ceph_mds_client *mdsc,
2260                             u64 want_flush_tid)
2261 {
2262         struct ceph_client *cl = mdsc->fsc->client;
2263         int ret = 1;
2264
2265         spin_lock(&mdsc->cap_dirty_lock);
2266         if (!list_empty(&mdsc->cap_flush_list)) {
2267                 struct ceph_cap_flush *cf =
2268                         list_first_entry(&mdsc->cap_flush_list,
2269                                          struct ceph_cap_flush, g_list);
2270                 if (cf->tid <= want_flush_tid) {
2271                         doutc(cl, "still flushing tid %llu <= %llu\n",
2272                               cf->tid, want_flush_tid);
2273                         ret = 0;
2274                 }
2275         }
2276         spin_unlock(&mdsc->cap_dirty_lock);
2277         return ret;
2278 }
2279
2280 /*
2281  * flush all dirty inode data to disk.
2282  *
2283  * returns true if we've flushed through want_flush_tid
2284  */
2285 static void wait_caps_flush(struct ceph_mds_client *mdsc,
2286                             u64 want_flush_tid)
2287 {
2288         struct ceph_client *cl = mdsc->fsc->client;
2289
2290         doutc(cl, "want %llu\n", want_flush_tid);
2291
2292         wait_event(mdsc->cap_flushing_wq,
2293                    check_caps_flush(mdsc, want_flush_tid));
2294
2295         doutc(cl, "ok, flushed thru %llu\n", want_flush_tid);
2296 }
2297
2298 /*
2299  * called under s_mutex
2300  */
2301 static void ceph_send_cap_releases(struct ceph_mds_client *mdsc,
2302                                    struct ceph_mds_session *session)
2303 {
2304         struct ceph_client *cl = mdsc->fsc->client;
2305         struct ceph_msg *msg = NULL;
2306         struct ceph_mds_cap_release *head;
2307         struct ceph_mds_cap_item *item;
2308         struct ceph_osd_client *osdc = &mdsc->fsc->client->osdc;
2309         struct ceph_cap *cap;
2310         LIST_HEAD(tmp_list);
2311         int num_cap_releases;
2312         __le32  barrier, *cap_barrier;
2313
2314         down_read(&osdc->lock);
2315         barrier = cpu_to_le32(osdc->epoch_barrier);
2316         up_read(&osdc->lock);
2317
2318         spin_lock(&session->s_cap_lock);
2319 again:
2320         list_splice_init(&session->s_cap_releases, &tmp_list);
2321         num_cap_releases = session->s_num_cap_releases;
2322         session->s_num_cap_releases = 0;
2323         spin_unlock(&session->s_cap_lock);
2324
2325         while (!list_empty(&tmp_list)) {
2326                 if (!msg) {
2327                         msg = ceph_msg_new(CEPH_MSG_CLIENT_CAPRELEASE,
2328                                         PAGE_SIZE, GFP_NOFS, false);
2329                         if (!msg)
2330                                 goto out_err;
2331                         head = msg->front.iov_base;
2332                         head->num = cpu_to_le32(0);
2333                         msg->front.iov_len = sizeof(*head);
2334
2335                         msg->hdr.version = cpu_to_le16(2);
2336                         msg->hdr.compat_version = cpu_to_le16(1);
2337                 }
2338
2339                 cap = list_first_entry(&tmp_list, struct ceph_cap,
2340                                         session_caps);
2341                 list_del(&cap->session_caps);
2342                 num_cap_releases--;
2343
2344                 head = msg->front.iov_base;
2345                 put_unaligned_le32(get_unaligned_le32(&head->num) + 1,
2346                                    &head->num);
2347                 item = msg->front.iov_base + msg->front.iov_len;
2348                 item->ino = cpu_to_le64(cap->cap_ino);
2349                 item->cap_id = cpu_to_le64(cap->cap_id);
2350                 item->migrate_seq = cpu_to_le32(cap->mseq);
2351                 item->seq = cpu_to_le32(cap->issue_seq);
2352                 msg->front.iov_len += sizeof(*item);
2353
2354                 ceph_put_cap(mdsc, cap);
2355
2356                 if (le32_to_cpu(head->num) == CEPH_CAPS_PER_RELEASE) {
2357                         // Append cap_barrier field
2358                         cap_barrier = msg->front.iov_base + msg->front.iov_len;
2359                         *cap_barrier = barrier;
2360                         msg->front.iov_len += sizeof(*cap_barrier);
2361
2362                         msg->hdr.front_len = cpu_to_le32(msg->front.iov_len);
2363                         doutc(cl, "mds%d %p\n", session->s_mds, msg);
2364                         ceph_con_send(&session->s_con, msg);
2365                         msg = NULL;
2366                 }
2367         }
2368
2369         BUG_ON(num_cap_releases != 0);
2370
2371         spin_lock(&session->s_cap_lock);
2372         if (!list_empty(&session->s_cap_releases))
2373                 goto again;
2374         spin_unlock(&session->s_cap_lock);
2375
2376         if (msg) {
2377                 // Append cap_barrier field
2378                 cap_barrier = msg->front.iov_base + msg->front.iov_len;
2379                 *cap_barrier = barrier;
2380                 msg->front.iov_len += sizeof(*cap_barrier);
2381
2382                 msg->hdr.front_len = cpu_to_le32(msg->front.iov_len);
2383                 doutc(cl, "mds%d %p\n", session->s_mds, msg);
2384                 ceph_con_send(&session->s_con, msg);
2385         }
2386         return;
2387 out_err:
2388         pr_err_client(cl, "mds%d, failed to allocate message\n",
2389                       session->s_mds);
2390         spin_lock(&session->s_cap_lock);
2391         list_splice(&tmp_list, &session->s_cap_releases);
2392         session->s_num_cap_releases += num_cap_releases;
2393         spin_unlock(&session->s_cap_lock);
2394 }
2395
2396 static void ceph_cap_release_work(struct work_struct *work)
2397 {
2398         struct ceph_mds_session *session =
2399                 container_of(work, struct ceph_mds_session, s_cap_release_work);
2400
2401         mutex_lock(&session->s_mutex);
2402         if (session->s_state == CEPH_MDS_SESSION_OPEN ||
2403             session->s_state == CEPH_MDS_SESSION_HUNG)
2404                 ceph_send_cap_releases(session->s_mdsc, session);
2405         mutex_unlock(&session->s_mutex);
2406         ceph_put_mds_session(session);
2407 }
2408
2409 void ceph_flush_cap_releases(struct ceph_mds_client *mdsc,
2410                              struct ceph_mds_session *session)
2411 {
2412         struct ceph_client *cl = mdsc->fsc->client;
2413         if (mdsc->stopping)
2414                 return;
2415
2416         ceph_get_mds_session(session);
2417         if (queue_work(mdsc->fsc->cap_wq,
2418                        &session->s_cap_release_work)) {
2419                 doutc(cl, "cap release work queued\n");
2420         } else {
2421                 ceph_put_mds_session(session);
2422                 doutc(cl, "failed to queue cap release work\n");
2423         }
2424 }
2425
2426 /*
2427  * caller holds session->s_cap_lock
2428  */
2429 void __ceph_queue_cap_release(struct ceph_mds_session *session,
2430                               struct ceph_cap *cap)
2431 {
2432         list_add_tail(&cap->session_caps, &session->s_cap_releases);
2433         session->s_num_cap_releases++;
2434
2435         if (!(session->s_num_cap_releases % CEPH_CAPS_PER_RELEASE))
2436                 ceph_flush_cap_releases(session->s_mdsc, session);
2437 }
2438
2439 static void ceph_cap_reclaim_work(struct work_struct *work)
2440 {
2441         struct ceph_mds_client *mdsc =
2442                 container_of(work, struct ceph_mds_client, cap_reclaim_work);
2443         int ret = ceph_trim_dentries(mdsc);
2444         if (ret == -EAGAIN)
2445                 ceph_queue_cap_reclaim_work(mdsc);
2446 }
2447
2448 void ceph_queue_cap_reclaim_work(struct ceph_mds_client *mdsc)
2449 {
2450         struct ceph_client *cl = mdsc->fsc->client;
2451         if (mdsc->stopping)
2452                 return;
2453
2454         if (queue_work(mdsc->fsc->cap_wq, &mdsc->cap_reclaim_work)) {
2455                 doutc(cl, "caps reclaim work queued\n");
2456         } else {
2457                 doutc(cl, "failed to queue caps release work\n");
2458         }
2459 }
2460
2461 void ceph_reclaim_caps_nr(struct ceph_mds_client *mdsc, int nr)
2462 {
2463         int val;
2464         if (!nr)
2465                 return;
2466         val = atomic_add_return(nr, &mdsc->cap_reclaim_pending);
2467         if ((val % CEPH_CAPS_PER_RELEASE) < nr) {
2468                 atomic_set(&mdsc->cap_reclaim_pending, 0);
2469                 ceph_queue_cap_reclaim_work(mdsc);
2470         }
2471 }
2472
2473 /*
2474  * requests
2475  */
2476
2477 int ceph_alloc_readdir_reply_buffer(struct ceph_mds_request *req,
2478                                     struct inode *dir)
2479 {
2480         struct ceph_inode_info *ci = ceph_inode(dir);
2481         struct ceph_mds_reply_info_parsed *rinfo = &req->r_reply_info;
2482         struct ceph_mount_options *opt = req->r_mdsc->fsc->mount_options;
2483         size_t size = sizeof(struct ceph_mds_reply_dir_entry);
2484         unsigned int num_entries;
2485         int order;
2486
2487         spin_lock(&ci->i_ceph_lock);
2488         num_entries = ci->i_files + ci->i_subdirs;
2489         spin_unlock(&ci->i_ceph_lock);
2490         num_entries = max(num_entries, 1U);
2491         num_entries = min(num_entries, opt->max_readdir);
2492
2493         order = get_order(size * num_entries);
2494         while (order >= 0) {
2495                 rinfo->dir_entries = (void*)__get_free_pages(GFP_KERNEL |
2496                                                              __GFP_NOWARN |
2497                                                              __GFP_ZERO,
2498                                                              order);
2499                 if (rinfo->dir_entries)
2500                         break;
2501                 order--;
2502         }
2503         if (!rinfo->dir_entries)
2504                 return -ENOMEM;
2505
2506         num_entries = (PAGE_SIZE << order) / size;
2507         num_entries = min(num_entries, opt->max_readdir);
2508
2509         rinfo->dir_buf_size = PAGE_SIZE << order;
2510         req->r_num_caps = num_entries + 1;
2511         req->r_args.readdir.max_entries = cpu_to_le32(num_entries);
2512         req->r_args.readdir.max_bytes = cpu_to_le32(opt->max_readdir_bytes);
2513         return 0;
2514 }
2515
2516 /*
2517  * Create an mds request.
2518  */
2519 struct ceph_mds_request *
2520 ceph_mdsc_create_request(struct ceph_mds_client *mdsc, int op, int mode)
2521 {
2522         struct ceph_mds_request *req;
2523
2524         req = kmem_cache_zalloc(ceph_mds_request_cachep, GFP_NOFS);
2525         if (!req)
2526                 return ERR_PTR(-ENOMEM);
2527
2528         mutex_init(&req->r_fill_mutex);
2529         req->r_mdsc = mdsc;
2530         req->r_started = jiffies;
2531         req->r_start_latency = ktime_get();
2532         req->r_resend_mds = -1;
2533         INIT_LIST_HEAD(&req->r_unsafe_dir_item);
2534         INIT_LIST_HEAD(&req->r_unsafe_target_item);
2535         req->r_fmode = -1;
2536         req->r_feature_needed = -1;
2537         kref_init(&req->r_kref);
2538         RB_CLEAR_NODE(&req->r_node);
2539         INIT_LIST_HEAD(&req->r_wait);
2540         init_completion(&req->r_completion);
2541         init_completion(&req->r_safe_completion);
2542         INIT_LIST_HEAD(&req->r_unsafe_item);
2543
2544         ktime_get_coarse_real_ts64(&req->r_stamp);
2545
2546         req->r_op = op;
2547         req->r_direct_mode = mode;
2548         return req;
2549 }
2550
2551 /*
2552  * return oldest (lowest) request, tid in request tree, 0 if none.
2553  *
2554  * called under mdsc->mutex.
2555  */
2556 static struct ceph_mds_request *__get_oldest_req(struct ceph_mds_client *mdsc)
2557 {
2558         if (RB_EMPTY_ROOT(&mdsc->request_tree))
2559                 return NULL;
2560         return rb_entry(rb_first(&mdsc->request_tree),
2561                         struct ceph_mds_request, r_node);
2562 }
2563
2564 static inline  u64 __get_oldest_tid(struct ceph_mds_client *mdsc)
2565 {
2566         return mdsc->oldest_tid;
2567 }
2568
2569 #if IS_ENABLED(CONFIG_FS_ENCRYPTION)
2570 static u8 *get_fscrypt_altname(const struct ceph_mds_request *req, u32 *plen)
2571 {
2572         struct inode *dir = req->r_parent;
2573         struct dentry *dentry = req->r_dentry;
2574         u8 *cryptbuf = NULL;
2575         u32 len = 0;
2576         int ret = 0;
2577
2578         /* only encode if we have parent and dentry */
2579         if (!dir || !dentry)
2580                 goto success;
2581
2582         /* No-op unless this is encrypted */
2583         if (!IS_ENCRYPTED(dir))
2584                 goto success;
2585
2586         ret = ceph_fscrypt_prepare_readdir(dir);
2587         if (ret < 0)
2588                 return ERR_PTR(ret);
2589
2590         /* No key? Just ignore it. */
2591         if (!fscrypt_has_encryption_key(dir))
2592                 goto success;
2593
2594         if (!fscrypt_fname_encrypted_size(dir, dentry->d_name.len, NAME_MAX,
2595                                           &len)) {
2596                 WARN_ON_ONCE(1);
2597                 return ERR_PTR(-ENAMETOOLONG);
2598         }
2599
2600         /* No need to append altname if name is short enough */
2601         if (len <= CEPH_NOHASH_NAME_MAX) {
2602                 len = 0;
2603                 goto success;
2604         }
2605
2606         cryptbuf = kmalloc(len, GFP_KERNEL);
2607         if (!cryptbuf)
2608                 return ERR_PTR(-ENOMEM);
2609
2610         ret = fscrypt_fname_encrypt(dir, &dentry->d_name, cryptbuf, len);
2611         if (ret) {
2612                 kfree(cryptbuf);
2613                 return ERR_PTR(ret);
2614         }
2615 success:
2616         *plen = len;
2617         return cryptbuf;
2618 }
2619 #else
2620 static u8 *get_fscrypt_altname(const struct ceph_mds_request *req, u32 *plen)
2621 {
2622         *plen = 0;
2623         return NULL;
2624 }
2625 #endif
2626
2627 /**
2628  * ceph_mdsc_build_path - build a path string to a given dentry
2629  * @mdsc: mds client
2630  * @dentry: dentry to which path should be built
2631  * @plen: returned length of string
2632  * @pbase: returned base inode number
2633  * @for_wire: is this path going to be sent to the MDS?
2634  *
2635  * Build a string that represents the path to the dentry. This is mostly called
2636  * for two different purposes:
2637  *
2638  * 1) we need to build a path string to send to the MDS (for_wire == true)
2639  * 2) we need a path string for local presentation (e.g. debugfs)
2640  *    (for_wire == false)
2641  *
2642  * The path is built in reverse, starting with the dentry. Walk back up toward
2643  * the root, building the path until the first non-snapped inode is reached
2644  * (for_wire) or the root inode is reached (!for_wire).
2645  *
2646  * Encode hidden .snap dirs as a double /, i.e.
2647  *   foo/.snap/bar -> foo//bar
2648  */
2649 char *ceph_mdsc_build_path(struct ceph_mds_client *mdsc, struct dentry *dentry,
2650                            int *plen, u64 *pbase, int for_wire)
2651 {
2652         struct ceph_client *cl = mdsc->fsc->client;
2653         struct dentry *cur;
2654         struct inode *inode;
2655         char *path;
2656         int pos;
2657         unsigned seq;
2658         u64 base;
2659
2660         if (!dentry)
2661                 return ERR_PTR(-EINVAL);
2662
2663         path = __getname();
2664         if (!path)
2665                 return ERR_PTR(-ENOMEM);
2666 retry:
2667         pos = PATH_MAX - 1;
2668         path[pos] = '\0';
2669
2670         seq = read_seqbegin(&rename_lock);
2671         cur = dget(dentry);
2672         for (;;) {
2673                 struct dentry *parent;
2674
2675                 spin_lock(&cur->d_lock);
2676                 inode = d_inode(cur);
2677                 if (inode && ceph_snap(inode) == CEPH_SNAPDIR) {
2678                         doutc(cl, "path+%d: %p SNAPDIR\n", pos, cur);
2679                         spin_unlock(&cur->d_lock);
2680                         parent = dget_parent(cur);
2681                 } else if (for_wire && inode && dentry != cur &&
2682                            ceph_snap(inode) == CEPH_NOSNAP) {
2683                         spin_unlock(&cur->d_lock);
2684                         pos++; /* get rid of any prepended '/' */
2685                         break;
2686                 } else if (!for_wire || !IS_ENCRYPTED(d_inode(cur->d_parent))) {
2687                         pos -= cur->d_name.len;
2688                         if (pos < 0) {
2689                                 spin_unlock(&cur->d_lock);
2690                                 break;
2691                         }
2692                         memcpy(path + pos, cur->d_name.name, cur->d_name.len);
2693                         spin_unlock(&cur->d_lock);
2694                         parent = dget_parent(cur);
2695                 } else {
2696                         int len, ret;
2697                         char buf[NAME_MAX];
2698
2699                         /*
2700                          * Proactively copy name into buf, in case we need to
2701                          * present it as-is.
2702                          */
2703                         memcpy(buf, cur->d_name.name, cur->d_name.len);
2704                         len = cur->d_name.len;
2705                         spin_unlock(&cur->d_lock);
2706                         parent = dget_parent(cur);
2707
2708                         ret = ceph_fscrypt_prepare_readdir(d_inode(parent));
2709                         if (ret < 0) {
2710                                 dput(parent);
2711                                 dput(cur);
2712                                 return ERR_PTR(ret);
2713                         }
2714
2715                         if (fscrypt_has_encryption_key(d_inode(parent))) {
2716                                 len = ceph_encode_encrypted_fname(d_inode(parent),
2717                                                                   cur, buf);
2718                                 if (len < 0) {
2719                                         dput(parent);
2720                                         dput(cur);
2721                                         return ERR_PTR(len);
2722                                 }
2723                         }
2724                         pos -= len;
2725                         if (pos < 0) {
2726                                 dput(parent);
2727                                 break;
2728                         }
2729                         memcpy(path + pos, buf, len);
2730                 }
2731                 dput(cur);
2732                 cur = parent;
2733
2734                 /* Are we at the root? */
2735                 if (IS_ROOT(cur))
2736                         break;
2737
2738                 /* Are we out of buffer? */
2739                 if (--pos < 0)
2740                         break;
2741
2742                 path[pos] = '/';
2743         }
2744         inode = d_inode(cur);
2745         base = inode ? ceph_ino(inode) : 0;
2746         dput(cur);
2747
2748         if (read_seqretry(&rename_lock, seq))
2749                 goto retry;
2750
2751         if (pos < 0) {
2752                 /*
2753                  * A rename didn't occur, but somehow we didn't end up where
2754                  * we thought we would. Throw a warning and try again.
2755                  */
2756                 pr_warn_client(cl, "did not end path lookup where expected (pos = %d)\n",
2757                                pos);
2758                 goto retry;
2759         }
2760
2761         *pbase = base;
2762         *plen = PATH_MAX - 1 - pos;
2763         doutc(cl, "on %p %d built %llx '%.*s'\n", dentry, d_count(dentry),
2764               base, *plen, path + pos);
2765         return path + pos;
2766 }
2767
2768 static int build_dentry_path(struct ceph_mds_client *mdsc, struct dentry *dentry,
2769                              struct inode *dir, const char **ppath, int *ppathlen,
2770                              u64 *pino, bool *pfreepath, bool parent_locked)
2771 {
2772         char *path;
2773
2774         rcu_read_lock();
2775         if (!dir)
2776                 dir = d_inode_rcu(dentry->d_parent);
2777         if (dir && parent_locked && ceph_snap(dir) == CEPH_NOSNAP &&
2778             !IS_ENCRYPTED(dir)) {
2779                 *pino = ceph_ino(dir);
2780                 rcu_read_unlock();
2781                 *ppath = dentry->d_name.name;
2782                 *ppathlen = dentry->d_name.len;
2783                 return 0;
2784         }
2785         rcu_read_unlock();
2786         path = ceph_mdsc_build_path(mdsc, dentry, ppathlen, pino, 1);
2787         if (IS_ERR(path))
2788                 return PTR_ERR(path);
2789         *ppath = path;
2790         *pfreepath = true;
2791         return 0;
2792 }
2793
2794 static int build_inode_path(struct inode *inode,
2795                             const char **ppath, int *ppathlen, u64 *pino,
2796                             bool *pfreepath)
2797 {
2798         struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb);
2799         struct dentry *dentry;
2800         char *path;
2801
2802         if (ceph_snap(inode) == CEPH_NOSNAP) {
2803                 *pino = ceph_ino(inode);
2804                 *ppathlen = 0;
2805                 return 0;
2806         }
2807         dentry = d_find_alias(inode);
2808         path = ceph_mdsc_build_path(mdsc, dentry, ppathlen, pino, 1);
2809         dput(dentry);
2810         if (IS_ERR(path))
2811                 return PTR_ERR(path);
2812         *ppath = path;
2813         *pfreepath = true;
2814         return 0;
2815 }
2816
2817 /*
2818  * request arguments may be specified via an inode *, a dentry *, or
2819  * an explicit ino+path.
2820  */
2821 static int set_request_path_attr(struct ceph_mds_client *mdsc, struct inode *rinode,
2822                                  struct dentry *rdentry, struct inode *rdiri,
2823                                  const char *rpath, u64 rino, const char **ppath,
2824                                  int *pathlen, u64 *ino, bool *freepath,
2825                                  bool parent_locked)
2826 {
2827         struct ceph_client *cl = mdsc->fsc->client;
2828         int r = 0;
2829
2830         if (rinode) {
2831                 r = build_inode_path(rinode, ppath, pathlen, ino, freepath);
2832                 doutc(cl, " inode %p %llx.%llx\n", rinode, ceph_ino(rinode),
2833                       ceph_snap(rinode));
2834         } else if (rdentry) {
2835                 r = build_dentry_path(mdsc, rdentry, rdiri, ppath, pathlen, ino,
2836                                         freepath, parent_locked);
2837                 doutc(cl, " dentry %p %llx/%.*s\n", rdentry, *ino, *pathlen, *ppath);
2838         } else if (rpath || rino) {
2839                 *ino = rino;
2840                 *ppath = rpath;
2841                 *pathlen = rpath ? strlen(rpath) : 0;
2842                 doutc(cl, " path %.*s\n", *pathlen, rpath);
2843         }
2844
2845         return r;
2846 }
2847
2848 static void encode_mclientrequest_tail(void **p,
2849                                        const struct ceph_mds_request *req)
2850 {
2851         struct ceph_timespec ts;
2852         int i;
2853
2854         ceph_encode_timespec64(&ts, &req->r_stamp);
2855         ceph_encode_copy(p, &ts, sizeof(ts));
2856
2857         /* v4: gid_list */
2858         ceph_encode_32(p, req->r_cred->group_info->ngroups);
2859         for (i = 0; i < req->r_cred->group_info->ngroups; i++)
2860                 ceph_encode_64(p, from_kgid(&init_user_ns,
2861                                             req->r_cred->group_info->gid[i]));
2862
2863         /* v5: altname */
2864         ceph_encode_32(p, req->r_altname_len);
2865         ceph_encode_copy(p, req->r_altname, req->r_altname_len);
2866
2867         /* v6: fscrypt_auth and fscrypt_file */
2868         if (req->r_fscrypt_auth) {
2869                 u32 authlen = ceph_fscrypt_auth_len(req->r_fscrypt_auth);
2870
2871                 ceph_encode_32(p, authlen);
2872                 ceph_encode_copy(p, req->r_fscrypt_auth, authlen);
2873         } else {
2874                 ceph_encode_32(p, 0);
2875         }
2876         if (test_bit(CEPH_MDS_R_FSCRYPT_FILE, &req->r_req_flags)) {
2877                 ceph_encode_32(p, sizeof(__le64));
2878                 ceph_encode_64(p, req->r_fscrypt_file);
2879         } else {
2880                 ceph_encode_32(p, 0);
2881         }
2882 }
2883
2884 static inline u16 mds_supported_head_version(struct ceph_mds_session *session)
2885 {
2886         if (!test_bit(CEPHFS_FEATURE_32BITS_RETRY_FWD, &session->s_features))
2887                 return 1;
2888
2889         if (!test_bit(CEPHFS_FEATURE_HAS_OWNER_UIDGID, &session->s_features))
2890                 return 2;
2891
2892         return CEPH_MDS_REQUEST_HEAD_VERSION;
2893 }
2894
2895 static struct ceph_mds_request_head_legacy *
2896 find_legacy_request_head(void *p, u64 features)
2897 {
2898         bool legacy = !(features & CEPH_FEATURE_FS_BTIME);
2899         struct ceph_mds_request_head_old *ohead;
2900
2901         if (legacy)
2902                 return (struct ceph_mds_request_head_legacy *)p;
2903         ohead = (struct ceph_mds_request_head_old *)p;
2904         return (struct ceph_mds_request_head_legacy *)&ohead->oldest_client_tid;
2905 }
2906
2907 /*
2908  * called under mdsc->mutex
2909  */
2910 static struct ceph_msg *create_request_message(struct ceph_mds_session *session,
2911                                                struct ceph_mds_request *req,
2912                                                bool drop_cap_releases)
2913 {
2914         int mds = session->s_mds;
2915         struct ceph_mds_client *mdsc = session->s_mdsc;
2916         struct ceph_client *cl = mdsc->fsc->client;
2917         struct ceph_msg *msg;
2918         struct ceph_mds_request_head_legacy *lhead;
2919         const char *path1 = NULL;
2920         const char *path2 = NULL;
2921         u64 ino1 = 0, ino2 = 0;
2922         int pathlen1 = 0, pathlen2 = 0;
2923         bool freepath1 = false, freepath2 = false;
2924         struct dentry *old_dentry = NULL;
2925         int len;
2926         u16 releases;
2927         void *p, *end;
2928         int ret;
2929         bool legacy = !(session->s_con.peer_features & CEPH_FEATURE_FS_BTIME);
2930         u16 request_head_version = mds_supported_head_version(session);
2931         kuid_t caller_fsuid = req->r_cred->fsuid;
2932         kgid_t caller_fsgid = req->r_cred->fsgid;
2933
2934         ret = set_request_path_attr(mdsc, req->r_inode, req->r_dentry,
2935                               req->r_parent, req->r_path1, req->r_ino1.ino,
2936                               &path1, &pathlen1, &ino1, &freepath1,
2937                               test_bit(CEPH_MDS_R_PARENT_LOCKED,
2938                                         &req->r_req_flags));
2939         if (ret < 0) {
2940                 msg = ERR_PTR(ret);
2941                 goto out;
2942         }
2943
2944         /* If r_old_dentry is set, then assume that its parent is locked */
2945         if (req->r_old_dentry &&
2946             !(req->r_old_dentry->d_flags & DCACHE_DISCONNECTED))
2947                 old_dentry = req->r_old_dentry;
2948         ret = set_request_path_attr(mdsc, NULL, old_dentry,
2949                               req->r_old_dentry_dir,
2950                               req->r_path2, req->r_ino2.ino,
2951                               &path2, &pathlen2, &ino2, &freepath2, true);
2952         if (ret < 0) {
2953                 msg = ERR_PTR(ret);
2954                 goto out_free1;
2955         }
2956
2957         req->r_altname = get_fscrypt_altname(req, &req->r_altname_len);
2958         if (IS_ERR(req->r_altname)) {
2959                 msg = ERR_CAST(req->r_altname);
2960                 req->r_altname = NULL;
2961                 goto out_free2;
2962         }
2963
2964         /*
2965          * For old cephs without supporting the 32bit retry/fwd feature
2966          * it will copy the raw memories directly when decoding the
2967          * requests. While new cephs will decode the head depending the
2968          * version member, so we need to make sure it will be compatible
2969          * with them both.
2970          */
2971         if (legacy)
2972                 len = sizeof(struct ceph_mds_request_head_legacy);
2973         else if (request_head_version == 1)
2974                 len = sizeof(struct ceph_mds_request_head_old);
2975         else if (request_head_version == 2)
2976                 len = offsetofend(struct ceph_mds_request_head, ext_num_fwd);
2977         else
2978                 len = sizeof(struct ceph_mds_request_head);
2979
2980         /* filepaths */
2981         len += 2 * (1 + sizeof(u32) + sizeof(u64));
2982         len += pathlen1 + pathlen2;
2983
2984         /* cap releases */
2985         len += sizeof(struct ceph_mds_request_release) *
2986                 (!!req->r_inode_drop + !!req->r_dentry_drop +
2987                  !!req->r_old_inode_drop + !!req->r_old_dentry_drop);
2988
2989         if (req->r_dentry_drop)
2990                 len += pathlen1;
2991         if (req->r_old_dentry_drop)
2992                 len += pathlen2;
2993
2994         /* MClientRequest tail */
2995
2996         /* req->r_stamp */
2997         len += sizeof(struct ceph_timespec);
2998
2999         /* gid list */
3000         len += sizeof(u32) + (sizeof(u64) * req->r_cred->group_info->ngroups);
3001
3002         /* alternate name */
3003         len += sizeof(u32) + req->r_altname_len;
3004
3005         /* fscrypt_auth */
3006         len += sizeof(u32); // fscrypt_auth
3007         if (req->r_fscrypt_auth)
3008                 len += ceph_fscrypt_auth_len(req->r_fscrypt_auth);
3009
3010         /* fscrypt_file */
3011         len += sizeof(u32);
3012         if (test_bit(CEPH_MDS_R_FSCRYPT_FILE, &req->r_req_flags))
3013                 len += sizeof(__le64);
3014
3015         msg = ceph_msg_new2(CEPH_MSG_CLIENT_REQUEST, len, 1, GFP_NOFS, false);
3016         if (!msg) {
3017                 msg = ERR_PTR(-ENOMEM);
3018                 goto out_free2;
3019         }
3020
3021         msg->hdr.tid = cpu_to_le64(req->r_tid);
3022
3023         lhead = find_legacy_request_head(msg->front.iov_base,
3024                                          session->s_con.peer_features);
3025
3026         if ((req->r_mnt_idmap != &nop_mnt_idmap) &&
3027             !test_bit(CEPHFS_FEATURE_HAS_OWNER_UIDGID, &session->s_features)) {
3028                 WARN_ON_ONCE(!IS_CEPH_MDS_OP_NEWINODE(req->r_op));
3029
3030                 if (enable_unsafe_idmap) {
3031                         pr_warn_once_client(cl,
3032                                 "idmapped mount is used and CEPHFS_FEATURE_HAS_OWNER_UIDGID"
3033                                 " is not supported by MDS. UID/GID-based restrictions may"
3034                                 " not work properly.\n");
3035
3036                         caller_fsuid = from_vfsuid(req->r_mnt_idmap, &init_user_ns,
3037                                                    VFSUIDT_INIT(req->r_cred->fsuid));
3038                         caller_fsgid = from_vfsgid(req->r_mnt_idmap, &init_user_ns,
3039                                                    VFSGIDT_INIT(req->r_cred->fsgid));
3040                 } else {
3041                         pr_err_ratelimited_client(cl,
3042                                 "idmapped mount is used and CEPHFS_FEATURE_HAS_OWNER_UIDGID"
3043                                 " is not supported by MDS. Fail request with -EIO.\n");
3044
3045                         ret = -EIO;
3046                         goto out_err;
3047                 }
3048         }
3049
3050         /*
3051          * The ceph_mds_request_head_legacy didn't contain a version field, and
3052          * one was added when we moved the message version from 3->4.
3053          */
3054         if (legacy) {
3055                 msg->hdr.version = cpu_to_le16(3);
3056                 p = msg->front.iov_base + sizeof(*lhead);
3057         } else if (request_head_version == 1) {
3058                 struct ceph_mds_request_head_old *ohead = msg->front.iov_base;
3059
3060                 msg->hdr.version = cpu_to_le16(4);
3061                 ohead->version = cpu_to_le16(1);
3062                 p = msg->front.iov_base + sizeof(*ohead);
3063         } else if (request_head_version == 2) {
3064                 struct ceph_mds_request_head *nhead = msg->front.iov_base;
3065
3066                 msg->hdr.version = cpu_to_le16(6);
3067                 nhead->version = cpu_to_le16(2);
3068
3069                 p = msg->front.iov_base + offsetofend(struct ceph_mds_request_head, ext_num_fwd);
3070         } else {
3071                 struct ceph_mds_request_head *nhead = msg->front.iov_base;
3072                 kuid_t owner_fsuid;
3073                 kgid_t owner_fsgid;
3074
3075                 msg->hdr.version = cpu_to_le16(6);
3076                 nhead->version = cpu_to_le16(CEPH_MDS_REQUEST_HEAD_VERSION);
3077                 nhead->struct_len = cpu_to_le32(sizeof(struct ceph_mds_request_head));
3078
3079                 if (IS_CEPH_MDS_OP_NEWINODE(req->r_op)) {
3080                         owner_fsuid = from_vfsuid(req->r_mnt_idmap, &init_user_ns,
3081                                                 VFSUIDT_INIT(req->r_cred->fsuid));
3082                         owner_fsgid = from_vfsgid(req->r_mnt_idmap, &init_user_ns,
3083                                                 VFSGIDT_INIT(req->r_cred->fsgid));
3084                         nhead->owner_uid = cpu_to_le32(from_kuid(&init_user_ns, owner_fsuid));
3085                         nhead->owner_gid = cpu_to_le32(from_kgid(&init_user_ns, owner_fsgid));
3086                 } else {
3087                         nhead->owner_uid = cpu_to_le32(-1);
3088                         nhead->owner_gid = cpu_to_le32(-1);
3089                 }
3090
3091                 p = msg->front.iov_base + sizeof(*nhead);
3092         }
3093
3094         end = msg->front.iov_base + msg->front.iov_len;
3095
3096         lhead->mdsmap_epoch = cpu_to_le32(mdsc->mdsmap->m_epoch);
3097         lhead->op = cpu_to_le32(req->r_op);
3098         lhead->caller_uid = cpu_to_le32(from_kuid(&init_user_ns,
3099                                                   caller_fsuid));
3100         lhead->caller_gid = cpu_to_le32(from_kgid(&init_user_ns,
3101                                                   caller_fsgid));
3102         lhead->ino = cpu_to_le64(req->r_deleg_ino);
3103         lhead->args = req->r_args;
3104
3105         ceph_encode_filepath(&p, end, ino1, path1);
3106         ceph_encode_filepath(&p, end, ino2, path2);
3107
3108         /* make note of release offset, in case we need to replay */
3109         req->r_request_release_offset = p - msg->front.iov_base;
3110
3111         /* cap releases */
3112         releases = 0;
3113         if (req->r_inode_drop)
3114                 releases += ceph_encode_inode_release(&p,
3115                       req->r_inode ? req->r_inode : d_inode(req->r_dentry),
3116                       mds, req->r_inode_drop, req->r_inode_unless,
3117                       req->r_op == CEPH_MDS_OP_READDIR);
3118         if (req->r_dentry_drop) {
3119                 ret = ceph_encode_dentry_release(&p, req->r_dentry,
3120                                 req->r_parent, mds, req->r_dentry_drop,
3121                                 req->r_dentry_unless);
3122                 if (ret < 0)
3123                         goto out_err;
3124                 releases += ret;
3125         }
3126         if (req->r_old_dentry_drop) {
3127                 ret = ceph_encode_dentry_release(&p, req->r_old_dentry,
3128                                 req->r_old_dentry_dir, mds,
3129                                 req->r_old_dentry_drop,
3130                                 req->r_old_dentry_unless);
3131                 if (ret < 0)
3132                         goto out_err;
3133                 releases += ret;
3134         }
3135         if (req->r_old_inode_drop)
3136                 releases += ceph_encode_inode_release(&p,
3137                       d_inode(req->r_old_dentry),
3138                       mds, req->r_old_inode_drop, req->r_old_inode_unless, 0);
3139
3140         if (drop_cap_releases) {
3141                 releases = 0;
3142                 p = msg->front.iov_base + req->r_request_release_offset;
3143         }
3144
3145         lhead->num_releases = cpu_to_le16(releases);
3146
3147         encode_mclientrequest_tail(&p, req);
3148
3149         if (WARN_ON_ONCE(p > end)) {
3150                 ceph_msg_put(msg);
3151                 msg = ERR_PTR(-ERANGE);
3152                 goto out_free2;
3153         }
3154
3155         msg->front.iov_len = p - msg->front.iov_base;
3156         msg->hdr.front_len = cpu_to_le32(msg->front.iov_len);
3157
3158         if (req->r_pagelist) {
3159                 struct ceph_pagelist *pagelist = req->r_pagelist;
3160                 ceph_msg_data_add_pagelist(msg, pagelist);
3161                 msg->hdr.data_len = cpu_to_le32(pagelist->length);
3162         } else {
3163                 msg->hdr.data_len = 0;
3164         }
3165
3166         msg->hdr.data_off = cpu_to_le16(0);
3167
3168 out_free2:
3169         if (freepath2)
3170                 ceph_mdsc_free_path((char *)path2, pathlen2);
3171 out_free1:
3172         if (freepath1)
3173                 ceph_mdsc_free_path((char *)path1, pathlen1);
3174 out:
3175         return msg;
3176 out_err:
3177         ceph_msg_put(msg);
3178         msg = ERR_PTR(ret);
3179         goto out_free2;
3180 }
3181
3182 /*
3183  * called under mdsc->mutex if error, under no mutex if
3184  * success.
3185  */
3186 static void complete_request(struct ceph_mds_client *mdsc,
3187                              struct ceph_mds_request *req)
3188 {
3189         req->r_end_latency = ktime_get();
3190
3191         if (req->r_callback)
3192                 req->r_callback(mdsc, req);
3193         complete_all(&req->r_completion);
3194 }
3195
3196 /*
3197  * called under mdsc->mutex
3198  */
3199 static int __prepare_send_request(struct ceph_mds_session *session,
3200                                   struct ceph_mds_request *req,
3201                                   bool drop_cap_releases)
3202 {
3203         int mds = session->s_mds;
3204         struct ceph_mds_client *mdsc = session->s_mdsc;
3205         struct ceph_client *cl = mdsc->fsc->client;
3206         struct ceph_mds_request_head_legacy *lhead;
3207         struct ceph_mds_request_head *nhead;
3208         struct ceph_msg *msg;
3209         int flags = 0, old_max_retry;
3210         bool old_version = !test_bit(CEPHFS_FEATURE_32BITS_RETRY_FWD,
3211                                      &session->s_features);
3212
3213         /*
3214          * Avoid inifinite retrying after overflow. The client will
3215          * increase the retry count and if the MDS is old version,
3216          * so we limit to retry at most 256 times.
3217          */
3218         if (req->r_attempts) {
3219                old_max_retry = sizeof_field(struct ceph_mds_request_head_old,
3220                                             num_retry);
3221                old_max_retry = 1 << (old_max_retry * BITS_PER_BYTE);
3222                if ((old_version && req->r_attempts >= old_max_retry) ||
3223                    ((uint32_t)req->r_attempts >= U32_MAX)) {
3224                         pr_warn_ratelimited_client(cl, "request tid %llu seq overflow\n",
3225                                                    req->r_tid);
3226                         return -EMULTIHOP;
3227                }
3228         }
3229
3230         req->r_attempts++;
3231         if (req->r_inode) {
3232                 struct ceph_cap *cap =
3233                         ceph_get_cap_for_mds(ceph_inode(req->r_inode), mds);
3234
3235                 if (cap)
3236                         req->r_sent_on_mseq = cap->mseq;
3237                 else
3238                         req->r_sent_on_mseq = -1;
3239         }
3240         doutc(cl, "%p tid %lld %s (attempt %d)\n", req, req->r_tid,
3241               ceph_mds_op_name(req->r_op), req->r_attempts);
3242
3243         if (test_bit(CEPH_MDS_R_GOT_UNSAFE, &req->r_req_flags)) {
3244                 void *p;
3245
3246                 /*
3247                  * Replay.  Do not regenerate message (and rebuild
3248                  * paths, etc.); just use the original message.
3249                  * Rebuilding paths will break for renames because
3250                  * d_move mangles the src name.
3251                  */
3252                 msg = req->r_request;
3253                 lhead = find_legacy_request_head(msg->front.iov_base,
3254                                                  session->s_con.peer_features);
3255
3256                 flags = le32_to_cpu(lhead->flags);
3257                 flags |= CEPH_MDS_FLAG_REPLAY;
3258                 lhead->flags = cpu_to_le32(flags);
3259
3260                 if (req->r_target_inode)
3261                         lhead->ino = cpu_to_le64(ceph_ino(req->r_target_inode));
3262
3263                 lhead->num_retry = req->r_attempts - 1;
3264                 if (!old_version) {
3265                         nhead = (struct ceph_mds_request_head*)msg->front.iov_base;
3266                         nhead->ext_num_retry = cpu_to_le32(req->r_attempts - 1);
3267                 }
3268
3269                 /* remove cap/dentry releases from message */
3270                 lhead->num_releases = 0;
3271
3272                 p = msg->front.iov_base + req->r_request_release_offset;
3273                 encode_mclientrequest_tail(&p, req);
3274
3275                 msg->front.iov_len = p - msg->front.iov_base;
3276                 msg->hdr.front_len = cpu_to_le32(msg->front.iov_len);
3277                 return 0;
3278         }
3279
3280         if (req->r_request) {
3281                 ceph_msg_put(req->r_request);
3282                 req->r_request = NULL;
3283         }
3284         msg = create_request_message(session, req, drop_cap_releases);
3285         if (IS_ERR(msg)) {
3286                 req->r_err = PTR_ERR(msg);
3287                 return PTR_ERR(msg);
3288         }
3289         req->r_request = msg;
3290
3291         lhead = find_legacy_request_head(msg->front.iov_base,
3292                                          session->s_con.peer_features);
3293         lhead->oldest_client_tid = cpu_to_le64(__get_oldest_tid(mdsc));
3294         if (test_bit(CEPH_MDS_R_GOT_UNSAFE, &req->r_req_flags))
3295                 flags |= CEPH_MDS_FLAG_REPLAY;
3296         if (test_bit(CEPH_MDS_R_ASYNC, &req->r_req_flags))
3297                 flags |= CEPH_MDS_FLAG_ASYNC;
3298         if (req->r_parent)
3299                 flags |= CEPH_MDS_FLAG_WANT_DENTRY;
3300         lhead->flags = cpu_to_le32(flags);
3301         lhead->num_fwd = req->r_num_fwd;
3302         lhead->num_retry = req->r_attempts - 1;
3303         if (!old_version) {
3304                 nhead = (struct ceph_mds_request_head*)msg->front.iov_base;
3305                 nhead->ext_num_fwd = cpu_to_le32(req->r_num_fwd);
3306                 nhead->ext_num_retry = cpu_to_le32(req->r_attempts - 1);
3307         }
3308
3309         doutc(cl, " r_parent = %p\n", req->r_parent);
3310         return 0;
3311 }
3312
3313 /*
3314  * called under mdsc->mutex
3315  */
3316 static int __send_request(struct ceph_mds_session *session,
3317                           struct ceph_mds_request *req,
3318                           bool drop_cap_releases)
3319 {
3320         int err;
3321
3322         err = __prepare_send_request(session, req, drop_cap_releases);
3323         if (!err) {
3324                 ceph_msg_get(req->r_request);
3325                 ceph_con_send(&session->s_con, req->r_request);
3326         }
3327
3328         return err;
3329 }
3330
3331 /*
3332  * send request, or put it on the appropriate wait list.
3333  */
3334 static void __do_request(struct ceph_mds_client *mdsc,
3335                         struct ceph_mds_request *req)
3336 {
3337         struct ceph_client *cl = mdsc->fsc->client;
3338         struct ceph_mds_session *session = NULL;
3339         int mds = -1;
3340         int err = 0;
3341         bool random;
3342
3343         if (req->r_err || test_bit(CEPH_MDS_R_GOT_RESULT, &req->r_req_flags)) {
3344                 if (test_bit(CEPH_MDS_R_ABORTED, &req->r_req_flags))
3345                         __unregister_request(mdsc, req);
3346                 return;
3347         }
3348
3349         if (READ_ONCE(mdsc->fsc->mount_state) == CEPH_MOUNT_FENCE_IO) {
3350                 doutc(cl, "metadata corrupted\n");
3351                 err = -EIO;
3352                 goto finish;
3353         }
3354         if (req->r_timeout &&
3355             time_after_eq(jiffies, req->r_started + req->r_timeout)) {
3356                 doutc(cl, "timed out\n");
3357                 err = -ETIMEDOUT;
3358                 goto finish;
3359         }
3360         if (READ_ONCE(mdsc->fsc->mount_state) == CEPH_MOUNT_SHUTDOWN) {
3361                 doutc(cl, "forced umount\n");
3362                 err = -EIO;
3363                 goto finish;
3364         }
3365         if (READ_ONCE(mdsc->fsc->mount_state) == CEPH_MOUNT_MOUNTING) {
3366                 if (mdsc->mdsmap_err) {
3367                         err = mdsc->mdsmap_err;
3368                         doutc(cl, "mdsmap err %d\n", err);
3369                         goto finish;
3370                 }
3371                 if (mdsc->mdsmap->m_epoch == 0) {
3372                         doutc(cl, "no mdsmap, waiting for map\n");
3373                         list_add(&req->r_wait, &mdsc->waiting_for_map);
3374                         return;
3375                 }
3376                 if (!(mdsc->fsc->mount_options->flags &
3377                       CEPH_MOUNT_OPT_MOUNTWAIT) &&
3378                     !ceph_mdsmap_is_cluster_available(mdsc->mdsmap)) {
3379                         err = -EHOSTUNREACH;
3380                         goto finish;
3381                 }
3382         }
3383
3384         put_request_session(req);
3385
3386         mds = __choose_mds(mdsc, req, &random);
3387         if (mds < 0 ||
3388             ceph_mdsmap_get_state(mdsc->mdsmap, mds) < CEPH_MDS_STATE_ACTIVE) {
3389                 if (test_bit(CEPH_MDS_R_ASYNC, &req->r_req_flags)) {
3390                         err = -EJUKEBOX;
3391                         goto finish;
3392                 }
3393                 doutc(cl, "no mds or not active, waiting for map\n");
3394                 list_add(&req->r_wait, &mdsc->waiting_for_map);
3395                 return;
3396         }
3397
3398         /* get, open session */
3399         session = __ceph_lookup_mds_session(mdsc, mds);
3400         if (!session) {
3401                 session = register_session(mdsc, mds);
3402                 if (IS_ERR(session)) {
3403                         err = PTR_ERR(session);
3404                         goto finish;
3405                 }
3406         }
3407         req->r_session = ceph_get_mds_session(session);
3408
3409         doutc(cl, "mds%d session %p state %s\n", mds, session,
3410               ceph_session_state_name(session->s_state));
3411
3412         /*
3413          * The old ceph will crash the MDSs when see unknown OPs
3414          */
3415         if (req->r_feature_needed > 0 &&
3416             !test_bit(req->r_feature_needed, &session->s_features)) {
3417                 err = -EOPNOTSUPP;
3418                 goto out_session;
3419         }
3420
3421         if (session->s_state != CEPH_MDS_SESSION_OPEN &&
3422             session->s_state != CEPH_MDS_SESSION_HUNG) {
3423                 /*
3424                  * We cannot queue async requests since the caps and delegated
3425                  * inodes are bound to the session. Just return -EJUKEBOX and
3426                  * let the caller retry a sync request in that case.
3427                  */
3428                 if (test_bit(CEPH_MDS_R_ASYNC, &req->r_req_flags)) {
3429                         err = -EJUKEBOX;
3430                         goto out_session;
3431                 }
3432
3433                 /*
3434                  * If the session has been REJECTED, then return a hard error,
3435                  * unless it's a CLEANRECOVER mount, in which case we'll queue
3436                  * it to the mdsc queue.
3437                  */
3438                 if (session->s_state == CEPH_MDS_SESSION_REJECTED) {
3439                         if (ceph_test_mount_opt(mdsc->fsc, CLEANRECOVER))
3440                                 list_add(&req->r_wait, &mdsc->waiting_for_map);
3441                         else
3442                                 err = -EACCES;
3443                         goto out_session;
3444                 }
3445
3446                 if (session->s_state == CEPH_MDS_SESSION_NEW ||
3447                     session->s_state == CEPH_MDS_SESSION_CLOSING) {
3448                         err = __open_session(mdsc, session);
3449                         if (err)
3450                                 goto out_session;
3451                         /* retry the same mds later */
3452                         if (random)
3453                                 req->r_resend_mds = mds;
3454                 }
3455                 list_add(&req->r_wait, &session->s_waiting);
3456                 goto out_session;
3457         }
3458
3459         /* send request */
3460         req->r_resend_mds = -1;   /* forget any previous mds hint */
3461
3462         if (req->r_request_started == 0)   /* note request start time */
3463                 req->r_request_started = jiffies;
3464
3465         /*
3466          * For async create we will choose the auth MDS of frag in parent
3467          * directory to send the request and ususally this works fine, but
3468          * if the migrated the dirtory to another MDS before it could handle
3469          * it the request will be forwarded.
3470          *
3471          * And then the auth cap will be changed.
3472          */
3473         if (test_bit(CEPH_MDS_R_ASYNC, &req->r_req_flags) && req->r_num_fwd) {
3474                 struct ceph_dentry_info *di = ceph_dentry(req->r_dentry);
3475                 struct ceph_inode_info *ci;
3476                 struct ceph_cap *cap;
3477
3478                 /*
3479                  * The request maybe handled very fast and the new inode
3480                  * hasn't been linked to the dentry yet. We need to wait
3481                  * for the ceph_finish_async_create(), which shouldn't be
3482                  * stuck too long or fail in thoery, to finish when forwarding
3483                  * the request.
3484                  */
3485                 if (!d_inode(req->r_dentry)) {
3486                         err = wait_on_bit(&di->flags, CEPH_DENTRY_ASYNC_CREATE_BIT,
3487                                           TASK_KILLABLE);
3488                         if (err) {
3489                                 mutex_lock(&req->r_fill_mutex);
3490                                 set_bit(CEPH_MDS_R_ABORTED, &req->r_req_flags);
3491                                 mutex_unlock(&req->r_fill_mutex);
3492                                 goto out_session;
3493                         }
3494                 }
3495
3496                 ci = ceph_inode(d_inode(req->r_dentry));
3497
3498                 spin_lock(&ci->i_ceph_lock);
3499                 cap = ci->i_auth_cap;
3500                 if (ci->i_ceph_flags & CEPH_I_ASYNC_CREATE && mds != cap->mds) {
3501                         doutc(cl, "session changed for auth cap %d -> %d\n",
3502                               cap->session->s_mds, session->s_mds);
3503
3504                         /* Remove the auth cap from old session */
3505                         spin_lock(&cap->session->s_cap_lock);
3506                         cap->session->s_nr_caps--;
3507                         list_del_init(&cap->session_caps);
3508                         spin_unlock(&cap->session->s_cap_lock);
3509
3510                         /* Add the auth cap to the new session */
3511                         cap->mds = mds;
3512                         cap->session = session;
3513                         spin_lock(&session->s_cap_lock);
3514                         session->s_nr_caps++;
3515                         list_add_tail(&cap->session_caps, &session->s_caps);
3516                         spin_unlock(&session->s_cap_lock);
3517
3518                         change_auth_cap_ses(ci, session);
3519                 }
3520                 spin_unlock(&ci->i_ceph_lock);
3521         }
3522
3523         err = __send_request(session, req, false);
3524
3525 out_session:
3526         ceph_put_mds_session(session);
3527 finish:
3528         if (err) {
3529                 doutc(cl, "early error %d\n", err);
3530                 req->r_err = err;
3531                 complete_request(mdsc, req);
3532                 __unregister_request(mdsc, req);
3533         }
3534         return;
3535 }
3536
3537 /*
3538  * called under mdsc->mutex
3539  */
3540 static void __wake_requests(struct ceph_mds_client *mdsc,
3541                             struct list_head *head)
3542 {
3543         struct ceph_client *cl = mdsc->fsc->client;
3544         struct ceph_mds_request *req;
3545         LIST_HEAD(tmp_list);
3546
3547         list_splice_init(head, &tmp_list);
3548
3549         while (!list_empty(&tmp_list)) {
3550                 req = list_entry(tmp_list.next,
3551                                  struct ceph_mds_request, r_wait);
3552                 list_del_init(&req->r_wait);
3553                 doutc(cl, " wake request %p tid %llu\n", req,
3554                       req->r_tid);
3555                 __do_request(mdsc, req);
3556         }
3557 }
3558
3559 /*
3560  * Wake up threads with requests pending for @mds, so that they can
3561  * resubmit their requests to a possibly different mds.
3562  */
3563 static void kick_requests(struct ceph_mds_client *mdsc, int mds)
3564 {
3565         struct ceph_client *cl = mdsc->fsc->client;
3566         struct ceph_mds_request *req;
3567         struct rb_node *p = rb_first(&mdsc->request_tree);
3568
3569         doutc(cl, "kick_requests mds%d\n", mds);
3570         while (p) {
3571                 req = rb_entry(p, struct ceph_mds_request, r_node);
3572                 p = rb_next(p);
3573                 if (test_bit(CEPH_MDS_R_GOT_UNSAFE, &req->r_req_flags))
3574                         continue;
3575                 if (req->r_attempts > 0)
3576                         continue; /* only new requests */
3577                 if (req->r_session &&
3578                     req->r_session->s_mds == mds) {
3579                         doutc(cl, " kicking tid %llu\n", req->r_tid);
3580                         list_del_init(&req->r_wait);
3581                         __do_request(mdsc, req);
3582                 }
3583         }
3584 }
3585
3586 int ceph_mdsc_submit_request(struct ceph_mds_client *mdsc, struct inode *dir,
3587                               struct ceph_mds_request *req)
3588 {
3589         struct ceph_client *cl = mdsc->fsc->client;
3590         int err = 0;
3591
3592         /* take CAP_PIN refs for r_inode, r_parent, r_old_dentry */
3593         if (req->r_inode)
3594                 ceph_get_cap_refs(ceph_inode(req->r_inode), CEPH_CAP_PIN);
3595         if (req->r_parent) {
3596                 struct ceph_inode_info *ci = ceph_inode(req->r_parent);
3597                 int fmode = (req->r_op & CEPH_MDS_OP_WRITE) ?
3598                             CEPH_FILE_MODE_WR : CEPH_FILE_MODE_RD;
3599                 spin_lock(&ci->i_ceph_lock);
3600                 ceph_take_cap_refs(ci, CEPH_CAP_PIN, false);
3601                 __ceph_touch_fmode(ci, mdsc, fmode);
3602                 spin_unlock(&ci->i_ceph_lock);
3603         }
3604         if (req->r_old_dentry_dir)
3605                 ceph_get_cap_refs(ceph_inode(req->r_old_dentry_dir),
3606                                   CEPH_CAP_PIN);
3607
3608         if (req->r_inode) {
3609                 err = ceph_wait_on_async_create(req->r_inode);
3610                 if (err) {
3611                         doutc(cl, "wait for async create returned: %d\n", err);
3612                         return err;
3613                 }
3614         }
3615
3616         if (!err && req->r_old_inode) {
3617                 err = ceph_wait_on_async_create(req->r_old_inode);
3618                 if (err) {
3619                         doutc(cl, "wait for async create returned: %d\n", err);
3620                         return err;
3621                 }
3622         }
3623
3624         doutc(cl, "submit_request on %p for inode %p\n", req, dir);
3625         mutex_lock(&mdsc->mutex);
3626         __register_request(mdsc, req, dir);
3627         __do_request(mdsc, req);
3628         err = req->r_err;
3629         mutex_unlock(&mdsc->mutex);
3630         return err;
3631 }
3632
3633 int ceph_mdsc_wait_request(struct ceph_mds_client *mdsc,
3634                            struct ceph_mds_request *req,
3635                            ceph_mds_request_wait_callback_t wait_func)
3636 {
3637         struct ceph_client *cl = mdsc->fsc->client;
3638         int err;
3639
3640         /* wait */
3641         doutc(cl, "do_request waiting\n");
3642         if (wait_func) {
3643                 err = wait_func(mdsc, req);
3644         } else {
3645                 long timeleft = wait_for_completion_killable_timeout(
3646                                         &req->r_completion,
3647                                         ceph_timeout_jiffies(req->r_timeout));
3648                 if (timeleft > 0)
3649                         err = 0;
3650                 else if (!timeleft)
3651                         err = -ETIMEDOUT;  /* timed out */
3652                 else
3653                         err = timeleft;  /* killed */
3654         }
3655         doutc(cl, "do_request waited, got %d\n", err);
3656         mutex_lock(&mdsc->mutex);
3657
3658         /* only abort if we didn't race with a real reply */
3659         if (test_bit(CEPH_MDS_R_GOT_RESULT, &req->r_req_flags)) {
3660                 err = le32_to_cpu(req->r_reply_info.head->result);
3661         } else if (err < 0) {
3662                 doutc(cl, "aborted request %lld with %d\n", req->r_tid, err);
3663
3664                 /*
3665                  * ensure we aren't running concurrently with
3666                  * ceph_fill_trace or ceph_readdir_prepopulate, which
3667                  * rely on locks (dir mutex) held by our caller.
3668                  */
3669                 mutex_lock(&req->r_fill_mutex);
3670                 req->r_err = err;
3671                 set_bit(CEPH_MDS_R_ABORTED, &req->r_req_flags);
3672                 mutex_unlock(&req->r_fill_mutex);
3673
3674                 if (req->r_parent &&
3675                     (req->r_op & CEPH_MDS_OP_WRITE))
3676                         ceph_invalidate_dir_request(req);
3677         } else {
3678                 err = req->r_err;
3679         }
3680
3681         mutex_unlock(&mdsc->mutex);
3682         return err;
3683 }
3684
3685 /*
3686  * Synchrously perform an mds request.  Take care of all of the
3687  * session setup, forwarding, retry details.
3688  */
3689 int ceph_mdsc_do_request(struct ceph_mds_client *mdsc,
3690                          struct inode *dir,
3691                          struct ceph_mds_request *req)
3692 {
3693         struct ceph_client *cl = mdsc->fsc->client;
3694         int err;
3695
3696         doutc(cl, "do_request on %p\n", req);
3697
3698         /* issue */
3699         err = ceph_mdsc_submit_request(mdsc, dir, req);
3700         if (!err)
3701                 err = ceph_mdsc_wait_request(mdsc, req, NULL);
3702         doutc(cl, "do_request %p done, result %d\n", req, err);
3703         return err;
3704 }
3705
3706 /*
3707  * Invalidate dir's completeness, dentry lease state on an aborted MDS
3708  * namespace request.
3709  */
3710 void ceph_invalidate_dir_request(struct ceph_mds_request *req)
3711 {
3712         struct inode *dir = req->r_parent;
3713         struct inode *old_dir = req->r_old_dentry_dir;
3714         struct ceph_client *cl = req->r_mdsc->fsc->client;
3715
3716         doutc(cl, "invalidate_dir_request %p %p (complete, lease(s))\n",
3717               dir, old_dir);
3718
3719         ceph_dir_clear_complete(dir);
3720         if (old_dir)
3721                 ceph_dir_clear_complete(old_dir);
3722         if (req->r_dentry)
3723                 ceph_invalidate_dentry_lease(req->r_dentry);
3724         if (req->r_old_dentry)
3725                 ceph_invalidate_dentry_lease(req->r_old_dentry);
3726 }
3727
3728 /*
3729  * Handle mds reply.
3730  *
3731  * We take the session mutex and parse and process the reply immediately.
3732  * This preserves the logical ordering of replies, capabilities, etc., sent
3733  * by the MDS as they are applied to our local cache.
3734  */
3735 static void handle_reply(struct ceph_mds_session *session, struct ceph_msg *msg)
3736 {
3737         struct ceph_mds_client *mdsc = session->s_mdsc;
3738         struct ceph_client *cl = mdsc->fsc->client;
3739         struct ceph_mds_request *req;
3740         struct ceph_mds_reply_head *head = msg->front.iov_base;
3741         struct ceph_mds_reply_info_parsed *rinfo;  /* parsed reply info */
3742         struct ceph_snap_realm *realm;
3743         u64 tid;
3744         int err, result;
3745         int mds = session->s_mds;
3746         bool close_sessions = false;
3747
3748         if (msg->front.iov_len < sizeof(*head)) {
3749                 pr_err_client(cl, "got corrupt (short) reply\n");
3750                 ceph_msg_dump(msg);
3751                 return;
3752         }
3753
3754         /* get request, session */
3755         tid = le64_to_cpu(msg->hdr.tid);
3756         mutex_lock(&mdsc->mutex);
3757         req = lookup_get_request(mdsc, tid);
3758         if (!req) {
3759                 doutc(cl, "on unknown tid %llu\n", tid);
3760                 mutex_unlock(&mdsc->mutex);
3761                 return;
3762         }
3763         doutc(cl, "handle_reply %p\n", req);
3764
3765         /* correct session? */
3766         if (req->r_session != session) {
3767                 pr_err_client(cl, "got %llu on session mds%d not mds%d\n",
3768                               tid, session->s_mds,
3769                               req->r_session ? req->r_session->s_mds : -1);
3770                 mutex_unlock(&mdsc->mutex);
3771                 goto out;
3772         }
3773
3774         /* dup? */
3775         if ((test_bit(CEPH_MDS_R_GOT_UNSAFE, &req->r_req_flags) && !head->safe) ||
3776             (test_bit(CEPH_MDS_R_GOT_SAFE, &req->r_req_flags) && head->safe)) {
3777                 pr_warn_client(cl, "got a dup %s reply on %llu from mds%d\n",
3778                                head->safe ? "safe" : "unsafe", tid, mds);
3779                 mutex_unlock(&mdsc->mutex);
3780                 goto out;
3781         }
3782         if (test_bit(CEPH_MDS_R_GOT_SAFE, &req->r_req_flags)) {
3783                 pr_warn_client(cl, "got unsafe after safe on %llu from mds%d\n",
3784                                tid, mds);
3785                 mutex_unlock(&mdsc->mutex);
3786                 goto out;
3787         }
3788
3789         result = le32_to_cpu(head->result);
3790
3791         if (head->safe) {
3792                 set_bit(CEPH_MDS_R_GOT_SAFE, &req->r_req_flags);
3793                 __unregister_request(mdsc, req);
3794
3795                 /* last request during umount? */
3796                 if (mdsc->stopping && !__get_oldest_req(mdsc))
3797                         complete_all(&mdsc->safe_umount_waiters);
3798
3799                 if (test_bit(CEPH_MDS_R_GOT_UNSAFE, &req->r_req_flags)) {
3800                         /*
3801                          * We already handled the unsafe response, now do the
3802                          * cleanup.  No need to examine the response; the MDS
3803                          * doesn't include any result info in the safe
3804                          * response.  And even if it did, there is nothing
3805                          * useful we could do with a revised return value.
3806                          */
3807                         doutc(cl, "got safe reply %llu, mds%d\n", tid, mds);
3808
3809                         mutex_unlock(&mdsc->mutex);
3810                         goto out;
3811                 }
3812         } else {
3813                 set_bit(CEPH_MDS_R_GOT_UNSAFE, &req->r_req_flags);
3814                 list_add_tail(&req->r_unsafe_item, &req->r_session->s_unsafe);
3815         }
3816
3817         doutc(cl, "tid %lld result %d\n", tid, result);
3818         if (test_bit(CEPHFS_FEATURE_REPLY_ENCODING, &session->s_features))
3819                 err = parse_reply_info(session, msg, req, (u64)-1);
3820         else
3821                 err = parse_reply_info(session, msg, req,
3822                                        session->s_con.peer_features);
3823         mutex_unlock(&mdsc->mutex);
3824
3825         /* Must find target inode outside of mutexes to avoid deadlocks */
3826         rinfo = &req->r_reply_info;
3827         if ((err >= 0) && rinfo->head->is_target) {
3828                 struct inode *in = xchg(&req->r_new_inode, NULL);
3829                 struct ceph_vino tvino = {
3830                         .ino  = le64_to_cpu(rinfo->targeti.in->ino),
3831                         .snap = le64_to_cpu(rinfo->targeti.in->snapid)
3832                 };
3833
3834                 /*
3835                  * If we ended up opening an existing inode, discard
3836                  * r_new_inode
3837                  */
3838                 if (req->r_op == CEPH_MDS_OP_CREATE &&
3839                     !req->r_reply_info.has_create_ino) {
3840                         /* This should never happen on an async create */
3841                         WARN_ON_ONCE(req->r_deleg_ino);
3842                         iput(in);
3843                         in = NULL;
3844                 }
3845
3846                 in = ceph_get_inode(mdsc->fsc->sb, tvino, in);
3847                 if (IS_ERR(in)) {
3848                         err = PTR_ERR(in);
3849                         mutex_lock(&session->s_mutex);
3850                         goto out_err;
3851                 }
3852                 req->r_target_inode = in;
3853         }
3854
3855         mutex_lock(&session->s_mutex);
3856         if (err < 0) {
3857                 pr_err_client(cl, "got corrupt reply mds%d(tid:%lld)\n",
3858                               mds, tid);
3859                 ceph_msg_dump(msg);
3860                 goto out_err;
3861         }
3862
3863         /* snap trace */
3864         realm = NULL;
3865         if (rinfo->snapblob_len) {
3866                 down_write(&mdsc->snap_rwsem);
3867                 err = ceph_update_snap_trace(mdsc, rinfo->snapblob,
3868                                 rinfo->snapblob + rinfo->snapblob_len,
3869                                 le32_to_cpu(head->op) == CEPH_MDS_OP_RMSNAP,
3870                                 &realm);
3871                 if (err) {
3872                         up_write(&mdsc->snap_rwsem);
3873                         close_sessions = true;
3874                         if (err == -EIO)
3875                                 ceph_msg_dump(msg);
3876                         goto out_err;
3877                 }
3878                 downgrade_write(&mdsc->snap_rwsem);
3879         } else {
3880                 down_read(&mdsc->snap_rwsem);
3881         }
3882
3883         /* insert trace into our cache */
3884         mutex_lock(&req->r_fill_mutex);
3885         current->journal_info = req;
3886         err = ceph_fill_trace(mdsc->fsc->sb, req);
3887         if (err == 0) {
3888                 if (result == 0 && (req->r_op == CEPH_MDS_OP_READDIR ||
3889                                     req->r_op == CEPH_MDS_OP_LSSNAP))
3890                         err = ceph_readdir_prepopulate(req, req->r_session);
3891         }
3892         current->journal_info = NULL;
3893         mutex_unlock(&req->r_fill_mutex);
3894
3895         up_read(&mdsc->snap_rwsem);
3896         if (realm)
3897                 ceph_put_snap_realm(mdsc, realm);
3898
3899         if (err == 0) {
3900                 if (req->r_target_inode &&
3901                     test_bit(CEPH_MDS_R_GOT_UNSAFE, &req->r_req_flags)) {
3902                         struct ceph_inode_info *ci =
3903                                 ceph_inode(req->r_target_inode);
3904                         spin_lock(&ci->i_unsafe_lock);
3905                         list_add_tail(&req->r_unsafe_target_item,
3906                                       &ci->i_unsafe_iops);
3907                         spin_unlock(&ci->i_unsafe_lock);
3908                 }
3909
3910                 ceph_unreserve_caps(mdsc, &req->r_caps_reservation);
3911         }
3912 out_err:
3913         mutex_lock(&mdsc->mutex);
3914         if (!test_bit(CEPH_MDS_R_ABORTED, &req->r_req_flags)) {
3915                 if (err) {
3916                         req->r_err = err;
3917                 } else {
3918                         req->r_reply =  ceph_msg_get(msg);
3919                         set_bit(CEPH_MDS_R_GOT_RESULT, &req->r_req_flags);
3920                 }
3921         } else {
3922                 doutc(cl, "reply arrived after request %lld was aborted\n", tid);
3923         }
3924         mutex_unlock(&mdsc->mutex);
3925
3926         mutex_unlock(&session->s_mutex);
3927
3928         /* kick calling process */
3929         complete_request(mdsc, req);
3930
3931         ceph_update_metadata_metrics(&mdsc->metric, req->r_start_latency,
3932                                      req->r_end_latency, err);
3933 out:
3934         ceph_mdsc_put_request(req);
3935
3936         /* Defer closing the sessions after s_mutex lock being released */
3937         if (close_sessions)
3938                 ceph_mdsc_close_sessions(mdsc);
3939         return;
3940 }
3941
3942
3943
3944 /*
3945  * handle mds notification that our request has been forwarded.
3946  */
3947 static void handle_forward(struct ceph_mds_client *mdsc,
3948                            struct ceph_mds_session *session,
3949                            struct ceph_msg *msg)
3950 {
3951         struct ceph_client *cl = mdsc->fsc->client;
3952         struct ceph_mds_request *req;
3953         u64 tid = le64_to_cpu(msg->hdr.tid);
3954         u32 next_mds;
3955         u32 fwd_seq;
3956         int err = -EINVAL;
3957         void *p = msg->front.iov_base;
3958         void *end = p + msg->front.iov_len;
3959         bool aborted = false;
3960
3961         ceph_decode_need(&p, end, 2*sizeof(u32), bad);
3962         next_mds = ceph_decode_32(&p);
3963         fwd_seq = ceph_decode_32(&p);
3964
3965         mutex_lock(&mdsc->mutex);
3966         req = lookup_get_request(mdsc, tid);
3967         if (!req) {
3968                 mutex_unlock(&mdsc->mutex);
3969                 doutc(cl, "forward tid %llu to mds%d - req dne\n", tid, next_mds);
3970                 return;  /* dup reply? */
3971         }
3972
3973         if (test_bit(CEPH_MDS_R_ABORTED, &req->r_req_flags)) {
3974                 doutc(cl, "forward tid %llu aborted, unregistering\n", tid);
3975                 __unregister_request(mdsc, req);
3976         } else if (fwd_seq <= req->r_num_fwd || (uint32_t)fwd_seq >= U32_MAX) {
3977                 /*
3978                  * Avoid inifinite retrying after overflow.
3979                  *
3980                  * The MDS will increase the fwd count and in client side
3981                  * if the num_fwd is less than the one saved in request
3982                  * that means the MDS is an old version and overflowed of
3983                  * 8 bits.
3984                  */
3985                 mutex_lock(&req->r_fill_mutex);
3986                 req->r_err = -EMULTIHOP;
3987                 set_bit(CEPH_MDS_R_ABORTED, &req->r_req_flags);
3988                 mutex_unlock(&req->r_fill_mutex);
3989                 aborted = true;
3990                 pr_warn_ratelimited_client(cl, "forward tid %llu seq overflow\n",
3991                                            tid);
3992         } else {
3993                 /* resend. forward race not possible; mds would drop */
3994                 doutc(cl, "forward tid %llu to mds%d (we resend)\n", tid, next_mds);
3995                 BUG_ON(req->r_err);
3996                 BUG_ON(test_bit(CEPH_MDS_R_GOT_RESULT, &req->r_req_flags));
3997                 req->r_attempts = 0;
3998                 req->r_num_fwd = fwd_seq;
3999                 req->r_resend_mds = next_mds;
4000                 put_request_session(req);
4001                 __do_request(mdsc, req);
4002         }
4003         mutex_unlock(&mdsc->mutex);
4004
4005         /* kick calling process */
4006         if (aborted)
4007                 complete_request(mdsc, req);
4008         ceph_mdsc_put_request(req);
4009         return;
4010
4011 bad:
4012         pr_err_client(cl, "decode error err=%d\n", err);
4013         ceph_msg_dump(msg);
4014 }
4015
4016 static int __decode_session_metadata(void **p, void *end,
4017                                      bool *blocklisted)
4018 {
4019         /* map<string,string> */
4020         u32 n;
4021         bool err_str;
4022         ceph_decode_32_safe(p, end, n, bad);
4023         while (n-- > 0) {
4024                 u32 len;
4025                 ceph_decode_32_safe(p, end, len, bad);
4026                 ceph_decode_need(p, end, len, bad);
4027                 err_str = !strncmp(*p, "error_string", len);
4028                 *p += len;
4029                 ceph_decode_32_safe(p, end, len, bad);
4030                 ceph_decode_need(p, end, len, bad);
4031                 /*
4032                  * Match "blocklisted (blacklisted)" from newer MDSes,
4033                  * or "blacklisted" from older MDSes.
4034                  */
4035                 if (err_str && strnstr(*p, "blacklisted", len))
4036                         *blocklisted = true;
4037                 *p += len;
4038         }
4039         return 0;
4040 bad:
4041         return -1;
4042 }
4043
4044 /*
4045  * handle a mds session control message
4046  */
4047 static void handle_session(struct ceph_mds_session *session,
4048                            struct ceph_msg *msg)
4049 {
4050         struct ceph_mds_client *mdsc = session->s_mdsc;
4051         struct ceph_client *cl = mdsc->fsc->client;
4052         int mds = session->s_mds;
4053         int msg_version = le16_to_cpu(msg->hdr.version);
4054         void *p = msg->front.iov_base;
4055         void *end = p + msg->front.iov_len;
4056         struct ceph_mds_session_head *h;
4057         u32 op;
4058         u64 seq, features = 0;
4059         int wake = 0;
4060         bool blocklisted = false;
4061
4062         /* decode */
4063         ceph_decode_need(&p, end, sizeof(*h), bad);
4064         h = p;
4065         p += sizeof(*h);
4066
4067         op = le32_to_cpu(h->op);
4068         seq = le64_to_cpu(h->seq);
4069
4070         if (msg_version >= 3) {
4071                 u32 len;
4072                 /* version >= 2 and < 5, decode metadata, skip otherwise
4073                  * as it's handled via flags.
4074                  */
4075                 if (msg_version >= 5)
4076                         ceph_decode_skip_map(&p, end, string, string, bad);
4077                 else if (__decode_session_metadata(&p, end, &blocklisted) < 0)
4078                         goto bad;
4079
4080                 /* version >= 3, feature bits */
4081                 ceph_decode_32_safe(&p, end, len, bad);
4082                 if (len) {
4083                         ceph_decode_64_safe(&p, end, features, bad);
4084                         p += len - sizeof(features);
4085                 }
4086         }
4087
4088         if (msg_version >= 5) {
4089                 u32 flags, len;
4090
4091                 /* version >= 4 */
4092                 ceph_decode_skip_16(&p, end, bad); /* struct_v, struct_cv */
4093                 ceph_decode_32_safe(&p, end, len, bad); /* len */
4094                 ceph_decode_skip_n(&p, end, len, bad); /* metric_spec */
4095
4096                 /* version >= 5, flags   */
4097                 ceph_decode_32_safe(&p, end, flags, bad);
4098                 if (flags & CEPH_SESSION_BLOCKLISTED) {
4099                         pr_warn_client(cl, "mds%d session blocklisted\n",
4100                                        session->s_mds);
4101                         blocklisted = true;
4102                 }
4103         }
4104
4105         mutex_lock(&mdsc->mutex);
4106         if (op == CEPH_SESSION_CLOSE) {
4107                 ceph_get_mds_session(session);
4108                 __unregister_session(mdsc, session);
4109         }
4110         /* FIXME: this ttl calculation is generous */
4111         session->s_ttl = jiffies + HZ*mdsc->mdsmap->m_session_autoclose;
4112         mutex_unlock(&mdsc->mutex);
4113
4114         mutex_lock(&session->s_mutex);
4115
4116         doutc(cl, "mds%d %s %p state %s seq %llu\n", mds,
4117               ceph_session_op_name(op), session,
4118               ceph_session_state_name(session->s_state), seq);
4119
4120         if (session->s_state == CEPH_MDS_SESSION_HUNG) {
4121                 session->s_state = CEPH_MDS_SESSION_OPEN;
4122                 pr_info_client(cl, "mds%d came back\n", session->s_mds);
4123         }
4124
4125         switch (op) {
4126         case CEPH_SESSION_OPEN:
4127                 if (session->s_state == CEPH_MDS_SESSION_RECONNECTING)
4128                         pr_info_client(cl, "mds%d reconnect success\n",
4129                                        session->s_mds);
4130
4131                 if (session->s_state == CEPH_MDS_SESSION_OPEN) {
4132                         pr_notice_client(cl, "mds%d is already opened\n",
4133                                          session->s_mds);
4134                 } else {
4135                         session->s_state = CEPH_MDS_SESSION_OPEN;
4136                         session->s_features = features;
4137                         renewed_caps(mdsc, session, 0);
4138                         if (test_bit(CEPHFS_FEATURE_METRIC_COLLECT,
4139                                      &session->s_features))
4140                                 metric_schedule_delayed(&mdsc->metric);
4141                 }
4142
4143                 /*
4144                  * The connection maybe broken and the session in client
4145                  * side has been reinitialized, need to update the seq
4146                  * anyway.
4147                  */
4148                 if (!session->s_seq && seq)
4149                         session->s_seq = seq;
4150
4151                 wake = 1;
4152                 if (mdsc->stopping)
4153                         __close_session(mdsc, session);
4154                 break;
4155
4156         case CEPH_SESSION_RENEWCAPS:
4157                 if (session->s_renew_seq == seq)
4158                         renewed_caps(mdsc, session, 1);
4159                 break;
4160
4161         case CEPH_SESSION_CLOSE:
4162                 if (session->s_state == CEPH_MDS_SESSION_RECONNECTING)
4163                         pr_info_client(cl, "mds%d reconnect denied\n",
4164                                        session->s_mds);
4165                 session->s_state = CEPH_MDS_SESSION_CLOSED;
4166                 cleanup_session_requests(mdsc, session);
4167                 remove_session_caps(session);
4168                 wake = 2; /* for good measure */
4169                 wake_up_all(&mdsc->session_close_wq);
4170                 break;
4171
4172         case CEPH_SESSION_STALE:
4173                 pr_info_client(cl, "mds%d caps went stale, renewing\n",
4174                                session->s_mds);
4175                 atomic_inc(&session->s_cap_gen);
4176                 session->s_cap_ttl = jiffies - 1;
4177                 send_renew_caps(mdsc, session);
4178                 break;
4179
4180         case CEPH_SESSION_RECALL_STATE:
4181                 ceph_trim_caps(mdsc, session, le32_to_cpu(h->max_caps));
4182                 break;
4183
4184         case CEPH_SESSION_FLUSHMSG:
4185                 /* flush cap releases */
4186                 spin_lock(&session->s_cap_lock);
4187                 if (session->s_num_cap_releases)
4188                         ceph_flush_cap_releases(mdsc, session);
4189                 spin_unlock(&session->s_cap_lock);
4190
4191                 send_flushmsg_ack(mdsc, session, seq);
4192                 break;
4193
4194         case CEPH_SESSION_FORCE_RO:
4195                 doutc(cl, "force_session_readonly %p\n", session);
4196                 spin_lock(&session->s_cap_lock);
4197                 session->s_readonly = true;
4198                 spin_unlock(&session->s_cap_lock);
4199                 wake_up_session_caps(session, FORCE_RO);
4200                 break;
4201
4202         case CEPH_SESSION_REJECT:
4203                 WARN_ON(session->s_state != CEPH_MDS_SESSION_OPENING);
4204                 pr_info_client(cl, "mds%d rejected session\n",
4205                                session->s_mds);
4206                 session->s_state = CEPH_MDS_SESSION_REJECTED;
4207                 cleanup_session_requests(mdsc, session);
4208                 remove_session_caps(session);
4209                 if (blocklisted)
4210                         mdsc->fsc->blocklisted = true;
4211                 wake = 2; /* for good measure */
4212                 break;
4213
4214         default:
4215                 pr_err_client(cl, "bad op %d mds%d\n", op, mds);
4216                 WARN_ON(1);
4217         }
4218
4219         mutex_unlock(&session->s_mutex);
4220         if (wake) {
4221                 mutex_lock(&mdsc->mutex);
4222                 __wake_requests(mdsc, &session->s_waiting);
4223                 if (wake == 2)
4224                         kick_requests(mdsc, mds);
4225                 mutex_unlock(&mdsc->mutex);
4226         }
4227         if (op == CEPH_SESSION_CLOSE)
4228                 ceph_put_mds_session(session);
4229         return;
4230
4231 bad:
4232         pr_err_client(cl, "corrupt message mds%d len %d\n", mds,
4233                       (int)msg->front.iov_len);
4234         ceph_msg_dump(msg);
4235         return;
4236 }
4237
4238 void ceph_mdsc_release_dir_caps(struct ceph_mds_request *req)
4239 {
4240         struct ceph_client *cl = req->r_mdsc->fsc->client;
4241         int dcaps;
4242
4243         dcaps = xchg(&req->r_dir_caps, 0);
4244         if (dcaps) {
4245                 doutc(cl, "releasing r_dir_caps=%s\n", ceph_cap_string(dcaps));
4246                 ceph_put_cap_refs(ceph_inode(req->r_parent), dcaps);
4247         }
4248 }
4249
4250 void ceph_mdsc_release_dir_caps_no_check(struct ceph_mds_request *req)
4251 {
4252         struct ceph_client *cl = req->r_mdsc->fsc->client;
4253         int dcaps;
4254
4255         dcaps = xchg(&req->r_dir_caps, 0);
4256         if (dcaps) {
4257                 doutc(cl, "releasing r_dir_caps=%s\n", ceph_cap_string(dcaps));
4258                 ceph_put_cap_refs_no_check_caps(ceph_inode(req->r_parent),
4259                                                 dcaps);
4260         }
4261 }
4262
4263 /*
4264  * called under session->mutex.
4265  */
4266 static void replay_unsafe_requests(struct ceph_mds_client *mdsc,
4267                                    struct ceph_mds_session *session)
4268 {
4269         struct ceph_mds_request *req, *nreq;
4270         struct rb_node *p;
4271
4272         doutc(mdsc->fsc->client, "mds%d\n", session->s_mds);
4273
4274         mutex_lock(&mdsc->mutex);
4275         list_for_each_entry_safe(req, nreq, &session->s_unsafe, r_unsafe_item)
4276                 __send_request(session, req, true);
4277
4278         /*
4279          * also re-send old requests when MDS enters reconnect stage. So that MDS
4280          * can process completed request in clientreplay stage.
4281          */
4282         p = rb_first(&mdsc->request_tree);
4283         while (p) {
4284                 req = rb_entry(p, struct ceph_mds_request, r_node);
4285                 p = rb_next(p);
4286                 if (test_bit(CEPH_MDS_R_GOT_UNSAFE, &req->r_req_flags))
4287                         continue;
4288                 if (req->r_attempts == 0)
4289                         continue; /* only old requests */
4290                 if (!req->r_session)
4291                         continue;
4292                 if (req->r_session->s_mds != session->s_mds)
4293                         continue;
4294
4295                 ceph_mdsc_release_dir_caps_no_check(req);
4296
4297                 __send_request(session, req, true);
4298         }
4299         mutex_unlock(&mdsc->mutex);
4300 }
4301
4302 static int send_reconnect_partial(struct ceph_reconnect_state *recon_state)
4303 {
4304         struct ceph_msg *reply;
4305         struct ceph_pagelist *_pagelist;
4306         struct page *page;
4307         __le32 *addr;
4308         int err = -ENOMEM;
4309
4310         if (!recon_state->allow_multi)
4311                 return -ENOSPC;
4312
4313         /* can't handle message that contains both caps and realm */
4314         BUG_ON(!recon_state->nr_caps == !recon_state->nr_realms);
4315
4316         /* pre-allocate new pagelist */
4317         _pagelist = ceph_pagelist_alloc(GFP_NOFS);
4318         if (!_pagelist)
4319                 return -ENOMEM;
4320
4321         reply = ceph_msg_new2(CEPH_MSG_CLIENT_RECONNECT, 0, 1, GFP_NOFS, false);
4322         if (!reply)
4323                 goto fail_msg;
4324
4325         /* placeholder for nr_caps */
4326         err = ceph_pagelist_encode_32(_pagelist, 0);
4327         if (err < 0)
4328                 goto fail;
4329
4330         if (recon_state->nr_caps) {
4331                 /* currently encoding caps */
4332                 err = ceph_pagelist_encode_32(recon_state->pagelist, 0);
4333                 if (err)
4334                         goto fail;
4335         } else {
4336                 /* placeholder for nr_realms (currently encoding relams) */
4337                 err = ceph_pagelist_encode_32(_pagelist, 0);
4338                 if (err < 0)
4339                         goto fail;
4340         }
4341
4342         err = ceph_pagelist_encode_8(recon_state->pagelist, 1);
4343         if (err)
4344                 goto fail;
4345
4346         page = list_first_entry(&recon_state->pagelist->head, struct page, lru);
4347         addr = kmap_atomic(page);
4348         if (recon_state->nr_caps) {
4349                 /* currently encoding caps */
4350                 *addr = cpu_to_le32(recon_state->nr_caps);
4351         } else {
4352                 /* currently encoding relams */
4353                 *(addr + 1) = cpu_to_le32(recon_state->nr_realms);
4354         }
4355         kunmap_atomic(addr);
4356
4357         reply->hdr.version = cpu_to_le16(5);
4358         reply->hdr.compat_version = cpu_to_le16(4);
4359
4360         reply->hdr.data_len = cpu_to_le32(recon_state->pagelist->length);
4361         ceph_msg_data_add_pagelist(reply, recon_state->pagelist);
4362
4363         ceph_con_send(&recon_state->session->s_con, reply);
4364         ceph_pagelist_release(recon_state->pagelist);
4365
4366         recon_state->pagelist = _pagelist;
4367         recon_state->nr_caps = 0;
4368         recon_state->nr_realms = 0;
4369         recon_state->msg_version = 5;
4370         return 0;
4371 fail:
4372         ceph_msg_put(reply);
4373 fail_msg:
4374         ceph_pagelist_release(_pagelist);
4375         return err;
4376 }
4377
4378 static struct dentry* d_find_primary(struct inode *inode)
4379 {
4380         struct dentry *alias, *dn = NULL;
4381
4382         if (hlist_empty(&inode->i_dentry))
4383                 return NULL;
4384
4385         spin_lock(&inode->i_lock);
4386         if (hlist_empty(&inode->i_dentry))
4387                 goto out_unlock;
4388
4389         if (S_ISDIR(inode->i_mode)) {
4390                 alias = hlist_entry(inode->i_dentry.first, struct dentry, d_u.d_alias);
4391                 if (!IS_ROOT(alias))
4392                         dn = dget(alias);
4393                 goto out_unlock;
4394         }
4395
4396         hlist_for_each_entry(alias, &inode->i_dentry, d_u.d_alias) {
4397                 spin_lock(&alias->d_lock);
4398                 if (!d_unhashed(alias) &&
4399                     (ceph_dentry(alias)->flags & CEPH_DENTRY_PRIMARY_LINK)) {
4400                         dn = dget_dlock(alias);
4401                 }
4402                 spin_unlock(&alias->d_lock);
4403                 if (dn)
4404                         break;
4405         }
4406 out_unlock:
4407         spin_unlock(&inode->i_lock);
4408         return dn;
4409 }
4410
4411 /*
4412  * Encode information about a cap for a reconnect with the MDS.
4413  */
4414 static int reconnect_caps_cb(struct inode *inode, int mds, void *arg)
4415 {
4416         struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb);
4417         struct ceph_client *cl = ceph_inode_to_client(inode);
4418         union {
4419                 struct ceph_mds_cap_reconnect v2;
4420                 struct ceph_mds_cap_reconnect_v1 v1;
4421         } rec;
4422         struct ceph_inode_info *ci = ceph_inode(inode);
4423         struct ceph_reconnect_state *recon_state = arg;
4424         struct ceph_pagelist *pagelist = recon_state->pagelist;
4425         struct dentry *dentry;
4426         struct ceph_cap *cap;
4427         char *path;
4428         int pathlen = 0, err;
4429         u64 pathbase;
4430         u64 snap_follows;
4431
4432         dentry = d_find_primary(inode);
4433         if (dentry) {
4434                 /* set pathbase to parent dir when msg_version >= 2 */
4435                 path = ceph_mdsc_build_path(mdsc, dentry, &pathlen, &pathbase,
4436                                             recon_state->msg_version >= 2);
4437                 dput(dentry);
4438                 if (IS_ERR(path)) {
4439                         err = PTR_ERR(path);
4440                         goto out_err;
4441                 }
4442         } else {
4443                 path = NULL;
4444                 pathbase = 0;
4445         }
4446
4447         spin_lock(&ci->i_ceph_lock);
4448         cap = __get_cap_for_mds(ci, mds);
4449         if (!cap) {
4450                 spin_unlock(&ci->i_ceph_lock);
4451                 err = 0;
4452                 goto out_err;
4453         }
4454         doutc(cl, " adding %p ino %llx.%llx cap %p %lld %s\n", inode,
4455               ceph_vinop(inode), cap, cap->cap_id,
4456               ceph_cap_string(cap->issued));
4457
4458         cap->seq = 0;        /* reset cap seq */
4459         cap->issue_seq = 0;  /* and issue_seq */
4460         cap->mseq = 0;       /* and migrate_seq */
4461         cap->cap_gen = atomic_read(&cap->session->s_cap_gen);
4462
4463         /* These are lost when the session goes away */
4464         if (S_ISDIR(inode->i_mode)) {
4465                 if (cap->issued & CEPH_CAP_DIR_CREATE) {
4466                         ceph_put_string(rcu_dereference_raw(ci->i_cached_layout.pool_ns));
4467                         memset(&ci->i_cached_layout, 0, sizeof(ci->i_cached_layout));
4468                 }
4469                 cap->issued &= ~CEPH_CAP_ANY_DIR_OPS;
4470         }
4471
4472         if (recon_state->msg_version >= 2) {
4473                 rec.v2.cap_id = cpu_to_le64(cap->cap_id);
4474                 rec.v2.wanted = cpu_to_le32(__ceph_caps_wanted(ci));
4475                 rec.v2.issued = cpu_to_le32(cap->issued);
4476                 rec.v2.snaprealm = cpu_to_le64(ci->i_snap_realm->ino);
4477                 rec.v2.pathbase = cpu_to_le64(pathbase);
4478                 rec.v2.flock_len = (__force __le32)
4479                         ((ci->i_ceph_flags & CEPH_I_ERROR_FILELOCK) ? 0 : 1);
4480         } else {
4481                 rec.v1.cap_id = cpu_to_le64(cap->cap_id);
4482                 rec.v1.wanted = cpu_to_le32(__ceph_caps_wanted(ci));
4483                 rec.v1.issued = cpu_to_le32(cap->issued);
4484                 rec.v1.size = cpu_to_le64(i_size_read(inode));
4485                 ceph_encode_timespec64(&rec.v1.mtime, &inode->i_mtime);
4486                 ceph_encode_timespec64(&rec.v1.atime, &inode->i_atime);
4487                 rec.v1.snaprealm = cpu_to_le64(ci->i_snap_realm->ino);
4488                 rec.v1.pathbase = cpu_to_le64(pathbase);
4489         }
4490
4491         if (list_empty(&ci->i_cap_snaps)) {
4492                 snap_follows = ci->i_head_snapc ? ci->i_head_snapc->seq : 0;
4493         } else {
4494                 struct ceph_cap_snap *capsnap =
4495                         list_first_entry(&ci->i_cap_snaps,
4496                                          struct ceph_cap_snap, ci_item);
4497                 snap_follows = capsnap->follows;
4498         }
4499         spin_unlock(&ci->i_ceph_lock);
4500
4501         if (recon_state->msg_version >= 2) {
4502                 int num_fcntl_locks, num_flock_locks;
4503                 struct ceph_filelock *flocks = NULL;
4504                 size_t struct_len, total_len = sizeof(u64);
4505                 u8 struct_v = 0;
4506
4507 encode_again:
4508                 if (rec.v2.flock_len) {
4509                         ceph_count_locks(inode, &num_fcntl_locks, &num_flock_locks);
4510                 } else {
4511                         num_fcntl_locks = 0;
4512                         num_flock_locks = 0;
4513                 }
4514                 if (num_fcntl_locks + num_flock_locks > 0) {
4515                         flocks = kmalloc_array(num_fcntl_locks + num_flock_locks,
4516                                                sizeof(struct ceph_filelock),
4517                                                GFP_NOFS);
4518                         if (!flocks) {
4519                                 err = -ENOMEM;
4520                                 goto out_err;
4521                         }
4522                         err = ceph_encode_locks_to_buffer(inode, flocks,
4523                                                           num_fcntl_locks,
4524                                                           num_flock_locks);
4525                         if (err) {
4526                                 kfree(flocks);
4527                                 flocks = NULL;
4528                                 if (err == -ENOSPC)
4529                                         goto encode_again;
4530                                 goto out_err;
4531                         }
4532                 } else {
4533                         kfree(flocks);
4534                         flocks = NULL;
4535                 }
4536
4537                 if (recon_state->msg_version >= 3) {
4538                         /* version, compat_version and struct_len */
4539                         total_len += 2 * sizeof(u8) + sizeof(u32);
4540                         struct_v = 2;
4541                 }
4542                 /*
4543                  * number of encoded locks is stable, so copy to pagelist
4544                  */
4545                 struct_len = 2 * sizeof(u32) +
4546                             (num_fcntl_locks + num_flock_locks) *
4547                             sizeof(struct ceph_filelock);
4548                 rec.v2.flock_len = cpu_to_le32(struct_len);
4549
4550                 struct_len += sizeof(u32) + pathlen + sizeof(rec.v2);
4551
4552                 if (struct_v >= 2)
4553                         struct_len += sizeof(u64); /* snap_follows */
4554
4555                 total_len += struct_len;
4556
4557                 if (pagelist->length + total_len > RECONNECT_MAX_SIZE) {
4558                         err = send_reconnect_partial(recon_state);
4559                         if (err)
4560                                 goto out_freeflocks;
4561                         pagelist = recon_state->pagelist;
4562                 }
4563
4564                 err = ceph_pagelist_reserve(pagelist, total_len);
4565                 if (err)
4566                         goto out_freeflocks;
4567
4568                 ceph_pagelist_encode_64(pagelist, ceph_ino(inode));
4569                 if (recon_state->msg_version >= 3) {
4570                         ceph_pagelist_encode_8(pagelist, struct_v);
4571                         ceph_pagelist_encode_8(pagelist, 1);
4572                         ceph_pagelist_encode_32(pagelist, struct_len);
4573                 }
4574                 ceph_pagelist_encode_string(pagelist, path, pathlen);
4575                 ceph_pagelist_append(pagelist, &rec, sizeof(rec.v2));
4576                 ceph_locks_to_pagelist(flocks, pagelist,
4577                                        num_fcntl_locks, num_flock_locks);
4578                 if (struct_v >= 2)
4579                         ceph_pagelist_encode_64(pagelist, snap_follows);
4580 out_freeflocks:
4581                 kfree(flocks);
4582         } else {
4583                 err = ceph_pagelist_reserve(pagelist,
4584                                             sizeof(u64) + sizeof(u32) +
4585                                             pathlen + sizeof(rec.v1));
4586                 if (err)
4587                         goto out_err;
4588
4589                 ceph_pagelist_encode_64(pagelist, ceph_ino(inode));
4590                 ceph_pagelist_encode_string(pagelist, path, pathlen);
4591                 ceph_pagelist_append(pagelist, &rec, sizeof(rec.v1));
4592         }
4593
4594 out_err:
4595         ceph_mdsc_free_path(path, pathlen);
4596         if (!err)
4597                 recon_state->nr_caps++;
4598         return err;
4599 }
4600
4601 static int encode_snap_realms(struct ceph_mds_client *mdsc,
4602                               struct ceph_reconnect_state *recon_state)
4603 {
4604         struct rb_node *p;
4605         struct ceph_pagelist *pagelist = recon_state->pagelist;
4606         struct ceph_client *cl = mdsc->fsc->client;
4607         int err = 0;
4608
4609         if (recon_state->msg_version >= 4) {
4610                 err = ceph_pagelist_encode_32(pagelist, mdsc->num_snap_realms);
4611                 if (err < 0)
4612                         goto fail;
4613         }
4614
4615         /*
4616          * snaprealms.  we provide mds with the ino, seq (version), and
4617          * parent for all of our realms.  If the mds has any newer info,
4618          * it will tell us.
4619          */
4620         for (p = rb_first(&mdsc->snap_realms); p; p = rb_next(p)) {
4621                 struct ceph_snap_realm *realm =
4622                        rb_entry(p, struct ceph_snap_realm, node);
4623                 struct ceph_mds_snaprealm_reconnect sr_rec;
4624
4625                 if (recon_state->msg_version >= 4) {
4626                         size_t need = sizeof(u8) * 2 + sizeof(u32) +
4627                                       sizeof(sr_rec);
4628
4629                         if (pagelist->length + need > RECONNECT_MAX_SIZE) {
4630                                 err = send_reconnect_partial(recon_state);
4631                                 if (err)
4632                                         goto fail;
4633                                 pagelist = recon_state->pagelist;
4634                         }
4635
4636                         err = ceph_pagelist_reserve(pagelist, need);
4637                         if (err)
4638                                 goto fail;
4639
4640                         ceph_pagelist_encode_8(pagelist, 1);
4641                         ceph_pagelist_encode_8(pagelist, 1);
4642                         ceph_pagelist_encode_32(pagelist, sizeof(sr_rec));
4643                 }
4644
4645                 doutc(cl, " adding snap realm %llx seq %lld parent %llx\n",
4646                       realm->ino, realm->seq, realm->parent_ino);
4647                 sr_rec.ino = cpu_to_le64(realm->ino);
4648                 sr_rec.seq = cpu_to_le64(realm->seq);
4649                 sr_rec.parent = cpu_to_le64(realm->parent_ino);
4650
4651                 err = ceph_pagelist_append(pagelist, &sr_rec, sizeof(sr_rec));
4652                 if (err)
4653                         goto fail;
4654
4655                 recon_state->nr_realms++;
4656         }
4657 fail:
4658         return err;
4659 }
4660
4661
4662 /*
4663  * If an MDS fails and recovers, clients need to reconnect in order to
4664  * reestablish shared state.  This includes all caps issued through
4665  * this session _and_ the snap_realm hierarchy.  Because it's not
4666  * clear which snap realms the mds cares about, we send everything we
4667  * know about.. that ensures we'll then get any new info the
4668  * recovering MDS might have.
4669  *
4670  * This is a relatively heavyweight operation, but it's rare.
4671  */
4672 static void send_mds_reconnect(struct ceph_mds_client *mdsc,
4673                                struct ceph_mds_session *session)
4674 {
4675         struct ceph_client *cl = mdsc->fsc->client;
4676         struct ceph_msg *reply;
4677         int mds = session->s_mds;
4678         int err = -ENOMEM;
4679         struct ceph_reconnect_state recon_state = {
4680                 .session = session,
4681         };
4682         LIST_HEAD(dispose);
4683
4684         pr_info_client(cl, "mds%d reconnect start\n", mds);
4685
4686         recon_state.pagelist = ceph_pagelist_alloc(GFP_NOFS);
4687         if (!recon_state.pagelist)
4688                 goto fail_nopagelist;
4689
4690         reply = ceph_msg_new2(CEPH_MSG_CLIENT_RECONNECT, 0, 1, GFP_NOFS, false);
4691         if (!reply)
4692                 goto fail_nomsg;
4693
4694         xa_destroy(&session->s_delegated_inos);
4695
4696         mutex_lock(&session->s_mutex);
4697         session->s_state = CEPH_MDS_SESSION_RECONNECTING;
4698         session->s_seq = 0;
4699
4700         doutc(cl, "session %p state %s\n", session,
4701               ceph_session_state_name(session->s_state));
4702
4703         atomic_inc(&session->s_cap_gen);
4704
4705         spin_lock(&session->s_cap_lock);
4706         /* don't know if session is readonly */
4707         session->s_readonly = 0;
4708         /*
4709          * notify __ceph_remove_cap() that we are composing cap reconnect.
4710          * If a cap get released before being added to the cap reconnect,
4711          * __ceph_remove_cap() should skip queuing cap release.
4712          */
4713         session->s_cap_reconnect = 1;
4714         /* drop old cap expires; we're about to reestablish that state */
4715         detach_cap_releases(session, &dispose);
4716         spin_unlock(&session->s_cap_lock);
4717         dispose_cap_releases(mdsc, &dispose);
4718
4719         /* trim unused caps to reduce MDS's cache rejoin time */
4720         if (mdsc->fsc->sb->s_root)
4721                 shrink_dcache_parent(mdsc->fsc->sb->s_root);
4722
4723         ceph_con_close(&session->s_con);
4724         ceph_con_open(&session->s_con,
4725                       CEPH_ENTITY_TYPE_MDS, mds,
4726                       ceph_mdsmap_get_addr(mdsc->mdsmap, mds));
4727
4728         /* replay unsafe requests */
4729         replay_unsafe_requests(mdsc, session);
4730
4731         ceph_early_kick_flushing_caps(mdsc, session);
4732
4733         down_read(&mdsc->snap_rwsem);
4734
4735         /* placeholder for nr_caps */
4736         err = ceph_pagelist_encode_32(recon_state.pagelist, 0);
4737         if (err)
4738                 goto fail;
4739
4740         if (test_bit(CEPHFS_FEATURE_MULTI_RECONNECT, &session->s_features)) {
4741                 recon_state.msg_version = 3;
4742                 recon_state.allow_multi = true;
4743         } else if (session->s_con.peer_features & CEPH_FEATURE_MDSENC) {
4744                 recon_state.msg_version = 3;
4745         } else {
4746                 recon_state.msg_version = 2;
4747         }
4748         /* trsaverse this session's caps */
4749         err = ceph_iterate_session_caps(session, reconnect_caps_cb, &recon_state);
4750
4751         spin_lock(&session->s_cap_lock);
4752         session->s_cap_reconnect = 0;
4753         spin_unlock(&session->s_cap_lock);
4754
4755         if (err < 0)
4756                 goto fail;
4757
4758         /* check if all realms can be encoded into current message */
4759         if (mdsc->num_snap_realms) {
4760                 size_t total_len =
4761                         recon_state.pagelist->length +
4762                         mdsc->num_snap_realms *
4763                         sizeof(struct ceph_mds_snaprealm_reconnect);
4764                 if (recon_state.msg_version >= 4) {
4765                         /* number of realms */
4766                         total_len += sizeof(u32);
4767                         /* version, compat_version and struct_len */
4768                         total_len += mdsc->num_snap_realms *
4769                                      (2 * sizeof(u8) + sizeof(u32));
4770                 }
4771                 if (total_len > RECONNECT_MAX_SIZE) {
4772                         if (!recon_state.allow_multi) {
4773                                 err = -ENOSPC;
4774                                 goto fail;
4775                         }
4776                         if (recon_state.nr_caps) {
4777                                 err = send_reconnect_partial(&recon_state);
4778                                 if (err)
4779                                         goto fail;
4780                         }
4781                         recon_state.msg_version = 5;
4782                 }
4783         }
4784
4785         err = encode_snap_realms(mdsc, &recon_state);
4786         if (err < 0)
4787                 goto fail;
4788
4789         if (recon_state.msg_version >= 5) {
4790                 err = ceph_pagelist_encode_8(recon_state.pagelist, 0);
4791                 if (err < 0)
4792                         goto fail;
4793         }
4794
4795         if (recon_state.nr_caps || recon_state.nr_realms) {
4796                 struct page *page =
4797                         list_first_entry(&recon_state.pagelist->head,
4798                                         struct page, lru);
4799                 __le32 *addr = kmap_atomic(page);
4800                 if (recon_state.nr_caps) {
4801                         WARN_ON(recon_state.nr_realms != mdsc->num_snap_realms);
4802                         *addr = cpu_to_le32(recon_state.nr_caps);
4803                 } else if (recon_state.msg_version >= 4) {
4804                         *(addr + 1) = cpu_to_le32(recon_state.nr_realms);
4805                 }
4806                 kunmap_atomic(addr);
4807         }
4808
4809         reply->hdr.version = cpu_to_le16(recon_state.msg_version);
4810         if (recon_state.msg_version >= 4)
4811                 reply->hdr.compat_version = cpu_to_le16(4);
4812
4813         reply->hdr.data_len = cpu_to_le32(recon_state.pagelist->length);
4814         ceph_msg_data_add_pagelist(reply, recon_state.pagelist);
4815
4816         ceph_con_send(&session->s_con, reply);
4817
4818         mutex_unlock(&session->s_mutex);
4819
4820         mutex_lock(&mdsc->mutex);
4821         __wake_requests(mdsc, &session->s_waiting);
4822         mutex_unlock(&mdsc->mutex);
4823
4824         up_read(&mdsc->snap_rwsem);
4825         ceph_pagelist_release(recon_state.pagelist);
4826         return;
4827
4828 fail:
4829         ceph_msg_put(reply);
4830         up_read(&mdsc->snap_rwsem);
4831         mutex_unlock(&session->s_mutex);
4832 fail_nomsg:
4833         ceph_pagelist_release(recon_state.pagelist);
4834 fail_nopagelist:
4835         pr_err_client(cl, "error %d preparing reconnect for mds%d\n",
4836                       err, mds);
4837         return;
4838 }
4839
4840
4841 /*
4842  * compare old and new mdsmaps, kicking requests
4843  * and closing out old connections as necessary
4844  *
4845  * called under mdsc->mutex.
4846  */
4847 static void check_new_map(struct ceph_mds_client *mdsc,
4848                           struct ceph_mdsmap *newmap,
4849                           struct ceph_mdsmap *oldmap)
4850 {
4851         int i, j, err;
4852         int oldstate, newstate;
4853         struct ceph_mds_session *s;
4854         unsigned long targets[DIV_ROUND_UP(CEPH_MAX_MDS, sizeof(unsigned long))] = {0};
4855         struct ceph_client *cl = mdsc->fsc->client;
4856
4857         doutc(cl, "new %u old %u\n", newmap->m_epoch, oldmap->m_epoch);
4858
4859         if (newmap->m_info) {
4860                 for (i = 0; i < newmap->possible_max_rank; i++) {
4861                         for (j = 0; j < newmap->m_info[i].num_export_targets; j++)
4862                                 set_bit(newmap->m_info[i].export_targets[j], targets);
4863                 }
4864         }
4865
4866         for (i = 0; i < oldmap->possible_max_rank && i < mdsc->max_sessions; i++) {
4867                 if (!mdsc->sessions[i])
4868                         continue;
4869                 s = mdsc->sessions[i];
4870                 oldstate = ceph_mdsmap_get_state(oldmap, i);
4871                 newstate = ceph_mdsmap_get_state(newmap, i);
4872
4873                 doutc(cl, "mds%d state %s%s -> %s%s (session %s)\n",
4874                       i, ceph_mds_state_name(oldstate),
4875                       ceph_mdsmap_is_laggy(oldmap, i) ? " (laggy)" : "",
4876                       ceph_mds_state_name(newstate),
4877                       ceph_mdsmap_is_laggy(newmap, i) ? " (laggy)" : "",
4878                       ceph_session_state_name(s->s_state));
4879
4880                 if (i >= newmap->possible_max_rank) {
4881                         /* force close session for stopped mds */
4882                         ceph_get_mds_session(s);
4883                         __unregister_session(mdsc, s);
4884                         __wake_requests(mdsc, &s->s_waiting);
4885                         mutex_unlock(&mdsc->mutex);
4886
4887                         mutex_lock(&s->s_mutex);
4888                         cleanup_session_requests(mdsc, s);
4889                         remove_session_caps(s);
4890                         mutex_unlock(&s->s_mutex);
4891
4892                         ceph_put_mds_session(s);
4893
4894                         mutex_lock(&mdsc->mutex);
4895                         kick_requests(mdsc, i);
4896                         continue;
4897                 }
4898
4899                 if (memcmp(ceph_mdsmap_get_addr(oldmap, i),
4900                            ceph_mdsmap_get_addr(newmap, i),
4901                            sizeof(struct ceph_entity_addr))) {
4902                         /* just close it */
4903                         mutex_unlock(&mdsc->mutex);
4904                         mutex_lock(&s->s_mutex);
4905                         mutex_lock(&mdsc->mutex);
4906                         ceph_con_close(&s->s_con);
4907                         mutex_unlock(&s->s_mutex);
4908                         s->s_state = CEPH_MDS_SESSION_RESTARTING;
4909                 } else if (oldstate == newstate) {
4910                         continue;  /* nothing new with this mds */
4911                 }
4912
4913                 /*
4914                  * send reconnect?
4915                  */
4916                 if (s->s_state == CEPH_MDS_SESSION_RESTARTING &&
4917                     newstate >= CEPH_MDS_STATE_RECONNECT) {
4918                         mutex_unlock(&mdsc->mutex);
4919                         clear_bit(i, targets);
4920                         send_mds_reconnect(mdsc, s);
4921                         mutex_lock(&mdsc->mutex);
4922                 }
4923
4924                 /*
4925                  * kick request on any mds that has gone active.
4926                  */
4927                 if (oldstate < CEPH_MDS_STATE_ACTIVE &&
4928                     newstate >= CEPH_MDS_STATE_ACTIVE) {
4929                         if (oldstate != CEPH_MDS_STATE_CREATING &&
4930                             oldstate != CEPH_MDS_STATE_STARTING)
4931                                 pr_info_client(cl, "mds%d recovery completed\n",
4932                                                s->s_mds);
4933                         kick_requests(mdsc, i);
4934                         mutex_unlock(&mdsc->mutex);
4935                         mutex_lock(&s->s_mutex);
4936                         mutex_lock(&mdsc->mutex);
4937                         ceph_kick_flushing_caps(mdsc, s);
4938                         mutex_unlock(&s->s_mutex);
4939                         wake_up_session_caps(s, RECONNECT);
4940                 }
4941         }
4942
4943         /*
4944          * Only open and reconnect sessions that don't exist yet.
4945          */
4946         for (i = 0; i < newmap->possible_max_rank; i++) {
4947                 /*
4948                  * In case the import MDS is crashed just after
4949                  * the EImportStart journal is flushed, so when
4950                  * a standby MDS takes over it and is replaying
4951                  * the EImportStart journal the new MDS daemon
4952                  * will wait the client to reconnect it, but the
4953                  * client may never register/open the session yet.
4954                  *
4955                  * Will try to reconnect that MDS daemon if the
4956                  * rank number is in the export targets array and
4957                  * is the up:reconnect state.
4958                  */
4959                 newstate = ceph_mdsmap_get_state(newmap, i);
4960                 if (!test_bit(i, targets) || newstate != CEPH_MDS_STATE_RECONNECT)
4961                         continue;
4962
4963                 /*
4964                  * The session maybe registered and opened by some
4965                  * requests which were choosing random MDSes during
4966                  * the mdsc->mutex's unlock/lock gap below in rare
4967                  * case. But the related MDS daemon will just queue
4968                  * that requests and be still waiting for the client's
4969                  * reconnection request in up:reconnect state.
4970                  */
4971                 s = __ceph_lookup_mds_session(mdsc, i);
4972                 if (likely(!s)) {
4973                         s = __open_export_target_session(mdsc, i);
4974                         if (IS_ERR(s)) {
4975                                 err = PTR_ERR(s);
4976                                 pr_err_client(cl,
4977                                               "failed to open export target session, err %d\n",
4978                                               err);
4979                                 continue;
4980                         }
4981                 }
4982                 doutc(cl, "send reconnect to export target mds.%d\n", i);
4983                 mutex_unlock(&mdsc->mutex);
4984                 send_mds_reconnect(mdsc, s);
4985                 ceph_put_mds_session(s);
4986                 mutex_lock(&mdsc->mutex);
4987         }
4988
4989         for (i = 0; i < newmap->possible_max_rank && i < mdsc->max_sessions; i++) {
4990                 s = mdsc->sessions[i];
4991                 if (!s)
4992                         continue;
4993                 if (!ceph_mdsmap_is_laggy(newmap, i))
4994                         continue;
4995                 if (s->s_state == CEPH_MDS_SESSION_OPEN ||
4996                     s->s_state == CEPH_MDS_SESSION_HUNG ||
4997                     s->s_state == CEPH_MDS_SESSION_CLOSING) {
4998                         doutc(cl, " connecting to export targets of laggy mds%d\n", i);
4999                         __open_export_target_sessions(mdsc, s);
5000                 }
5001         }
5002 }
5003
5004
5005
5006 /*
5007  * leases
5008  */
5009
5010 /*
5011  * caller must hold session s_mutex, dentry->d_lock
5012  */
5013 void __ceph_mdsc_drop_dentry_lease(struct dentry *dentry)
5014 {
5015         struct ceph_dentry_info *di = ceph_dentry(dentry);
5016
5017         ceph_put_mds_session(di->lease_session);
5018         di->lease_session = NULL;
5019 }
5020
5021 static void handle_lease(struct ceph_mds_client *mdsc,
5022                          struct ceph_mds_session *session,
5023                          struct ceph_msg *msg)
5024 {
5025         struct ceph_client *cl = mdsc->fsc->client;
5026         struct super_block *sb = mdsc->fsc->sb;
5027         struct inode *inode;
5028         struct dentry *parent, *dentry;
5029         struct ceph_dentry_info *di;
5030         int mds = session->s_mds;
5031         struct ceph_mds_lease *h = msg->front.iov_base;
5032         u32 seq;
5033         struct ceph_vino vino;
5034         struct qstr dname;
5035         int release = 0;
5036
5037         doutc(cl, "from mds%d\n", mds);
5038
5039         if (!ceph_inc_mds_stopping_blocker(mdsc, session))
5040                 return;
5041
5042         /* decode */
5043         if (msg->front.iov_len < sizeof(*h) + sizeof(u32))
5044                 goto bad;
5045         vino.ino = le64_to_cpu(h->ino);
5046         vino.snap = CEPH_NOSNAP;
5047         seq = le32_to_cpu(h->seq);
5048         dname.len = get_unaligned_le32(h + 1);
5049         if (msg->front.iov_len < sizeof(*h) + sizeof(u32) + dname.len)
5050                 goto bad;
5051         dname.name = (void *)(h + 1) + sizeof(u32);
5052
5053         /* lookup inode */
5054         inode = ceph_find_inode(sb, vino);
5055         doutc(cl, "%s, ino %llx %p %.*s\n", ceph_lease_op_name(h->action),
5056               vino.ino, inode, dname.len, dname.name);
5057
5058         mutex_lock(&session->s_mutex);
5059         if (!inode) {
5060                 doutc(cl, "no inode %llx\n", vino.ino);
5061                 goto release;
5062         }
5063
5064         /* dentry */
5065         parent = d_find_alias(inode);
5066         if (!parent) {
5067                 doutc(cl, "no parent dentry on inode %p\n", inode);
5068                 WARN_ON(1);
5069                 goto release;  /* hrm... */
5070         }
5071         dname.hash = full_name_hash(parent, dname.name, dname.len);
5072         dentry = d_lookup(parent, &dname);
5073         dput(parent);
5074         if (!dentry)
5075                 goto release;
5076
5077         spin_lock(&dentry->d_lock);
5078         di = ceph_dentry(dentry);
5079         switch (h->action) {
5080         case CEPH_MDS_LEASE_REVOKE:
5081                 if (di->lease_session == session) {
5082                         if (ceph_seq_cmp(di->lease_seq, seq) > 0)
5083                                 h->seq = cpu_to_le32(di->lease_seq);
5084                         __ceph_mdsc_drop_dentry_lease(dentry);
5085                 }
5086                 release = 1;
5087                 break;
5088
5089         case CEPH_MDS_LEASE_RENEW:
5090                 if (di->lease_session == session &&
5091                     di->lease_gen == atomic_read(&session->s_cap_gen) &&
5092                     di->lease_renew_from &&
5093                     di->lease_renew_after == 0) {
5094                         unsigned long duration =
5095                                 msecs_to_jiffies(le32_to_cpu(h->duration_ms));
5096
5097                         di->lease_seq = seq;
5098                         di->time = di->lease_renew_from + duration;
5099                         di->lease_renew_after = di->lease_renew_from +
5100                                 (duration >> 1);
5101                         di->lease_renew_from = 0;
5102                 }
5103                 break;
5104         }
5105         spin_unlock(&dentry->d_lock);
5106         dput(dentry);
5107
5108         if (!release)
5109                 goto out;
5110
5111 release:
5112         /* let's just reuse the same message */
5113         h->action = CEPH_MDS_LEASE_REVOKE_ACK;
5114         ceph_msg_get(msg);
5115         ceph_con_send(&session->s_con, msg);
5116
5117 out:
5118         mutex_unlock(&session->s_mutex);
5119         iput(inode);
5120
5121         ceph_dec_mds_stopping_blocker(mdsc);
5122         return;
5123
5124 bad:
5125         ceph_dec_mds_stopping_blocker(mdsc);
5126
5127         pr_err_client(cl, "corrupt lease message\n");
5128         ceph_msg_dump(msg);
5129 }
5130
5131 void ceph_mdsc_lease_send_msg(struct ceph_mds_session *session,
5132                               struct dentry *dentry, char action,
5133                               u32 seq)
5134 {
5135         struct ceph_client *cl = session->s_mdsc->fsc->client;
5136         struct ceph_msg *msg;
5137         struct ceph_mds_lease *lease;
5138         struct inode *dir;
5139         int len = sizeof(*lease) + sizeof(u32) + NAME_MAX;
5140
5141         doutc(cl, "identry %p %s to mds%d\n", dentry, ceph_lease_op_name(action),
5142               session->s_mds);
5143
5144         msg = ceph_msg_new(CEPH_MSG_CLIENT_LEASE, len, GFP_NOFS, false);
5145         if (!msg)
5146                 return;
5147         lease = msg->front.iov_base;
5148         lease->action = action;
5149         lease->seq = cpu_to_le32(seq);
5150
5151         spin_lock(&dentry->d_lock);
5152         dir = d_inode(dentry->d_parent);
5153         lease->ino = cpu_to_le64(ceph_ino(dir));
5154         lease->first = lease->last = cpu_to_le64(ceph_snap(dir));
5155
5156         put_unaligned_le32(dentry->d_name.len, lease + 1);
5157         memcpy((void *)(lease + 1) + 4,
5158                dentry->d_name.name, dentry->d_name.len);
5159         spin_unlock(&dentry->d_lock);
5160
5161         ceph_con_send(&session->s_con, msg);
5162 }
5163
5164 /*
5165  * lock unlock the session, to wait ongoing session activities
5166  */
5167 static void lock_unlock_session(struct ceph_mds_session *s)
5168 {
5169         mutex_lock(&s->s_mutex);
5170         mutex_unlock(&s->s_mutex);
5171 }
5172
5173 static void maybe_recover_session(struct ceph_mds_client *mdsc)
5174 {
5175         struct ceph_client *cl = mdsc->fsc->client;
5176         struct ceph_fs_client *fsc = mdsc->fsc;
5177
5178         if (!ceph_test_mount_opt(fsc, CLEANRECOVER))
5179                 return;
5180
5181         if (READ_ONCE(fsc->mount_state) != CEPH_MOUNT_MOUNTED)
5182                 return;
5183
5184         if (!READ_ONCE(fsc->blocklisted))
5185                 return;
5186
5187         pr_info_client(cl, "auto reconnect after blocklisted\n");
5188         ceph_force_reconnect(fsc->sb);
5189 }
5190
5191 bool check_session_state(struct ceph_mds_session *s)
5192 {
5193         struct ceph_client *cl = s->s_mdsc->fsc->client;
5194
5195         switch (s->s_state) {
5196         case CEPH_MDS_SESSION_OPEN:
5197                 if (s->s_ttl && time_after(jiffies, s->s_ttl)) {
5198                         s->s_state = CEPH_MDS_SESSION_HUNG;
5199                         pr_info_client(cl, "mds%d hung\n", s->s_mds);
5200                 }
5201                 break;
5202         case CEPH_MDS_SESSION_CLOSING:
5203         case CEPH_MDS_SESSION_NEW:
5204         case CEPH_MDS_SESSION_RESTARTING:
5205         case CEPH_MDS_SESSION_CLOSED:
5206         case CEPH_MDS_SESSION_REJECTED:
5207                 return false;
5208         }
5209
5210         return true;
5211 }
5212
5213 /*
5214  * If the sequence is incremented while we're waiting on a REQUEST_CLOSE reply,
5215  * then we need to retransmit that request.
5216  */
5217 void inc_session_sequence(struct ceph_mds_session *s)
5218 {
5219         struct ceph_client *cl = s->s_mdsc->fsc->client;
5220
5221         lockdep_assert_held(&s->s_mutex);
5222
5223         s->s_seq++;
5224
5225         if (s->s_state == CEPH_MDS_SESSION_CLOSING) {
5226                 int ret;
5227
5228                 doutc(cl, "resending session close request for mds%d\n", s->s_mds);
5229                 ret = request_close_session(s);
5230                 if (ret < 0)
5231                         pr_err_client(cl, "unable to close session to mds%d: %d\n",
5232                                       s->s_mds, ret);
5233         }
5234 }
5235
5236 /*
5237  * delayed work -- periodically trim expired leases, renew caps with mds.  If
5238  * the @delay parameter is set to 0 or if it's more than 5 secs, the default
5239  * workqueue delay value of 5 secs will be used.
5240  */
5241 static void schedule_delayed(struct ceph_mds_client *mdsc, unsigned long delay)
5242 {
5243         unsigned long max_delay = HZ * 5;
5244
5245         /* 5 secs default delay */
5246         if (!delay || (delay > max_delay))
5247                 delay = max_delay;
5248         schedule_delayed_work(&mdsc->delayed_work,
5249                               round_jiffies_relative(delay));
5250 }
5251
5252 static void delayed_work(struct work_struct *work)
5253 {
5254         struct ceph_mds_client *mdsc =
5255                 container_of(work, struct ceph_mds_client, delayed_work.work);
5256         unsigned long delay;
5257         int renew_interval;
5258         int renew_caps;
5259         int i;
5260
5261         doutc(mdsc->fsc->client, "mdsc delayed_work\n");
5262
5263         if (mdsc->stopping >= CEPH_MDSC_STOPPING_FLUSHED)
5264                 return;
5265
5266         mutex_lock(&mdsc->mutex);
5267         renew_interval = mdsc->mdsmap->m_session_timeout >> 2;
5268         renew_caps = time_after_eq(jiffies, HZ*renew_interval +
5269                                    mdsc->last_renew_caps);
5270         if (renew_caps)
5271                 mdsc->last_renew_caps = jiffies;
5272
5273         for (i = 0; i < mdsc->max_sessions; i++) {
5274                 struct ceph_mds_session *s = __ceph_lookup_mds_session(mdsc, i);
5275                 if (!s)
5276                         continue;
5277
5278                 if (!check_session_state(s)) {
5279                         ceph_put_mds_session(s);
5280                         continue;
5281                 }
5282                 mutex_unlock(&mdsc->mutex);
5283
5284                 mutex_lock(&s->s_mutex);
5285                 if (renew_caps)
5286                         send_renew_caps(mdsc, s);
5287                 else
5288                         ceph_con_keepalive(&s->s_con);
5289                 if (s->s_state == CEPH_MDS_SESSION_OPEN ||
5290                     s->s_state == CEPH_MDS_SESSION_HUNG)
5291                         ceph_send_cap_releases(mdsc, s);
5292                 mutex_unlock(&s->s_mutex);
5293                 ceph_put_mds_session(s);
5294
5295                 mutex_lock(&mdsc->mutex);
5296         }
5297         mutex_unlock(&mdsc->mutex);
5298
5299         delay = ceph_check_delayed_caps(mdsc);
5300
5301         ceph_queue_cap_reclaim_work(mdsc);
5302
5303         ceph_trim_snapid_map(mdsc);
5304
5305         maybe_recover_session(mdsc);
5306
5307         schedule_delayed(mdsc, delay);
5308 }
5309
5310 int ceph_mdsc_init(struct ceph_fs_client *fsc)
5311
5312 {
5313         struct ceph_mds_client *mdsc;
5314         int err;
5315
5316         mdsc = kzalloc(sizeof(struct ceph_mds_client), GFP_NOFS);
5317         if (!mdsc)
5318                 return -ENOMEM;
5319         mdsc->fsc = fsc;
5320         mutex_init(&mdsc->mutex);
5321         mdsc->mdsmap = kzalloc(sizeof(*mdsc->mdsmap), GFP_NOFS);
5322         if (!mdsc->mdsmap) {
5323                 err = -ENOMEM;
5324                 goto err_mdsc;
5325         }
5326
5327         init_completion(&mdsc->safe_umount_waiters);
5328         spin_lock_init(&mdsc->stopping_lock);
5329         atomic_set(&mdsc->stopping_blockers, 0);
5330         init_completion(&mdsc->stopping_waiter);
5331         init_waitqueue_head(&mdsc->session_close_wq);
5332         INIT_LIST_HEAD(&mdsc->waiting_for_map);
5333         mdsc->quotarealms_inodes = RB_ROOT;
5334         mutex_init(&mdsc->quotarealms_inodes_mutex);
5335         init_rwsem(&mdsc->snap_rwsem);
5336         mdsc->snap_realms = RB_ROOT;
5337         INIT_LIST_HEAD(&mdsc->snap_empty);
5338         spin_lock_init(&mdsc->snap_empty_lock);
5339         mdsc->request_tree = RB_ROOT;
5340         INIT_DELAYED_WORK(&mdsc->delayed_work, delayed_work);
5341         mdsc->last_renew_caps = jiffies;
5342         INIT_LIST_HEAD(&mdsc->cap_delay_list);
5343         INIT_LIST_HEAD(&mdsc->cap_wait_list);
5344         spin_lock_init(&mdsc->cap_delay_lock);
5345         INIT_LIST_HEAD(&mdsc->snap_flush_list);
5346         spin_lock_init(&mdsc->snap_flush_lock);
5347         mdsc->last_cap_flush_tid = 1;
5348         INIT_LIST_HEAD(&mdsc->cap_flush_list);
5349         INIT_LIST_HEAD(&mdsc->cap_dirty_migrating);
5350         spin_lock_init(&mdsc->cap_dirty_lock);
5351         init_waitqueue_head(&mdsc->cap_flushing_wq);
5352         INIT_WORK(&mdsc->cap_reclaim_work, ceph_cap_reclaim_work);
5353         err = ceph_metric_init(&mdsc->metric);
5354         if (err)
5355                 goto err_mdsmap;
5356
5357         spin_lock_init(&mdsc->dentry_list_lock);
5358         INIT_LIST_HEAD(&mdsc->dentry_leases);
5359         INIT_LIST_HEAD(&mdsc->dentry_dir_leases);
5360
5361         ceph_caps_init(mdsc);
5362         ceph_adjust_caps_max_min(mdsc, fsc->mount_options);
5363
5364         spin_lock_init(&mdsc->snapid_map_lock);
5365         mdsc->snapid_map_tree = RB_ROOT;
5366         INIT_LIST_HEAD(&mdsc->snapid_map_lru);
5367
5368         init_rwsem(&mdsc->pool_perm_rwsem);
5369         mdsc->pool_perm_tree = RB_ROOT;
5370
5371         strscpy(mdsc->nodename, utsname()->nodename,
5372                 sizeof(mdsc->nodename));
5373
5374         fsc->mdsc = mdsc;
5375         return 0;
5376
5377 err_mdsmap:
5378         kfree(mdsc->mdsmap);
5379 err_mdsc:
5380         kfree(mdsc);
5381         return err;
5382 }
5383
5384 /*
5385  * Wait for safe replies on open mds requests.  If we time out, drop
5386  * all requests from the tree to avoid dangling dentry refs.
5387  */
5388 static void wait_requests(struct ceph_mds_client *mdsc)
5389 {
5390         struct ceph_client *cl = mdsc->fsc->client;
5391         struct ceph_options *opts = mdsc->fsc->client->options;
5392         struct ceph_mds_request *req;
5393
5394         mutex_lock(&mdsc->mutex);
5395         if (__get_oldest_req(mdsc)) {
5396                 mutex_unlock(&mdsc->mutex);
5397
5398                 doutc(cl, "waiting for requests\n");
5399                 wait_for_completion_timeout(&mdsc->safe_umount_waiters,
5400                                     ceph_timeout_jiffies(opts->mount_timeout));
5401
5402                 /* tear down remaining requests */
5403                 mutex_lock(&mdsc->mutex);
5404                 while ((req = __get_oldest_req(mdsc))) {
5405                         doutc(cl, "timed out on tid %llu\n", req->r_tid);
5406                         list_del_init(&req->r_wait);
5407                         __unregister_request(mdsc, req);
5408                 }
5409         }
5410         mutex_unlock(&mdsc->mutex);
5411         doutc(cl, "done\n");
5412 }
5413
5414 void send_flush_mdlog(struct ceph_mds_session *s)
5415 {
5416         struct ceph_client *cl = s->s_mdsc->fsc->client;
5417         struct ceph_msg *msg;
5418
5419         /*
5420          * Pre-luminous MDS crashes when it sees an unknown session request
5421          */
5422         if (!CEPH_HAVE_FEATURE(s->s_con.peer_features, SERVER_LUMINOUS))
5423                 return;
5424
5425         mutex_lock(&s->s_mutex);
5426         doutc(cl, "request mdlog flush to mds%d (%s)s seq %lld\n",
5427               s->s_mds, ceph_session_state_name(s->s_state), s->s_seq);
5428         msg = ceph_create_session_msg(CEPH_SESSION_REQUEST_FLUSH_MDLOG,
5429                                       s->s_seq);
5430         if (!msg) {
5431                 pr_err_client(cl, "failed to request mdlog flush to mds%d (%s) seq %lld\n",
5432                               s->s_mds, ceph_session_state_name(s->s_state), s->s_seq);
5433         } else {
5434                 ceph_con_send(&s->s_con, msg);
5435         }
5436         mutex_unlock(&s->s_mutex);
5437 }
5438
5439 /*
5440  * called before mount is ro, and before dentries are torn down.
5441  * (hmm, does this still race with new lookups?)
5442  */
5443 void ceph_mdsc_pre_umount(struct ceph_mds_client *mdsc)
5444 {
5445         doutc(mdsc->fsc->client, "begin\n");
5446         mdsc->stopping = CEPH_MDSC_STOPPING_BEGIN;
5447
5448         ceph_mdsc_iterate_sessions(mdsc, send_flush_mdlog, true);
5449         ceph_mdsc_iterate_sessions(mdsc, lock_unlock_session, false);
5450         ceph_flush_dirty_caps(mdsc);
5451         wait_requests(mdsc);
5452
5453         /*
5454          * wait for reply handlers to drop their request refs and
5455          * their inode/dcache refs
5456          */
5457         ceph_msgr_flush();
5458
5459         ceph_cleanup_quotarealms_inodes(mdsc);
5460         doutc(mdsc->fsc->client, "done\n");
5461 }
5462
5463 /*
5464  * flush the mdlog and wait for all write mds requests to flush.
5465  */
5466 static void flush_mdlog_and_wait_mdsc_unsafe_requests(struct ceph_mds_client *mdsc,
5467                                                  u64 want_tid)
5468 {
5469         struct ceph_client *cl = mdsc->fsc->client;
5470         struct ceph_mds_request *req = NULL, *nextreq;
5471         struct ceph_mds_session *last_session = NULL;
5472         struct rb_node *n;
5473
5474         mutex_lock(&mdsc->mutex);
5475         doutc(cl, "want %lld\n", want_tid);
5476 restart:
5477         req = __get_oldest_req(mdsc);
5478         while (req && req->r_tid <= want_tid) {
5479                 /* find next request */
5480                 n = rb_next(&req->r_node);
5481                 if (n)
5482                         nextreq = rb_entry(n, struct ceph_mds_request, r_node);
5483                 else
5484                         nextreq = NULL;
5485                 if (req->r_op != CEPH_MDS_OP_SETFILELOCK &&
5486                     (req->r_op & CEPH_MDS_OP_WRITE)) {
5487                         struct ceph_mds_session *s = req->r_session;
5488
5489                         if (!s) {
5490                                 req = nextreq;
5491                                 continue;
5492                         }
5493
5494                         /* write op */
5495                         ceph_mdsc_get_request(req);
5496                         if (nextreq)
5497                                 ceph_mdsc_get_request(nextreq);
5498                         s = ceph_get_mds_session(s);
5499                         mutex_unlock(&mdsc->mutex);
5500
5501                         /* send flush mdlog request to MDS */
5502                         if (last_session != s) {
5503                                 send_flush_mdlog(s);
5504                                 ceph_put_mds_session(last_session);
5505                                 last_session = s;
5506                         } else {
5507                                 ceph_put_mds_session(s);
5508                         }
5509                         doutc(cl, "wait on %llu (want %llu)\n",
5510                               req->r_tid, want_tid);
5511                         wait_for_completion(&req->r_safe_completion);
5512
5513                         mutex_lock(&mdsc->mutex);
5514                         ceph_mdsc_put_request(req);
5515                         if (!nextreq)
5516                                 break;  /* next dne before, so we're done! */
5517                         if (RB_EMPTY_NODE(&nextreq->r_node)) {
5518                                 /* next request was removed from tree */
5519                                 ceph_mdsc_put_request(nextreq);
5520                                 goto restart;
5521                         }
5522                         ceph_mdsc_put_request(nextreq);  /* won't go away */
5523                 }
5524                 req = nextreq;
5525         }
5526         mutex_unlock(&mdsc->mutex);
5527         ceph_put_mds_session(last_session);
5528         doutc(cl, "done\n");
5529 }
5530
5531 void ceph_mdsc_sync(struct ceph_mds_client *mdsc)
5532 {
5533         struct ceph_client *cl = mdsc->fsc->client;
5534         u64 want_tid, want_flush;
5535
5536         if (READ_ONCE(mdsc->fsc->mount_state) >= CEPH_MOUNT_SHUTDOWN)
5537                 return;
5538
5539         doutc(cl, "sync\n");
5540         mutex_lock(&mdsc->mutex);
5541         want_tid = mdsc->last_tid;
5542         mutex_unlock(&mdsc->mutex);
5543
5544         ceph_flush_dirty_caps(mdsc);
5545         spin_lock(&mdsc->cap_dirty_lock);
5546         want_flush = mdsc->last_cap_flush_tid;
5547         if (!list_empty(&mdsc->cap_flush_list)) {
5548                 struct ceph_cap_flush *cf =
5549                         list_last_entry(&mdsc->cap_flush_list,
5550                                         struct ceph_cap_flush, g_list);
5551                 cf->wake = true;
5552         }
5553         spin_unlock(&mdsc->cap_dirty_lock);
5554
5555         doutc(cl, "sync want tid %lld flush_seq %lld\n", want_tid, want_flush);
5556
5557         flush_mdlog_and_wait_mdsc_unsafe_requests(mdsc, want_tid);
5558         wait_caps_flush(mdsc, want_flush);
5559 }
5560
5561 /*
5562  * true if all sessions are closed, or we force unmount
5563  */
5564 static bool done_closing_sessions(struct ceph_mds_client *mdsc, int skipped)
5565 {
5566         if (READ_ONCE(mdsc->fsc->mount_state) == CEPH_MOUNT_SHUTDOWN)
5567                 return true;
5568         return atomic_read(&mdsc->num_sessions) <= skipped;
5569 }
5570
5571 /*
5572  * called after sb is ro or when metadata corrupted.
5573  */
5574 void ceph_mdsc_close_sessions(struct ceph_mds_client *mdsc)
5575 {
5576         struct ceph_options *opts = mdsc->fsc->client->options;
5577         struct ceph_client *cl = mdsc->fsc->client;
5578         struct ceph_mds_session *session;
5579         int i;
5580         int skipped = 0;
5581
5582         doutc(cl, "begin\n");
5583
5584         /* close sessions */
5585         mutex_lock(&mdsc->mutex);
5586         for (i = 0; i < mdsc->max_sessions; i++) {
5587                 session = __ceph_lookup_mds_session(mdsc, i);
5588                 if (!session)
5589                         continue;
5590                 mutex_unlock(&mdsc->mutex);
5591                 mutex_lock(&session->s_mutex);
5592                 if (__close_session(mdsc, session) <= 0)
5593                         skipped++;
5594                 mutex_unlock(&session->s_mutex);
5595                 ceph_put_mds_session(session);
5596                 mutex_lock(&mdsc->mutex);
5597         }
5598         mutex_unlock(&mdsc->mutex);
5599
5600         doutc(cl, "waiting for sessions to close\n");
5601         wait_event_timeout(mdsc->session_close_wq,
5602                            done_closing_sessions(mdsc, skipped),
5603                            ceph_timeout_jiffies(opts->mount_timeout));
5604
5605         /* tear down remaining sessions */
5606         mutex_lock(&mdsc->mutex);
5607         for (i = 0; i < mdsc->max_sessions; i++) {
5608                 if (mdsc->sessions[i]) {
5609                         session = ceph_get_mds_session(mdsc->sessions[i]);
5610                         __unregister_session(mdsc, session);
5611                         mutex_unlock(&mdsc->mutex);
5612                         mutex_lock(&session->s_mutex);
5613                         remove_session_caps(session);
5614                         mutex_unlock(&session->s_mutex);
5615                         ceph_put_mds_session(session);
5616                         mutex_lock(&mdsc->mutex);
5617                 }
5618         }
5619         WARN_ON(!list_empty(&mdsc->cap_delay_list));
5620         mutex_unlock(&mdsc->mutex);
5621
5622         ceph_cleanup_snapid_map(mdsc);
5623         ceph_cleanup_global_and_empty_realms(mdsc);
5624
5625         cancel_work_sync(&mdsc->cap_reclaim_work);
5626         cancel_delayed_work_sync(&mdsc->delayed_work); /* cancel timer */
5627
5628         doutc(cl, "done\n");
5629 }
5630
5631 void ceph_mdsc_force_umount(struct ceph_mds_client *mdsc)
5632 {
5633         struct ceph_mds_session *session;
5634         int mds;
5635
5636         doutc(mdsc->fsc->client, "force umount\n");
5637
5638         mutex_lock(&mdsc->mutex);
5639         for (mds = 0; mds < mdsc->max_sessions; mds++) {
5640                 session = __ceph_lookup_mds_session(mdsc, mds);
5641                 if (!session)
5642                         continue;
5643
5644                 if (session->s_state == CEPH_MDS_SESSION_REJECTED)
5645                         __unregister_session(mdsc, session);
5646                 __wake_requests(mdsc, &session->s_waiting);
5647                 mutex_unlock(&mdsc->mutex);
5648
5649                 mutex_lock(&session->s_mutex);
5650                 __close_session(mdsc, session);
5651                 if (session->s_state == CEPH_MDS_SESSION_CLOSING) {
5652                         cleanup_session_requests(mdsc, session);
5653                         remove_session_caps(session);
5654                 }
5655                 mutex_unlock(&session->s_mutex);
5656                 ceph_put_mds_session(session);
5657
5658                 mutex_lock(&mdsc->mutex);
5659                 kick_requests(mdsc, mds);
5660         }
5661         __wake_requests(mdsc, &mdsc->waiting_for_map);
5662         mutex_unlock(&mdsc->mutex);
5663 }
5664
5665 static void ceph_mdsc_stop(struct ceph_mds_client *mdsc)
5666 {
5667         doutc(mdsc->fsc->client, "stop\n");
5668         /*
5669          * Make sure the delayed work stopped before releasing
5670          * the resources.
5671          *
5672          * Because the cancel_delayed_work_sync() will only
5673          * guarantee that the work finishes executing. But the
5674          * delayed work will re-arm itself again after that.
5675          */
5676         flush_delayed_work(&mdsc->delayed_work);
5677
5678         if (mdsc->mdsmap)
5679                 ceph_mdsmap_destroy(mdsc->mdsmap);
5680         kfree(mdsc->sessions);
5681         ceph_caps_finalize(mdsc);
5682         ceph_pool_perm_destroy(mdsc);
5683 }
5684
5685 void ceph_mdsc_destroy(struct ceph_fs_client *fsc)
5686 {
5687         struct ceph_mds_client *mdsc = fsc->mdsc;
5688         doutc(fsc->client, "%p\n", mdsc);
5689
5690         if (!mdsc)
5691                 return;
5692
5693         /* flush out any connection work with references to us */
5694         ceph_msgr_flush();
5695
5696         ceph_mdsc_stop(mdsc);
5697
5698         ceph_metric_destroy(&mdsc->metric);
5699
5700         fsc->mdsc = NULL;
5701         kfree(mdsc);
5702         doutc(fsc->client, "%p done\n", mdsc);
5703 }
5704
5705 void ceph_mdsc_handle_fsmap(struct ceph_mds_client *mdsc, struct ceph_msg *msg)
5706 {
5707         struct ceph_fs_client *fsc = mdsc->fsc;
5708         struct ceph_client *cl = fsc->client;
5709         const char *mds_namespace = fsc->mount_options->mds_namespace;
5710         void *p = msg->front.iov_base;
5711         void *end = p + msg->front.iov_len;
5712         u32 epoch;
5713         u32 num_fs;
5714         u32 mount_fscid = (u32)-1;
5715         int err = -EINVAL;
5716
5717         ceph_decode_need(&p, end, sizeof(u32), bad);
5718         epoch = ceph_decode_32(&p);
5719
5720         doutc(cl, "epoch %u\n", epoch);
5721
5722         /* struct_v, struct_cv, map_len, epoch, legacy_client_fscid */
5723         ceph_decode_skip_n(&p, end, 2 + sizeof(u32) * 3, bad);
5724
5725         ceph_decode_32_safe(&p, end, num_fs, bad);
5726         while (num_fs-- > 0) {
5727                 void *info_p, *info_end;
5728                 u32 info_len;
5729                 u32 fscid, namelen;
5730
5731                 ceph_decode_need(&p, end, 2 + sizeof(u32), bad);
5732                 p += 2;         // info_v, info_cv
5733                 info_len = ceph_decode_32(&p);
5734                 ceph_decode_need(&p, end, info_len, bad);
5735                 info_p = p;
5736                 info_end = p + info_len;
5737                 p = info_end;
5738
5739                 ceph_decode_need(&info_p, info_end, sizeof(u32) * 2, bad);
5740                 fscid = ceph_decode_32(&info_p);
5741                 namelen = ceph_decode_32(&info_p);
5742                 ceph_decode_need(&info_p, info_end, namelen, bad);
5743
5744                 if (mds_namespace &&
5745                     strlen(mds_namespace) == namelen &&
5746                     !strncmp(mds_namespace, (char *)info_p, namelen)) {
5747                         mount_fscid = fscid;
5748                         break;
5749                 }
5750         }
5751
5752         ceph_monc_got_map(&fsc->client->monc, CEPH_SUB_FSMAP, epoch);
5753         if (mount_fscid != (u32)-1) {
5754                 fsc->client->monc.fs_cluster_id = mount_fscid;
5755                 ceph_monc_want_map(&fsc->client->monc, CEPH_SUB_MDSMAP,
5756                                    0, true);
5757                 ceph_monc_renew_subs(&fsc->client->monc);
5758         } else {
5759                 err = -ENOENT;
5760                 goto err_out;
5761         }
5762         return;
5763
5764 bad:
5765         pr_err_client(cl, "error decoding fsmap %d. Shutting down mount.\n",
5766                       err);
5767         ceph_umount_begin(mdsc->fsc->sb);
5768         ceph_msg_dump(msg);
5769 err_out:
5770         mutex_lock(&mdsc->mutex);
5771         mdsc->mdsmap_err = err;
5772         __wake_requests(mdsc, &mdsc->waiting_for_map);
5773         mutex_unlock(&mdsc->mutex);
5774 }
5775
5776 /*
5777  * handle mds map update.
5778  */
5779 void ceph_mdsc_handle_mdsmap(struct ceph_mds_client *mdsc, struct ceph_msg *msg)
5780 {
5781         struct ceph_client *cl = mdsc->fsc->client;
5782         u32 epoch;
5783         u32 maplen;
5784         void *p = msg->front.iov_base;
5785         void *end = p + msg->front.iov_len;
5786         struct ceph_mdsmap *newmap, *oldmap;
5787         struct ceph_fsid fsid;
5788         int err = -EINVAL;
5789
5790         ceph_decode_need(&p, end, sizeof(fsid)+2*sizeof(u32), bad);
5791         ceph_decode_copy(&p, &fsid, sizeof(fsid));
5792         if (ceph_check_fsid(mdsc->fsc->client, &fsid) < 0)
5793                 return;
5794         epoch = ceph_decode_32(&p);
5795         maplen = ceph_decode_32(&p);
5796         doutc(cl, "epoch %u len %d\n", epoch, (int)maplen);
5797
5798         /* do we need it? */
5799         mutex_lock(&mdsc->mutex);
5800         if (mdsc->mdsmap && epoch <= mdsc->mdsmap->m_epoch) {
5801                 doutc(cl, "epoch %u <= our %u\n", epoch, mdsc->mdsmap->m_epoch);
5802                 mutex_unlock(&mdsc->mutex);
5803                 return;
5804         }
5805
5806         newmap = ceph_mdsmap_decode(mdsc, &p, end, ceph_msgr2(mdsc->fsc->client));
5807         if (IS_ERR(newmap)) {
5808                 err = PTR_ERR(newmap);
5809                 goto bad_unlock;
5810         }
5811
5812         /* swap into place */
5813         if (mdsc->mdsmap) {
5814                 oldmap = mdsc->mdsmap;
5815                 mdsc->mdsmap = newmap;
5816                 check_new_map(mdsc, newmap, oldmap);
5817                 ceph_mdsmap_destroy(oldmap);
5818         } else {
5819                 mdsc->mdsmap = newmap;  /* first mds map */
5820         }
5821         mdsc->fsc->max_file_size = min((loff_t)mdsc->mdsmap->m_max_file_size,
5822                                         MAX_LFS_FILESIZE);
5823
5824         __wake_requests(mdsc, &mdsc->waiting_for_map);
5825         ceph_monc_got_map(&mdsc->fsc->client->monc, CEPH_SUB_MDSMAP,
5826                           mdsc->mdsmap->m_epoch);
5827
5828         mutex_unlock(&mdsc->mutex);
5829         schedule_delayed(mdsc, 0);
5830         return;
5831
5832 bad_unlock:
5833         mutex_unlock(&mdsc->mutex);
5834 bad:
5835         pr_err_client(cl, "error decoding mdsmap %d. Shutting down mount.\n",
5836                       err);
5837         ceph_umount_begin(mdsc->fsc->sb);
5838         ceph_msg_dump(msg);
5839         return;
5840 }
5841
5842 static struct ceph_connection *mds_get_con(struct ceph_connection *con)
5843 {
5844         struct ceph_mds_session *s = con->private;
5845
5846         if (ceph_get_mds_session(s))
5847                 return con;
5848         return NULL;
5849 }
5850
5851 static void mds_put_con(struct ceph_connection *con)
5852 {
5853         struct ceph_mds_session *s = con->private;
5854
5855         ceph_put_mds_session(s);
5856 }
5857
5858 /*
5859  * if the client is unresponsive for long enough, the mds will kill
5860  * the session entirely.
5861  */
5862 static void mds_peer_reset(struct ceph_connection *con)
5863 {
5864         struct ceph_mds_session *s = con->private;
5865         struct ceph_mds_client *mdsc = s->s_mdsc;
5866
5867         pr_warn_client(mdsc->fsc->client, "mds%d closed our session\n",
5868                        s->s_mds);
5869         if (READ_ONCE(mdsc->fsc->mount_state) != CEPH_MOUNT_FENCE_IO)
5870                 send_mds_reconnect(mdsc, s);
5871 }
5872
5873 static void mds_dispatch(struct ceph_connection *con, struct ceph_msg *msg)
5874 {
5875         struct ceph_mds_session *s = con->private;
5876         struct ceph_mds_client *mdsc = s->s_mdsc;
5877         struct ceph_client *cl = mdsc->fsc->client;
5878         int type = le16_to_cpu(msg->hdr.type);
5879
5880         mutex_lock(&mdsc->mutex);
5881         if (__verify_registered_session(mdsc, s) < 0) {
5882                 mutex_unlock(&mdsc->mutex);
5883                 goto out;
5884         }
5885         mutex_unlock(&mdsc->mutex);
5886
5887         switch (type) {
5888         case CEPH_MSG_MDS_MAP:
5889                 ceph_mdsc_handle_mdsmap(mdsc, msg);
5890                 break;
5891         case CEPH_MSG_FS_MAP_USER:
5892                 ceph_mdsc_handle_fsmap(mdsc, msg);
5893                 break;
5894         case CEPH_MSG_CLIENT_SESSION:
5895                 handle_session(s, msg);
5896                 break;
5897         case CEPH_MSG_CLIENT_REPLY:
5898                 handle_reply(s, msg);
5899                 break;
5900         case CEPH_MSG_CLIENT_REQUEST_FORWARD:
5901                 handle_forward(mdsc, s, msg);
5902                 break;
5903         case CEPH_MSG_CLIENT_CAPS:
5904                 ceph_handle_caps(s, msg);
5905                 break;
5906         case CEPH_MSG_CLIENT_SNAP:
5907                 ceph_handle_snap(mdsc, s, msg);
5908                 break;
5909         case CEPH_MSG_CLIENT_LEASE:
5910                 handle_lease(mdsc, s, msg);
5911                 break;
5912         case CEPH_MSG_CLIENT_QUOTA:
5913                 ceph_handle_quota(mdsc, s, msg);
5914                 break;
5915
5916         default:
5917                 pr_err_client(cl, "received unknown message type %d %s\n",
5918                               type, ceph_msg_type_name(type));
5919         }
5920 out:
5921         ceph_msg_put(msg);
5922 }
5923
5924 /*
5925  * authentication
5926  */
5927
5928 /*
5929  * Note: returned pointer is the address of a structure that's
5930  * managed separately.  Caller must *not* attempt to free it.
5931  */
5932 static struct ceph_auth_handshake *
5933 mds_get_authorizer(struct ceph_connection *con, int *proto, int force_new)
5934 {
5935         struct ceph_mds_session *s = con->private;
5936         struct ceph_mds_client *mdsc = s->s_mdsc;
5937         struct ceph_auth_client *ac = mdsc->fsc->client->monc.auth;
5938         struct ceph_auth_handshake *auth = &s->s_auth;
5939         int ret;
5940
5941         ret = __ceph_auth_get_authorizer(ac, auth, CEPH_ENTITY_TYPE_MDS,
5942                                          force_new, proto, NULL, NULL);
5943         if (ret)
5944                 return ERR_PTR(ret);
5945
5946         return auth;
5947 }
5948
5949 static int mds_add_authorizer_challenge(struct ceph_connection *con,
5950                                     void *challenge_buf, int challenge_buf_len)
5951 {
5952         struct ceph_mds_session *s = con->private;
5953         struct ceph_mds_client *mdsc = s->s_mdsc;
5954         struct ceph_auth_client *ac = mdsc->fsc->client->monc.auth;
5955
5956         return ceph_auth_add_authorizer_challenge(ac, s->s_auth.authorizer,
5957                                             challenge_buf, challenge_buf_len);
5958 }
5959
5960 static int mds_verify_authorizer_reply(struct ceph_connection *con)
5961 {
5962         struct ceph_mds_session *s = con->private;
5963         struct ceph_mds_client *mdsc = s->s_mdsc;
5964         struct ceph_auth_client *ac = mdsc->fsc->client->monc.auth;
5965         struct ceph_auth_handshake *auth = &s->s_auth;
5966
5967         return ceph_auth_verify_authorizer_reply(ac, auth->authorizer,
5968                 auth->authorizer_reply_buf, auth->authorizer_reply_buf_len,
5969                 NULL, NULL, NULL, NULL);
5970 }
5971
5972 static int mds_invalidate_authorizer(struct ceph_connection *con)
5973 {
5974         struct ceph_mds_session *s = con->private;
5975         struct ceph_mds_client *mdsc = s->s_mdsc;
5976         struct ceph_auth_client *ac = mdsc->fsc->client->monc.auth;
5977
5978         ceph_auth_invalidate_authorizer(ac, CEPH_ENTITY_TYPE_MDS);
5979
5980         return ceph_monc_validate_auth(&mdsc->fsc->client->monc);
5981 }
5982
5983 static int mds_get_auth_request(struct ceph_connection *con,
5984                                 void *buf, int *buf_len,
5985                                 void **authorizer, int *authorizer_len)
5986 {
5987         struct ceph_mds_session *s = con->private;
5988         struct ceph_auth_client *ac = s->s_mdsc->fsc->client->monc.auth;
5989         struct ceph_auth_handshake *auth = &s->s_auth;
5990         int ret;
5991
5992         ret = ceph_auth_get_authorizer(ac, auth, CEPH_ENTITY_TYPE_MDS,
5993                                        buf, buf_len);
5994         if (ret)
5995                 return ret;
5996
5997         *authorizer = auth->authorizer_buf;
5998         *authorizer_len = auth->authorizer_buf_len;
5999         return 0;
6000 }
6001
6002 static int mds_handle_auth_reply_more(struct ceph_connection *con,
6003                                       void *reply, int reply_len,
6004                                       void *buf, int *buf_len,
6005                                       void **authorizer, int *authorizer_len)
6006 {
6007         struct ceph_mds_session *s = con->private;
6008         struct ceph_auth_client *ac = s->s_mdsc->fsc->client->monc.auth;
6009         struct ceph_auth_handshake *auth = &s->s_auth;
6010         int ret;
6011
6012         ret = ceph_auth_handle_svc_reply_more(ac, auth, reply, reply_len,
6013                                               buf, buf_len);
6014         if (ret)
6015                 return ret;
6016
6017         *authorizer = auth->authorizer_buf;
6018         *authorizer_len = auth->authorizer_buf_len;
6019         return 0;
6020 }
6021
6022 static int mds_handle_auth_done(struct ceph_connection *con,
6023                                 u64 global_id, void *reply, int reply_len,
6024                                 u8 *session_key, int *session_key_len,
6025                                 u8 *con_secret, int *con_secret_len)
6026 {
6027         struct ceph_mds_session *s = con->private;
6028         struct ceph_auth_client *ac = s->s_mdsc->fsc->client->monc.auth;
6029         struct ceph_auth_handshake *auth = &s->s_auth;
6030
6031         return ceph_auth_handle_svc_reply_done(ac, auth, reply, reply_len,
6032                                                session_key, session_key_len,
6033                                                con_secret, con_secret_len);
6034 }
6035
6036 static int mds_handle_auth_bad_method(struct ceph_connection *con,
6037                                       int used_proto, int result,
6038                                       const int *allowed_protos, int proto_cnt,
6039                                       const int *allowed_modes, int mode_cnt)
6040 {
6041         struct ceph_mds_session *s = con->private;
6042         struct ceph_mon_client *monc = &s->s_mdsc->fsc->client->monc;
6043         int ret;
6044
6045         if (ceph_auth_handle_bad_authorizer(monc->auth, CEPH_ENTITY_TYPE_MDS,
6046                                             used_proto, result,
6047                                             allowed_protos, proto_cnt,
6048                                             allowed_modes, mode_cnt)) {
6049                 ret = ceph_monc_validate_auth(monc);
6050                 if (ret)
6051                         return ret;
6052         }
6053
6054         return -EACCES;
6055 }
6056
6057 static struct ceph_msg *mds_alloc_msg(struct ceph_connection *con,
6058                                 struct ceph_msg_header *hdr, int *skip)
6059 {
6060         struct ceph_msg *msg;
6061         int type = (int) le16_to_cpu(hdr->type);
6062         int front_len = (int) le32_to_cpu(hdr->front_len);
6063
6064         if (con->in_msg)
6065                 return con->in_msg;
6066
6067         *skip = 0;
6068         msg = ceph_msg_new(type, front_len, GFP_NOFS, false);
6069         if (!msg) {
6070                 pr_err("unable to allocate msg type %d len %d\n",
6071                        type, front_len);
6072                 return NULL;
6073         }
6074
6075         return msg;
6076 }
6077
6078 static int mds_sign_message(struct ceph_msg *msg)
6079 {
6080        struct ceph_mds_session *s = msg->con->private;
6081        struct ceph_auth_handshake *auth = &s->s_auth;
6082
6083        return ceph_auth_sign_message(auth, msg);
6084 }
6085
6086 static int mds_check_message_signature(struct ceph_msg *msg)
6087 {
6088        struct ceph_mds_session *s = msg->con->private;
6089        struct ceph_auth_handshake *auth = &s->s_auth;
6090
6091        return ceph_auth_check_message_signature(auth, msg);
6092 }
6093
6094 static const struct ceph_connection_operations mds_con_ops = {
6095         .get = mds_get_con,
6096         .put = mds_put_con,
6097         .alloc_msg = mds_alloc_msg,
6098         .dispatch = mds_dispatch,
6099         .peer_reset = mds_peer_reset,
6100         .get_authorizer = mds_get_authorizer,
6101         .add_authorizer_challenge = mds_add_authorizer_challenge,
6102         .verify_authorizer_reply = mds_verify_authorizer_reply,
6103         .invalidate_authorizer = mds_invalidate_authorizer,
6104         .sign_message = mds_sign_message,
6105         .check_message_signature = mds_check_message_signature,
6106         .get_auth_request = mds_get_auth_request,
6107         .handle_auth_reply_more = mds_handle_auth_reply_more,
6108         .handle_auth_done = mds_handle_auth_done,
6109         .handle_auth_bad_method = mds_handle_auth_bad_method,
6110 };
6111
6112 /* eof */