drivers/staging/lustre/lnet/selftest/console.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  19  *
  20  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  21  * CA 95054 USA or visit www.sun.com if you need additional information or
  22  * have any questions.
  23  *
  24  * GPL HEADER END
  25  */
  26 /*
  27  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  28  * Use is subject to license terms.
  29  *
  30  * Copyright (c) 2012, Intel Corporation.
  31  */
  32 /*
  33  * This file is part of Lustre, http://www.lustre.org/
  34  * Lustre is a trademark of Sun Microsystems, Inc.
  35  *
  36  * lnet/selftest/conctl.c
  37  *
  38  * Infrastructure of LST console
  39  *
  40  * Author: Liang Zhen <liangzhen@clusterfs.com>
  41  */
  42
  43
  44 #include <linux/libcfs/libcfs.h>
  45 #include <linux/lnet/lib-lnet.h>
  46 #include "console.h"
  47 #include "conrpc.h"
  48
  49 #define LST_NODE_STATE_COUNTER(nd, p)              \
  50 do {                                                \
  51         if ((nd)->nd_state == LST_NODE_ACTIVE)    \
  52                 (p)->nle_nactive ++;                \
  53         else if ((nd)->nd_state == LST_NODE_BUSY)       \
  54                 (p)->nle_nbusy ++;                    \
  55         else if ((nd)->nd_state == LST_NODE_DOWN)       \
  56                 (p)->nle_ndown ++;                    \
  57         else                                        \
  58                 (p)->nle_nunknown ++;              \
  59         (p)->nle_nnode ++;                            \
  60 } while (0)
  61
  62 lstcon_session_t        console_session;
  63
  64 void
  65 lstcon_node_get(lstcon_node_t *nd)
  66 {
  67         LASSERT (nd->nd_ref >= 1);
  68
  69         nd->nd_ref++;
  70 }
  71
  72 static int
  73 lstcon_node_find(lnet_process_id_t id, lstcon_node_t **ndpp, int create)
  74 {
  75         lstcon_ndlink_t *ndl;
  76         unsigned int     idx = LNET_NIDADDR(id.nid) % LST_GLOBAL_HASHSIZE;
  77
  78         LASSERT (id.nid != LNET_NID_ANY);
  79
  80         list_for_each_entry(ndl, &console_session.ses_ndl_hash[idx], ndl_hlink) {
  81                 if (ndl->ndl_node->nd_id.nid != id.nid ||
  82                     ndl->ndl_node->nd_id.pid != id.pid)
  83                         continue;
  84
  85                 lstcon_node_get(ndl->ndl_node);
  86                 *ndpp = ndl->ndl_node;
  87                 return 0;
  88         }
  89
  90         if (!create)
  91                 return -ENOENT;
  92
  93         LIBCFS_ALLOC(*ndpp, sizeof(lstcon_node_t) + sizeof(lstcon_ndlink_t));
  94         if (*ndpp == NULL)
  95                 return -ENOMEM;
  96
  97         ndl = (lstcon_ndlink_t *)(*ndpp + 1);
  98
  99         ndl->ndl_node = *ndpp;
 100
 101         ndl->ndl_node->nd_ref   = 1;
 102         ndl->ndl_node->nd_id    = id;
 103         ndl->ndl_node->nd_stamp = cfs_time_current();
 104         ndl->ndl_node->nd_state = LST_NODE_UNKNOWN;
 105         ndl->ndl_node->nd_timeout = 0;
 106         memset(&ndl->ndl_node->nd_ping, 0, sizeof(lstcon_rpc_t));
 107
 108         /* queued in global hash & list, no refcount is taken by
 109          * global hash & list, if caller release his refcount,
 110          * node will be released */
 111         list_add_tail(&ndl->ndl_hlink, &console_session.ses_ndl_hash[idx]);
 112         list_add_tail(&ndl->ndl_link, &console_session.ses_ndl_list);
 113
 114         return 0;
 115 }
 116
 117 void
 118 lstcon_node_put(lstcon_node_t *nd)
 119 {
 120         lstcon_ndlink_t  *ndl;
 121
 122         LASSERT (nd->nd_ref > 0);
 123
 124         if (--nd->nd_ref > 0)
 125                 return;
 126
 127         ndl = (lstcon_ndlink_t *)(nd + 1);
 128
 129         LASSERT (!list_empty(&ndl->ndl_link));
 130         LASSERT (!list_empty(&ndl->ndl_hlink));
 131
 132         /* remove from session */
 133         list_del(&ndl->ndl_link);
 134         list_del(&ndl->ndl_hlink);
 135
 136         LIBCFS_FREE(nd, sizeof(lstcon_node_t) + sizeof(lstcon_ndlink_t));
 137 }
 138
 139 static int
 140 lstcon_ndlink_find(struct list_head *hash,
 141                    lnet_process_id_t id, lstcon_ndlink_t **ndlpp, int create)
 142 {
 143         unsigned int     idx = LNET_NIDADDR(id.nid) % LST_NODE_HASHSIZE;
 144         lstcon_ndlink_t *ndl;
 145         lstcon_node_t   *nd;
 146         int           rc;
 147
 148         if (id.nid == LNET_NID_ANY)
 149                 return -EINVAL;
 150
 151         /* search in hash */
 152         list_for_each_entry(ndl, &hash[idx], ndl_hlink) {
 153                 if (ndl->ndl_node->nd_id.nid != id.nid ||
 154                     ndl->ndl_node->nd_id.pid != id.pid)
 155                         continue;
 156
 157                 *ndlpp = ndl;
 158                 return 0;
 159         }
 160
 161         if (create == 0)
 162                 return -ENOENT;
 163
 164         /* find or create in session hash */
 165         rc = lstcon_node_find(id, &nd, (create == 1) ? 1 : 0);
 166         if (rc != 0)
 167                 return rc;
 168
 169         LIBCFS_ALLOC(ndl, sizeof(lstcon_ndlink_t));
 170         if (ndl == NULL) {
 171                 lstcon_node_put(nd);
 172                 return -ENOMEM;
 173         }
 174
 175         *ndlpp = ndl;
 176
 177         ndl->ndl_node = nd;
 178         INIT_LIST_HEAD(&ndl->ndl_link);
 179         list_add_tail(&ndl->ndl_hlink, &hash[idx]);
 180
 181         return  0;
 182 }
 183
 184 static void
 185 lstcon_ndlink_release(lstcon_ndlink_t *ndl)
 186 {
 187         LASSERT (list_empty(&ndl->ndl_link));
 188         LASSERT (!list_empty(&ndl->ndl_hlink));
 189
 190         list_del(&ndl->ndl_hlink); /* delete from hash */
 191         lstcon_node_put(ndl->ndl_node);
 192
 193         LIBCFS_FREE(ndl, sizeof(*ndl));
 194 }
 195
 196 static int
 197 lstcon_group_alloc(char *name, lstcon_group_t **grpp)
 198 {
 199         lstcon_group_t *grp;
 200         int          i;
 201
 202         LIBCFS_ALLOC(grp, offsetof(lstcon_group_t,
 203                                    grp_ndl_hash[LST_NODE_HASHSIZE]));
 204         if (grp == NULL)
 205                 return -ENOMEM;
 206
 207         memset(grp, 0, offsetof(lstcon_group_t,
 208                                 grp_ndl_hash[LST_NODE_HASHSIZE]));
 209
 210         grp->grp_ref = 1;
 211         if (name != NULL)
 212                 strcpy(grp->grp_name, name);
 213
 214         INIT_LIST_HEAD(&grp->grp_link);
 215         INIT_LIST_HEAD(&grp->grp_ndl_list);
 216         INIT_LIST_HEAD(&grp->grp_trans_list);
 217
 218         for (i = 0; i < LST_NODE_HASHSIZE; i++)
 219                 INIT_LIST_HEAD(&grp->grp_ndl_hash[i]);
 220
 221         *grpp = grp;
 222
 223         return 0;
 224 }
 225
 226 static void
 227 lstcon_group_addref(lstcon_group_t *grp)
 228 {
 229         grp->grp_ref ++;
 230 }
 231
 232 static void lstcon_group_ndlink_release(lstcon_group_t *, lstcon_ndlink_t *);
 233
 234 static void
 235 lstcon_group_drain(lstcon_group_t *grp, int keep)
 236 {
 237         lstcon_ndlink_t *ndl;
 238         lstcon_ndlink_t *tmp;
 239
 240         list_for_each_entry_safe(ndl, tmp, &grp->grp_ndl_list, ndl_link) {
 241                 if ((ndl->ndl_node->nd_state & keep) == 0)
 242                         lstcon_group_ndlink_release(grp, ndl);
 243         }
 244 }
 245
 246 static void
 247 lstcon_group_decref(lstcon_group_t *grp)
 248 {
 249         int     i;
 250
 251         if (--grp->grp_ref > 0)
 252                 return;
 253
 254         if (!list_empty(&grp->grp_link))
 255                 list_del(&grp->grp_link);
 256
 257         lstcon_group_drain(grp, 0);
 258
 259         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
 260                 LASSERT (list_empty(&grp->grp_ndl_hash[i]));
 261         }
 262
 263         LIBCFS_FREE(grp, offsetof(lstcon_group_t,
 264                                   grp_ndl_hash[LST_NODE_HASHSIZE]));
 265 }
 266
 267 static int
 268 lstcon_group_find(const char *name, lstcon_group_t **grpp)
 269 {
 270         lstcon_group_t   *grp;
 271
 272         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
 273                 if (strncmp(grp->grp_name, name, LST_NAME_SIZE) != 0)
 274                         continue;
 275
 276                 lstcon_group_addref(grp);  /* +1 ref for caller */
 277                 *grpp = grp;
 278                 return 0;
 279         }
 280
 281         return -ENOENT;
 282 }
 283
 284 static void
 285 lstcon_group_put(lstcon_group_t *grp)
 286 {
 287         lstcon_group_decref(grp);
 288 }
 289
 290 static int
 291 lstcon_group_ndlink_find(lstcon_group_t *grp, lnet_process_id_t id,
 292                          lstcon_ndlink_t **ndlpp, int create)
 293 {
 294         int     rc;
 295
 296         rc = lstcon_ndlink_find(&grp->grp_ndl_hash[0], id, ndlpp, create);
 297         if (rc != 0)
 298                 return rc;
 299
 300         if (!list_empty(&(*ndlpp)->ndl_link))
 301                 return 0;
 302
 303         list_add_tail(&(*ndlpp)->ndl_link, &grp->grp_ndl_list);
 304         grp->grp_nnode ++;
 305
 306         return 0;
 307 }
 308
 309 static void
 310 lstcon_group_ndlink_release(lstcon_group_t *grp, lstcon_ndlink_t *ndl)
 311 {
 312         list_del_init(&ndl->ndl_link);
 313         lstcon_ndlink_release(ndl);
 314         grp->grp_nnode --;
 315 }
 316
 317 static void
 318 lstcon_group_ndlink_move(lstcon_group_t *old,
 319                          lstcon_group_t *new, lstcon_ndlink_t *ndl)
 320 {
 321         unsigned int idx = LNET_NIDADDR(ndl->ndl_node->nd_id.nid) %
 322                            LST_NODE_HASHSIZE;
 323
 324         list_del(&ndl->ndl_hlink);
 325         list_del(&ndl->ndl_link);
 326         old->grp_nnode --;
 327
 328         list_add_tail(&ndl->ndl_hlink, &new->grp_ndl_hash[idx]);
 329         list_add_tail(&ndl->ndl_link, &new->grp_ndl_list);
 330         new->grp_nnode ++;
 331
 332         return;
 333 }
 334
 335 static void
 336 lstcon_group_move(lstcon_group_t *old, lstcon_group_t *new)
 337 {
 338         lstcon_ndlink_t *ndl;
 339
 340         while (!list_empty(&old->grp_ndl_list)) {
 341                 ndl = list_entry(old->grp_ndl_list.next,
 342                                      lstcon_ndlink_t, ndl_link);
 343                 lstcon_group_ndlink_move(old, new, ndl);
 344         }
 345 }
 346
 347 int
 348 lstcon_sesrpc_condition(int transop, lstcon_node_t *nd, void *arg)
 349 {
 350         lstcon_group_t *grp = (lstcon_group_t *)arg;
 351
 352         switch (transop) {
 353         case LST_TRANS_SESNEW:
 354                 if (nd->nd_state == LST_NODE_ACTIVE)
 355                         return 0;
 356                 break;
 357
 358         case LST_TRANS_SESEND:
 359                 if (nd->nd_state != LST_NODE_ACTIVE)
 360                         return 0;
 361
 362                 if (grp != NULL && nd->nd_ref > 1)
 363                         return 0;
 364                 break;
 365
 366         case LST_TRANS_SESQRY:
 367                 break;
 368
 369         default:
 370                 LBUG();
 371         }
 372
 373         return 1;
 374 }
 375
 376 int
 377 lstcon_sesrpc_readent(int transop, srpc_msg_t *msg,
 378                       lstcon_rpc_ent_t *ent_up)
 379 {
 380         srpc_debug_reply_t *rep;
 381
 382         switch (transop) {
 383         case LST_TRANS_SESNEW:
 384         case LST_TRANS_SESEND:
 385                 return 0;
 386
 387         case LST_TRANS_SESQRY:
 388                 rep = &msg->msg_body.dbg_reply;
 389
 390                 if (copy_to_user(&ent_up->rpe_priv[0],
 391                                      &rep->dbg_timeout, sizeof(int)) ||
 392                     copy_to_user(&ent_up->rpe_payload[0],
 393                                      &rep->dbg_name, LST_NAME_SIZE))
 394                         return -EFAULT;
 395
 396                 return 0;
 397
 398         default:
 399                 LBUG();
 400         }
 401
 402         return 0;
 403 }
 404
 405 static int
 406 lstcon_group_nodes_add(lstcon_group_t *grp,
 407                        int count, lnet_process_id_t *ids_up,
 408                        unsigned *featp, struct list_head *result_up)
 409 {
 410         lstcon_rpc_trans_t      *trans;
 411         lstcon_ndlink_t  *ndl;
 412         lstcon_group_t    *tmp;
 413         lnet_process_id_t       id;
 414         int                   i;
 415         int                   rc;
 416
 417         rc = lstcon_group_alloc(NULL, &tmp);
 418         if (rc != 0) {
 419                 CERROR("Out of memory\n");
 420                 return -ENOMEM;
 421         }
 422
 423         for (i = 0 ; i < count; i++) {
 424                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
 425                         rc = -EFAULT;
 426                         break;
 427                 }
 428
 429                 /* skip if it's in this group already */
 430                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 0);
 431                 if (rc == 0)
 432                         continue;
 433
 434                 /* add to tmp group */
 435                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 1);
 436                 if (rc != 0) {
 437                         CERROR("Can't create ndlink, out of memory\n");
 438                         break;
 439                 }
 440         }
 441
 442         if (rc != 0) {
 443                 lstcon_group_put(tmp);
 444                 return rc;
 445         }
 446
 447         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 448                                      &tmp->grp_trans_list, LST_TRANS_SESNEW,
 449                                      tmp, lstcon_sesrpc_condition, &trans);
 450         if (rc != 0) {
 451                 CERROR("Can't create transaction: %d\n", rc);
 452                 lstcon_group_put(tmp);
 453                 return rc;
 454         }
 455
 456         /* post all RPCs */
 457         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 458
 459         rc = lstcon_rpc_trans_interpreter(trans, result_up,
 460                                           lstcon_sesrpc_readent);
 461         *featp = trans->tas_features;
 462
 463         /* destroy all RPGs */
 464         lstcon_rpc_trans_destroy(trans);
 465
 466         lstcon_group_move(tmp, grp);
 467         lstcon_group_put(tmp);
 468
 469         return rc;
 470 }
 471
 472 static int
 473 lstcon_group_nodes_remove(lstcon_group_t *grp,
 474                           int count, lnet_process_id_t *ids_up,
 475                           struct list_head *result_up)
 476 {
 477         lstcon_rpc_trans_t     *trans;
 478         lstcon_ndlink_t *ndl;
 479         lstcon_group_t   *tmp;
 480         lnet_process_id_t       id;
 481         int                  rc;
 482         int                  i;
 483
 484         /* End session and remove node from the group */
 485
 486         rc = lstcon_group_alloc(NULL, &tmp);
 487         if (rc != 0) {
 488                 CERROR("Out of memory\n");
 489                 return -ENOMEM;
 490         }
 491
 492         for (i = 0; i < count; i++) {
 493                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
 494                         rc = -EFAULT;
 495                         goto error;
 496                 }
 497
 498                 /* move node to tmp group */
 499                 if (lstcon_group_ndlink_find(grp, id, &ndl, 0) == 0)
 500                         lstcon_group_ndlink_move(grp, tmp, ndl);
 501         }
 502
 503         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 504                                      &tmp->grp_trans_list, LST_TRANS_SESEND,
 505                                      tmp, lstcon_sesrpc_condition, &trans);
 506         if (rc != 0) {
 507                 CERROR("Can't create transaction: %d\n", rc);
 508                 goto error;
 509         }
 510
 511         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 512
 513         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 514
 515         lstcon_rpc_trans_destroy(trans);
 516         /* release nodes anyway, because we can't rollback status */
 517         lstcon_group_put(tmp);
 518
 519         return rc;
 520 error:
 521         lstcon_group_move(tmp, grp);
 522         lstcon_group_put(tmp);
 523
 524         return rc;
 525 }
 526
 527 int
 528 lstcon_group_add(char *name)
 529 {
 530         lstcon_group_t *grp;
 531         int          rc;
 532
 533         rc = (lstcon_group_find(name, &grp) == 0)? -EEXIST: 0;
 534         if (rc != 0) {
 535                 /* find a group with same name */
 536                 lstcon_group_put(grp);
 537                 return rc;
 538         }
 539
 540         rc = lstcon_group_alloc(name, &grp);
 541         if (rc != 0) {
 542                 CERROR("Can't allocate descriptor for group %s\n", name);
 543                 return -ENOMEM;
 544         }
 545
 546         list_add_tail(&grp->grp_link, &console_session.ses_grp_list);
 547
 548         return rc;
 549 }
 550
 551 int
 552 lstcon_nodes_add(char *name, int count, lnet_process_id_t *ids_up,
 553                  unsigned *featp, struct list_head *result_up)
 554 {
 555         lstcon_group_t   *grp;
 556         int                  rc;
 557
 558         LASSERT (count > 0);
 559         LASSERT (ids_up != NULL);
 560
 561         rc = lstcon_group_find(name, &grp);
 562         if (rc != 0) {
 563                 CDEBUG(D_NET, "Can't find group %s\n", name);
 564                 return rc;
 565         }
 566
 567         if (grp->grp_ref > 2) {
 568                 /* referred by other threads or test */
 569                 CDEBUG(D_NET, "Group %s is busy\n", name);
 570                 lstcon_group_put(grp);
 571
 572                 return -EBUSY;
 573         }
 574
 575         rc = lstcon_group_nodes_add(grp, count, ids_up, featp, result_up);
 576
 577         lstcon_group_put(grp);
 578
 579         return rc;
 580 }
 581
 582 int
 583 lstcon_group_del(char *name)
 584 {
 585         lstcon_rpc_trans_t *trans;
 586         lstcon_group_t     *grp;
 587         int              rc;
 588
 589         rc = lstcon_group_find(name, &grp);
 590         if (rc != 0) {
 591                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 592                 return rc;
 593         }
 594
 595         if (grp->grp_ref > 2) {
 596                 /* referred by others threads or test */
 597                 CDEBUG(D_NET, "Group %s is busy\n", name);
 598                 lstcon_group_put(grp);
 599                 return -EBUSY;
 600         }
 601
 602         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 603                                      &grp->grp_trans_list, LST_TRANS_SESEND,
 604                                      grp, lstcon_sesrpc_condition, &trans);
 605         if (rc != 0) {
 606                 CERROR("Can't create transaction: %d\n", rc);
 607                 lstcon_group_put(grp);
 608                 return rc;
 609         }
 610
 611         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 612
 613         lstcon_rpc_trans_destroy(trans);
 614
 615         lstcon_group_put(grp);
 616         /* -ref for session, it's destroyed,
 617          * status can't be rolled back, destroy group anway */
 618         lstcon_group_put(grp);
 619
 620         return rc;
 621 }
 622
 623 int
 624 lstcon_group_clean(char *name, int args)
 625 {
 626         lstcon_group_t *grp = NULL;
 627         int          rc;
 628
 629         rc = lstcon_group_find(name, &grp);
 630         if (rc != 0) {
 631                 CDEBUG(D_NET, "Can't find group %s\n", name);
 632                 return rc;
 633         }
 634
 635         if (grp->grp_ref > 2) {
 636                 /* referred by test */
 637                 CDEBUG(D_NET, "Group %s is busy\n", name);
 638                 lstcon_group_put(grp);
 639                 return -EBUSY;
 640         }
 641
 642         args = (LST_NODE_ACTIVE | LST_NODE_BUSY |
 643                 LST_NODE_DOWN | LST_NODE_UNKNOWN) & ~args;
 644
 645         lstcon_group_drain(grp, args);
 646
 647         lstcon_group_put(grp);
 648         /* release empty group */
 649         if (list_empty(&grp->grp_ndl_list))
 650                 lstcon_group_put(grp);
 651
 652         return 0;
 653 }
 654
 655 int
 656 lstcon_nodes_remove(char *name, int count,
 657                     lnet_process_id_t *ids_up, struct list_head *result_up)
 658 {
 659         lstcon_group_t *grp = NULL;
 660         int          rc;
 661
 662         rc = lstcon_group_find(name, &grp);
 663         if (rc != 0) {
 664                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 665                 return rc;
 666         }
 667
 668         if (grp->grp_ref > 2) {
 669                 /* referred by test */
 670                 CDEBUG(D_NET, "Group %s is busy\n", name);
 671                 lstcon_group_put(grp);
 672                 return -EBUSY;
 673         }
 674
 675         rc = lstcon_group_nodes_remove(grp, count, ids_up, result_up);
 676
 677         lstcon_group_put(grp);
 678         /* release empty group */
 679         if (list_empty(&grp->grp_ndl_list))
 680                 lstcon_group_put(grp);
 681
 682         return rc;
 683 }
 684
 685 int
 686 lstcon_group_refresh(char *name, struct list_head *result_up)
 687 {
 688         lstcon_rpc_trans_t      *trans;
 689         lstcon_group_t    *grp;
 690         int                   rc;
 691
 692         rc = lstcon_group_find(name, &grp);
 693         if (rc != 0) {
 694                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 695                 return rc;
 696         }
 697
 698         if (grp->grp_ref > 2) {
 699                 /* referred by test */
 700                 CDEBUG(D_NET, "Group %s is busy\n", name);
 701                 lstcon_group_put(grp);
 702                 return -EBUSY;
 703         }
 704
 705         /* re-invite all inactive nodes int the group */
 706         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 707                                      &grp->grp_trans_list, LST_TRANS_SESNEW,
 708                                      grp, lstcon_sesrpc_condition, &trans);
 709         if (rc != 0) {
 710                 /* local error, return */
 711                 CDEBUG(D_NET, "Can't create transaction: %d\n", rc);
 712                 lstcon_group_put(grp);
 713                 return rc;
 714         }
 715
 716         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 717
 718         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 719
 720         lstcon_rpc_trans_destroy(trans);
 721         /* -ref for me */
 722         lstcon_group_put(grp);
 723
 724         return rc;
 725 }
 726
 727 int
 728 lstcon_group_list(int index, int len, char *name_up)
 729 {
 730         lstcon_group_t *grp;
 731
 732         LASSERT (index >= 0);
 733         LASSERT (name_up != NULL);
 734
 735         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
 736                 if (index-- == 0) {
 737                         return copy_to_user(name_up, grp->grp_name, len) ?
 738                                -EFAULT : 0;
 739                 }
 740         }
 741
 742         return -ENOENT;
 743 }
 744
 745 static int
 746 lstcon_nodes_getent(struct list_head *head, int *index_p,
 747                     int *count_p, lstcon_node_ent_t *dents_up)
 748 {
 749         lstcon_ndlink_t  *ndl;
 750         lstcon_node_t    *nd;
 751         int            count = 0;
 752         int            index = 0;
 753
 754         LASSERT (index_p != NULL && count_p != NULL);
 755         LASSERT (dents_up != NULL);
 756         LASSERT (*index_p >= 0);
 757         LASSERT (*count_p > 0);
 758
 759         list_for_each_entry(ndl, head, ndl_link) {
 760                 if (index++ < *index_p)
 761                         continue;
 762
 763                 if (count >= *count_p)
 764                         break;
 765
 766                 nd = ndl->ndl_node;
 767                 if (copy_to_user(&dents_up[count].nde_id,
 768                                      &nd->nd_id, sizeof(nd->nd_id)) ||
 769                     copy_to_user(&dents_up[count].nde_state,
 770                                      &nd->nd_state, sizeof(nd->nd_state)))
 771                         return -EFAULT;
 772
 773                 count ++;
 774         }
 775
 776         if (index <= *index_p)
 777                 return -ENOENT;
 778
 779         *count_p = count;
 780         *index_p = index;
 781
 782         return 0;
 783 }
 784
 785 int
 786 lstcon_group_info(char *name, lstcon_ndlist_ent_t *gents_p,
 787                   int *index_p, int *count_p, lstcon_node_ent_t *dents_up)
 788 {
 789         lstcon_ndlist_ent_t *gentp;
 790         lstcon_group_t      *grp;
 791         lstcon_ndlink_t     *ndl;
 792         int               rc;
 793
 794         rc = lstcon_group_find(name, &grp);
 795         if (rc != 0) {
 796                 CDEBUG(D_NET, "Can't find group %s\n", name);
 797                 return rc;
 798         }
 799
 800         if (dents_up) {
 801                 /* verbose query */
 802                 rc = lstcon_nodes_getent(&grp->grp_ndl_list,
 803                                          index_p, count_p, dents_up);
 804                 lstcon_group_put(grp);
 805
 806                 return rc;
 807         }
 808
 809         /* non-verbose query */
 810         LIBCFS_ALLOC(gentp, sizeof(lstcon_ndlist_ent_t));
 811         if (gentp == NULL) {
 812                 CERROR("Can't allocate ndlist_ent\n");
 813                 lstcon_group_put(grp);
 814
 815                 return -ENOMEM;
 816         }
 817
 818         memset(gentp, 0, sizeof(lstcon_ndlist_ent_t));
 819
 820         list_for_each_entry(ndl, &grp->grp_ndl_list, ndl_link)
 821                 LST_NODE_STATE_COUNTER(ndl->ndl_node, gentp);
 822
 823         rc = copy_to_user(gents_p, gentp,
 824                               sizeof(lstcon_ndlist_ent_t)) ? -EFAULT: 0;
 825
 826         LIBCFS_FREE(gentp, sizeof(lstcon_ndlist_ent_t));
 827
 828         lstcon_group_put(grp);
 829
 830         return 0;
 831 }
 832
 833 int
 834 lstcon_batch_find(const char *name, lstcon_batch_t **batpp)
 835 {
 836         lstcon_batch_t   *bat;
 837
 838         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
 839                 if (strncmp(bat->bat_name, name, LST_NAME_SIZE) == 0) {
 840                         *batpp = bat;
 841                         return 0;
 842                 }
 843         }
 844
 845         return -ENOENT;
 846 }
 847
 848 int
 849 lstcon_batch_add(char *name)
 850 {
 851         lstcon_batch_t   *bat;
 852         int            i;
 853         int            rc;
 854
 855         rc = (lstcon_batch_find(name, &bat) == 0)? -EEXIST: 0;
 856         if (rc != 0) {
 857                 CDEBUG(D_NET, "Batch %s already exists\n", name);
 858                 return rc;
 859         }
 860
 861         LIBCFS_ALLOC(bat, sizeof(lstcon_batch_t));
 862         if (bat == NULL) {
 863                 CERROR("Can't allocate descriptor for batch %s\n", name);
 864                 return -ENOMEM;
 865         }
 866
 867         LIBCFS_ALLOC(bat->bat_cli_hash,
 868                      sizeof(struct list_head) * LST_NODE_HASHSIZE);
 869         if (bat->bat_cli_hash == NULL) {
 870                 CERROR("Can't allocate hash for batch %s\n", name);
 871                 LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
 872
 873                 return -ENOMEM;
 874         }
 875
 876         LIBCFS_ALLOC(bat->bat_srv_hash,
 877                      sizeof(struct list_head) * LST_NODE_HASHSIZE);
 878         if (bat->bat_srv_hash == NULL) {
 879                 CERROR("Can't allocate hash for batch %s\n", name);
 880                 LIBCFS_FREE(bat->bat_cli_hash, LST_NODE_HASHSIZE);
 881                 LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
 882
 883                 return -ENOMEM;
 884         }
 885
 886         strcpy(bat->bat_name, name);
 887         bat->bat_hdr.tsb_index = 0;
 888         bat->bat_hdr.tsb_id.bat_id = ++console_session.ses_id_cookie;
 889
 890         bat->bat_ntest = 0;
 891         bat->bat_state = LST_BATCH_IDLE;
 892
 893         INIT_LIST_HEAD(&bat->bat_cli_list);
 894         INIT_LIST_HEAD(&bat->bat_srv_list);
 895         INIT_LIST_HEAD(&bat->bat_test_list);
 896         INIT_LIST_HEAD(&bat->bat_trans_list);
 897
 898         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
 899                 INIT_LIST_HEAD(&bat->bat_cli_hash[i]);
 900                 INIT_LIST_HEAD(&bat->bat_srv_hash[i]);
 901         }
 902
 903         list_add_tail(&bat->bat_link, &console_session.ses_bat_list);
 904
 905         return rc;
 906 }
 907
 908 int
 909 lstcon_batch_list(int index, int len, char *name_up)
 910 {
 911         lstcon_batch_t    *bat;
 912
 913         LASSERT (name_up != NULL);
 914         LASSERT (index >= 0);
 915
 916         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
 917                 if (index-- == 0) {
 918                         return copy_to_user(name_up,bat->bat_name, len) ?
 919                                -EFAULT: 0;
 920                 }
 921         }
 922
 923         return -ENOENT;
 924 }
 925
 926 int
 927 lstcon_batch_info(char *name, lstcon_test_batch_ent_t *ent_up, int server,
 928                   int testidx, int *index_p, int *ndent_p,
 929                   lstcon_node_ent_t *dents_up)
 930 {
 931         lstcon_test_batch_ent_t *entp;
 932         struct list_head              *clilst;
 933         struct list_head              *srvlst;
 934         lstcon_test_t      *test = NULL;
 935         lstcon_batch_t    *bat;
 936         lstcon_ndlink_t  *ndl;
 937         int                   rc;
 938
 939         rc = lstcon_batch_find(name, &bat);
 940         if (rc != 0) {
 941                 CDEBUG(D_NET, "Can't find batch %s\n", name);
 942                 return -ENOENT;
 943         }
 944
 945         if (testidx > 0) {
 946                 /* query test, test index start from 1 */
 947                 list_for_each_entry(test, &bat->bat_test_list, tes_link) {
 948                         if (testidx-- == 1)
 949                                 break;
 950                 }
 951
 952                 if (testidx > 0) {
 953                         CDEBUG(D_NET, "Can't find specified test in batch\n");
 954                         return -ENOENT;
 955                 }
 956         }
 957
 958         clilst = (test == NULL) ? &bat->bat_cli_list :
 959                                   &test->tes_src_grp->grp_ndl_list;
 960         srvlst = (test == NULL) ? &bat->bat_srv_list :
 961                                   &test->tes_dst_grp->grp_ndl_list;
 962
 963         if (dents_up != NULL) {
 964                 rc = lstcon_nodes_getent((server ? srvlst: clilst),
 965                                          index_p, ndent_p, dents_up);
 966                 return rc;
 967         }
 968
 969         /* non-verbose query */
 970         LIBCFS_ALLOC(entp, sizeof(lstcon_test_batch_ent_t));
 971         if (entp == NULL)
 972                 return -ENOMEM;
 973
 974         memset(entp, 0, sizeof(lstcon_test_batch_ent_t));
 975
 976         if (test == NULL) {
 977                 entp->u.tbe_batch.bae_ntest = bat->bat_ntest;
 978                 entp->u.tbe_batch.bae_state = bat->bat_state;
 979
 980         } else {
 981
 982                 entp->u.tbe_test.tse_type   = test->tes_type;
 983                 entp->u.tbe_test.tse_loop   = test->tes_loop;
 984                 entp->u.tbe_test.tse_concur = test->tes_concur;
 985         }
 986
 987         list_for_each_entry(ndl, clilst, ndl_link)
 988                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_cli_nle);
 989
 990         list_for_each_entry(ndl, srvlst, ndl_link)
 991                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_srv_nle);
 992
 993         rc = copy_to_user(ent_up, entp,
 994                               sizeof(lstcon_test_batch_ent_t)) ? -EFAULT : 0;
 995
 996         LIBCFS_FREE(entp, sizeof(lstcon_test_batch_ent_t));
 997
 998         return rc;
 999 }
1000
1001 int
1002 lstcon_batrpc_condition(int transop, lstcon_node_t *nd, void *arg)
1003 {
1004         switch (transop) {
1005         case LST_TRANS_TSBRUN:
1006                 if (nd->nd_state != LST_NODE_ACTIVE)
1007                         return -ENETDOWN;
1008                 break;
1009
1010         case LST_TRANS_TSBSTOP:
1011                 if (nd->nd_state != LST_NODE_ACTIVE)
1012                         return 0;
1013                 break;
1014
1015         case LST_TRANS_TSBCLIQRY:
1016         case LST_TRANS_TSBSRVQRY:
1017                 break;
1018         }
1019
1020         return 1;
1021 }
1022
1023 static int
1024 lstcon_batch_op(lstcon_batch_t *bat, int transop,
1025                 struct list_head *result_up)
1026 {
1027         lstcon_rpc_trans_t *trans;
1028         int              rc;
1029
1030         rc = lstcon_rpc_trans_ndlist(&bat->bat_cli_list,
1031                                      &bat->bat_trans_list, transop,
1032                                      bat, lstcon_batrpc_condition, &trans);
1033         if (rc != 0) {
1034                 CERROR("Can't create transaction: %d\n", rc);
1035                 return rc;
1036         }
1037
1038         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1039
1040         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1041
1042         lstcon_rpc_trans_destroy(trans);
1043
1044         return rc;
1045 }
1046
1047 int
1048 lstcon_batch_run(char *name, int timeout, struct list_head *result_up)
1049 {
1050         lstcon_batch_t *bat;
1051         int          rc;
1052
1053         if (lstcon_batch_find(name, &bat) != 0) {
1054                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1055                 return -ENOENT;
1056         }
1057
1058         bat->bat_arg = timeout;
1059
1060         rc = lstcon_batch_op(bat, LST_TRANS_TSBRUN, result_up);
1061
1062         /* mark batch as running if it's started in any node */
1063         if (lstcon_tsbop_stat_success(lstcon_trans_stat(), 0) != 0)
1064                 bat->bat_state = LST_BATCH_RUNNING;
1065
1066         return rc;
1067 }
1068
1069 int
1070 lstcon_batch_stop(char *name, int force, struct list_head *result_up)
1071 {
1072         lstcon_batch_t *bat;
1073         int          rc;
1074
1075         if (lstcon_batch_find(name, &bat) != 0) {
1076                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1077                 return -ENOENT;
1078         }
1079
1080         bat->bat_arg = force;
1081
1082         rc = lstcon_batch_op(bat, LST_TRANS_TSBSTOP, result_up);
1083
1084         /* mark batch as stopped if all RPCs finished */
1085         if (lstcon_tsbop_stat_failure(lstcon_trans_stat(), 0) == 0)
1086                 bat->bat_state = LST_BATCH_IDLE;
1087
1088         return rc;
1089 }
1090
1091 static void
1092 lstcon_batch_destroy(lstcon_batch_t *bat)
1093 {
1094         lstcon_ndlink_t    *ndl;
1095         lstcon_test_t      *test;
1096         int              i;
1097
1098         list_del(&bat->bat_link);
1099
1100         while (!list_empty(&bat->bat_test_list)) {
1101                 test = list_entry(bat->bat_test_list.next,
1102                                       lstcon_test_t, tes_link);
1103                 LASSERT (list_empty(&test->tes_trans_list));
1104
1105                 list_del(&test->tes_link);
1106
1107                 lstcon_group_put(test->tes_src_grp);
1108                 lstcon_group_put(test->tes_dst_grp);
1109
1110                 LIBCFS_FREE(test, offsetof(lstcon_test_t,
1111                                            tes_param[test->tes_paramlen]));
1112         }
1113
1114         LASSERT (list_empty(&bat->bat_trans_list));
1115
1116         while (!list_empty(&bat->bat_cli_list)) {
1117                 ndl = list_entry(bat->bat_cli_list.next,
1118                                      lstcon_ndlink_t, ndl_link);
1119                 list_del_init(&ndl->ndl_link);
1120
1121                 lstcon_ndlink_release(ndl);
1122         }
1123
1124         while (!list_empty(&bat->bat_srv_list)) {
1125                 ndl = list_entry(bat->bat_srv_list.next,
1126                                      lstcon_ndlink_t, ndl_link);
1127                 list_del_init(&ndl->ndl_link);
1128
1129                 lstcon_ndlink_release(ndl);
1130         }
1131
1132         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
1133                 LASSERT (list_empty(&bat->bat_cli_hash[i]));
1134                 LASSERT (list_empty(&bat->bat_srv_hash[i]));
1135         }
1136
1137         LIBCFS_FREE(bat->bat_cli_hash,
1138                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1139         LIBCFS_FREE(bat->bat_srv_hash,
1140                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1141         LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
1142 }
1143
1144 int
1145 lstcon_testrpc_condition(int transop, lstcon_node_t *nd, void *arg)
1146 {
1147         lstcon_test_t    *test;
1148         lstcon_batch_t   *batch;
1149         lstcon_ndlink_t  *ndl;
1150         struct list_head       *hash;
1151         struct list_head       *head;
1152
1153         test = (lstcon_test_t *)arg;
1154         LASSERT (test != NULL);
1155
1156         batch = test->tes_batch;
1157         LASSERT (batch != NULL);
1158
1159         if (test->tes_oneside &&
1160             transop == LST_TRANS_TSBSRVADD)
1161                 return 0;
1162
1163         if (nd->nd_state != LST_NODE_ACTIVE)
1164                 return -ENETDOWN;
1165
1166         if (transop == LST_TRANS_TSBCLIADD) {
1167                 hash = batch->bat_cli_hash;
1168                 head = &batch->bat_cli_list;
1169
1170         } else {
1171                 LASSERT (transop == LST_TRANS_TSBSRVADD);
1172
1173                 hash = batch->bat_srv_hash;
1174                 head = &batch->bat_srv_list;
1175         }
1176
1177         LASSERT (nd->nd_id.nid != LNET_NID_ANY);
1178
1179         if (lstcon_ndlink_find(hash, nd->nd_id, &ndl, 1) != 0)
1180                 return -ENOMEM;
1181
1182         if (list_empty(&ndl->ndl_link))
1183                 list_add_tail(&ndl->ndl_link, head);
1184
1185         return 1;
1186 }
1187
1188 static int
1189 lstcon_test_nodes_add(lstcon_test_t *test, struct list_head *result_up)
1190 {
1191         lstcon_rpc_trans_t     *trans;
1192         lstcon_group_t   *grp;
1193         int                  transop;
1194         int                  rc;
1195
1196         LASSERT (test->tes_src_grp != NULL);
1197         LASSERT (test->tes_dst_grp != NULL);
1198
1199         transop = LST_TRANS_TSBSRVADD;
1200         grp  = test->tes_dst_grp;
1201 again:
1202         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
1203                                      &test->tes_trans_list, transop,
1204                                      test, lstcon_testrpc_condition, &trans);
1205         if (rc != 0) {
1206                 CERROR("Can't create transaction: %d\n", rc);
1207                 return rc;
1208         }
1209
1210         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1211
1212         if (lstcon_trans_stat()->trs_rpc_errno != 0 ||
1213             lstcon_trans_stat()->trs_fwk_errno != 0) {
1214                 lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1215
1216                 lstcon_rpc_trans_destroy(trans);
1217                 /* return if any error */
1218                 CDEBUG(D_NET, "Failed to add test %s, "
1219                               "RPC error %d, framework error %d\n",
1220                        transop == LST_TRANS_TSBCLIADD ? "client" : "server",
1221                        lstcon_trans_stat()->trs_rpc_errno,
1222                        lstcon_trans_stat()->trs_fwk_errno);
1223
1224                 return rc;
1225         }
1226
1227         lstcon_rpc_trans_destroy(trans);
1228
1229         if (transop == LST_TRANS_TSBCLIADD)
1230                 return rc;
1231
1232         transop = LST_TRANS_TSBCLIADD;
1233         grp = test->tes_src_grp;
1234         test->tes_cliidx = 0;
1235
1236         /* requests to test clients */
1237         goto again;
1238 }
1239
1240 static int
1241 lstcon_verify_batch(const char *name, lstcon_batch_t **batch)
1242 {
1243         int rc;
1244
1245         rc = lstcon_batch_find(name, batch);
1246         if (rc != 0) {
1247                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1248                 return rc;
1249         }
1250
1251         if ((*batch)->bat_state != LST_BATCH_IDLE) {
1252                 CDEBUG(D_NET, "Can't change running batch %s\n", name);
1253                 return -EINVAL;
1254         }
1255
1256         return 0;
1257 }
1258
1259 static int
1260 lstcon_verify_group(const char *name, lstcon_group_t **grp)
1261 {
1262         int                     rc;
1263         lstcon_ndlink_t         *ndl;
1264
1265         rc = lstcon_group_find(name, grp);
1266         if (rc != 0) {
1267                 CDEBUG(D_NET, "can't find group %s\n", name);
1268                 return rc;
1269         }
1270
1271         list_for_each_entry(ndl, &(*grp)->grp_ndl_list, ndl_link) {
1272                 if (ndl->ndl_node->nd_state == LST_NODE_ACTIVE)
1273                         return 0;
1274         }
1275
1276         CDEBUG(D_NET, "Group %s has no ACTIVE nodes\n", name);
1277
1278         return -EINVAL;
1279 }
1280
1281 int
1282 lstcon_test_add(char *batch_name, int type, int loop,
1283                 int concur, int dist, int span,
1284                 char *src_name, char *dst_name,
1285                 void *param, int paramlen, int *retp,
1286                 struct list_head *result_up)
1287 {
1288         lstcon_test_t    *test   = NULL;
1289         int              rc;
1290         lstcon_group_t   *src_grp = NULL;
1291         lstcon_group_t   *dst_grp = NULL;
1292         lstcon_batch_t   *batch = NULL;
1293
1294         /*
1295          * verify that a batch of the given name exists, and the groups
1296          * that will be part of the batch exist and have at least one
1297          * active node
1298          */
1299         rc = lstcon_verify_batch(batch_name, &batch);
1300         if (rc != 0)
1301                 goto out;
1302
1303         rc = lstcon_verify_group(src_name, &src_grp);
1304         if (rc != 0)
1305                 goto out;
1306
1307         rc = lstcon_verify_group(dst_name, &dst_grp);
1308         if (rc != 0)
1309                 goto out;
1310
1311         if (dst_grp->grp_userland)
1312                 *retp = 1;
1313
1314         LIBCFS_ALLOC(test, offsetof(lstcon_test_t, tes_param[paramlen]));
1315         if (!test) {
1316                 CERROR("Can't allocate test descriptor\n");
1317                 rc = -ENOMEM;
1318
1319                 goto out;
1320         }
1321
1322         memset(test, 0, offsetof(lstcon_test_t, tes_param[paramlen]));
1323         test->tes_hdr.tsb_id    = batch->bat_hdr.tsb_id;
1324         test->tes_batch         = batch;
1325         test->tes_type          = type;
1326         test->tes_oneside       = 0; /* TODO */
1327         test->tes_loop          = loop;
1328         test->tes_concur        = concur;
1329         test->tes_stop_onerr    = 1; /* TODO */
1330         test->tes_span          = span;
1331         test->tes_dist          = dist;
1332         test->tes_cliidx        = 0; /* just used for creating RPC */
1333         test->tes_src_grp       = src_grp;
1334         test->tes_dst_grp       = dst_grp;
1335         INIT_LIST_HEAD(&test->tes_trans_list);
1336
1337         if (param != NULL) {
1338                 test->tes_paramlen = paramlen;
1339                 memcpy(&test->tes_param[0], param, paramlen);
1340         }
1341
1342         rc = lstcon_test_nodes_add(test, result_up);
1343
1344         if (rc != 0)
1345                 goto out;
1346
1347         if (lstcon_trans_stat()->trs_rpc_errno != 0 ||
1348             lstcon_trans_stat()->trs_fwk_errno != 0)
1349                 CDEBUG(D_NET, "Failed to add test %d to batch %s\n", type,
1350                        batch_name);
1351
1352         /* add to test list anyway, so user can check what's going on */
1353         list_add_tail(&test->tes_link, &batch->bat_test_list);
1354
1355         batch->bat_ntest ++;
1356         test->tes_hdr.tsb_index = batch->bat_ntest;
1357
1358         /*  hold groups so nobody can change them */
1359         return rc;
1360 out:
1361         if (test != NULL)
1362                 LIBCFS_FREE(test, offsetof(lstcon_test_t, tes_param[paramlen]));
1363
1364         if (dst_grp != NULL)
1365                 lstcon_group_put(dst_grp);
1366
1367         if (src_grp != NULL)
1368                 lstcon_group_put(src_grp);
1369
1370         return rc;
1371 }
1372
1373 int
1374 lstcon_test_find(lstcon_batch_t *batch, int idx, lstcon_test_t **testpp)
1375 {
1376         lstcon_test_t *test;
1377
1378         list_for_each_entry(test, &batch->bat_test_list, tes_link) {
1379                 if (idx == test->tes_hdr.tsb_index) {
1380                         *testpp = test;
1381                         return 0;
1382                 }
1383         }
1384
1385         return -ENOENT;
1386 }
1387
1388 int
1389 lstcon_tsbrpc_readent(int transop, srpc_msg_t *msg,
1390                       lstcon_rpc_ent_t *ent_up)
1391 {
1392         srpc_batch_reply_t *rep = &msg->msg_body.bat_reply;
1393
1394         LASSERT (transop == LST_TRANS_TSBCLIQRY ||
1395                  transop == LST_TRANS_TSBSRVQRY);
1396
1397         /* positive errno, framework error code */
1398         if (copy_to_user(&ent_up->rpe_priv[0],
1399                              &rep->bar_active, sizeof(rep->bar_active)))
1400                 return -EFAULT;
1401
1402         return 0;
1403 }
1404
1405 int
1406 lstcon_test_batch_query(char *name, int testidx, int client,
1407                         int timeout, struct list_head *result_up)
1408 {
1409         lstcon_rpc_trans_t *trans;
1410         struct list_head         *translist;
1411         struct list_head         *ndlist;
1412         lstcon_tsb_hdr_t   *hdr;
1413         lstcon_batch_t     *batch;
1414         lstcon_test_t      *test = NULL;
1415         int              transop;
1416         int              rc;
1417
1418         rc = lstcon_batch_find(name, &batch);
1419         if (rc != 0) {
1420                 CDEBUG(D_NET, "Can't find batch: %s\n", name);
1421                 return rc;
1422         }
1423
1424         if (testidx == 0) {
1425                 translist = &batch->bat_trans_list;
1426                 ndlist    = &batch->bat_cli_list;
1427                 hdr       = &batch->bat_hdr;
1428
1429         } else {
1430                 /* query specified test only */
1431                 rc = lstcon_test_find(batch, testidx, &test);
1432                 if (rc != 0) {
1433                         CDEBUG(D_NET, "Can't find test: %d\n", testidx);
1434                         return rc;
1435                 }
1436
1437                 translist = &test->tes_trans_list;
1438                 ndlist    = &test->tes_src_grp->grp_ndl_list;
1439                 hdr       = &test->tes_hdr;
1440         }
1441
1442         transop = client ? LST_TRANS_TSBCLIQRY : LST_TRANS_TSBSRVQRY;
1443
1444         rc = lstcon_rpc_trans_ndlist(ndlist, translist, transop, hdr,
1445                                      lstcon_batrpc_condition, &trans);
1446         if (rc != 0) {
1447                 CERROR("Can't create transaction: %d\n", rc);
1448                 return rc;
1449         }
1450
1451         lstcon_rpc_trans_postwait(trans, timeout);
1452
1453         if (testidx == 0 && /* query a batch, not a test */
1454             lstcon_rpc_stat_failure(lstcon_trans_stat(), 0) == 0 &&
1455             lstcon_tsbqry_stat_run(lstcon_trans_stat(), 0) == 0) {
1456                 /* all RPCs finished, and no active test */
1457                 batch->bat_state = LST_BATCH_IDLE;
1458         }
1459
1460         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1461                                           lstcon_tsbrpc_readent);
1462         lstcon_rpc_trans_destroy(trans);
1463
1464         return rc;
1465 }
1466
1467 int
1468 lstcon_statrpc_readent(int transop, srpc_msg_t *msg,
1469                        lstcon_rpc_ent_t *ent_up)
1470 {
1471         srpc_stat_reply_t *rep = &msg->msg_body.stat_reply;
1472         sfw_counters_t    *sfwk_stat;
1473         srpc_counters_t   *srpc_stat;
1474         lnet_counters_t   *lnet_stat;
1475
1476         if (rep->str_status != 0)
1477                 return 0;
1478
1479         sfwk_stat = (sfw_counters_t *)&ent_up->rpe_payload[0];
1480         srpc_stat = (srpc_counters_t *)((char *)sfwk_stat + sizeof(*sfwk_stat));
1481         lnet_stat = (lnet_counters_t *)((char *)srpc_stat + sizeof(*srpc_stat));
1482
1483         if (copy_to_user(sfwk_stat, &rep->str_fw, sizeof(*sfwk_stat)) ||
1484             copy_to_user(srpc_stat, &rep->str_rpc, sizeof(*srpc_stat)) ||
1485             copy_to_user(lnet_stat, &rep->str_lnet, sizeof(*lnet_stat)))
1486                 return -EFAULT;
1487
1488         return 0;
1489 }
1490
1491 int
1492 lstcon_ndlist_stat(struct list_head *ndlist,
1493                    int timeout, struct list_head *result_up)
1494 {
1495         struct list_head          head;
1496         lstcon_rpc_trans_t *trans;
1497         int              rc;
1498
1499         INIT_LIST_HEAD(&head);
1500
1501         rc = lstcon_rpc_trans_ndlist(ndlist, &head,
1502                                      LST_TRANS_STATQRY, NULL, NULL, &trans);
1503         if (rc != 0) {
1504                 CERROR("Can't create transaction: %d\n", rc);
1505                 return rc;
1506         }
1507
1508         lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout));
1509
1510         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1511                                           lstcon_statrpc_readent);
1512         lstcon_rpc_trans_destroy(trans);
1513
1514         return rc;
1515 }
1516
1517 int
1518 lstcon_group_stat(char *grp_name, int timeout, struct list_head *result_up)
1519 {
1520         lstcon_group_t     *grp;
1521         int              rc;
1522
1523         rc = lstcon_group_find(grp_name, &grp);
1524         if (rc != 0) {
1525                 CDEBUG(D_NET, "Can't find group %s\n", grp_name);
1526                 return rc;
1527         }
1528
1529         rc = lstcon_ndlist_stat(&grp->grp_ndl_list, timeout, result_up);
1530
1531         lstcon_group_put(grp);
1532
1533         return rc;
1534 }
1535
1536 int
1537 lstcon_nodes_stat(int count, lnet_process_id_t *ids_up,
1538                   int timeout, struct list_head *result_up)
1539 {
1540         lstcon_ndlink_t  *ndl;
1541         lstcon_group_t    *tmp;
1542         lnet_process_id_t       id;
1543         int                   i;
1544         int                   rc;
1545
1546         rc = lstcon_group_alloc(NULL, &tmp);
1547         if (rc != 0) {
1548                 CERROR("Out of memory\n");
1549                 return -ENOMEM;
1550         }
1551
1552         for (i = 0 ; i < count; i++) {
1553                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1554                         rc = -EFAULT;
1555                         break;
1556                 }
1557
1558                 /* add to tmp group */
1559                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 2);
1560                 if (rc != 0) {
1561                         CDEBUG((rc == -ENOMEM) ? D_ERROR : D_NET,
1562                                "Failed to find or create %s: %d\n",
1563                                libcfs_id2str(id), rc);
1564                         break;
1565                 }
1566         }
1567
1568         if (rc != 0) {
1569                 lstcon_group_put(tmp);
1570                 return rc;
1571         }
1572
1573         rc = lstcon_ndlist_stat(&tmp->grp_ndl_list, timeout, result_up);
1574
1575         lstcon_group_put(tmp);
1576
1577         return rc;
1578 }
1579
1580 int
1581 lstcon_debug_ndlist(struct list_head *ndlist,
1582                     struct list_head *translist,
1583                     int timeout, struct list_head *result_up)
1584 {
1585         lstcon_rpc_trans_t *trans;
1586         int              rc;
1587
1588         rc = lstcon_rpc_trans_ndlist(ndlist, translist, LST_TRANS_SESQRY,
1589                                      NULL, lstcon_sesrpc_condition, &trans);
1590         if (rc != 0) {
1591                 CERROR("Can't create transaction: %d\n", rc);
1592                 return rc;
1593         }
1594
1595         lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout));
1596
1597         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1598                                           lstcon_sesrpc_readent);
1599         lstcon_rpc_trans_destroy(trans);
1600
1601         return rc;
1602 }
1603
1604 int
1605 lstcon_session_debug(int timeout, struct list_head *result_up)
1606 {
1607         return lstcon_debug_ndlist(&console_session.ses_ndl_list,
1608                                    NULL, timeout, result_up);
1609 }
1610
1611 int
1612 lstcon_batch_debug(int timeout, char *name,
1613                    int client, struct list_head *result_up)
1614 {
1615         lstcon_batch_t *bat;
1616         int          rc;
1617
1618         rc = lstcon_batch_find(name, &bat);
1619         if (rc != 0)
1620                 return -ENOENT;
1621
1622         rc = lstcon_debug_ndlist(client ? &bat->bat_cli_list :
1623                                           &bat->bat_srv_list,
1624                                  NULL, timeout, result_up);
1625
1626         return rc;
1627 }
1628
1629 int
1630 lstcon_group_debug(int timeout, char *name,
1631                    struct list_head *result_up)
1632 {
1633         lstcon_group_t *grp;
1634         int          rc;
1635
1636         rc = lstcon_group_find(name, &grp);
1637         if (rc != 0)
1638                 return -ENOENT;
1639
1640         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1641                                  timeout, result_up);
1642         lstcon_group_put(grp);
1643
1644         return rc;
1645 }
1646
1647 int
1648 lstcon_nodes_debug(int timeout,
1649                    int count, lnet_process_id_t *ids_up,
1650                    struct list_head *result_up)
1651 {
1652         lnet_process_id_t  id;
1653         lstcon_ndlink_t   *ndl;
1654         lstcon_group_t    *grp;
1655         int             i;
1656         int             rc;
1657
1658         rc = lstcon_group_alloc(NULL, &grp);
1659         if (rc != 0) {
1660                 CDEBUG(D_NET, "Out of memory\n");
1661                 return rc;
1662         }
1663
1664         for (i = 0; i < count; i++) {
1665                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1666                         rc = -EFAULT;
1667                         break;
1668                 }
1669
1670                 /* node is added to tmp group */
1671                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 1);
1672                 if (rc != 0) {
1673                         CERROR("Can't create node link\n");
1674                         break;
1675                 }
1676         }
1677
1678         if (rc != 0) {
1679                 lstcon_group_put(grp);
1680                 return rc;
1681         }
1682
1683         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1684                                  timeout, result_up);
1685
1686         lstcon_group_put(grp);
1687
1688         return rc;
1689 }
1690
1691 int
1692 lstcon_session_match(lst_sid_t sid)
1693 {
1694         return (console_session.ses_id.ses_nid   == sid.ses_nid &&
1695                 console_session.ses_id.ses_stamp == sid.ses_stamp) ?  1: 0;
1696 }
1697
1698 static void
1699 lstcon_new_session_id(lst_sid_t *sid)
1700 {
1701         lnet_process_id_t      id;
1702
1703         LASSERT (console_session.ses_state == LST_SESSION_NONE);
1704
1705         LNetGetId(1, &id);
1706         sid->ses_nid   = id.nid;
1707         sid->ses_stamp = cfs_time_current();
1708 }
1709
1710 extern srpc_service_t lstcon_acceptor_service;
1711
1712 int
1713 lstcon_session_new(char *name, int key, unsigned feats,
1714                    int timeout, int force, lst_sid_t *sid_up)
1715 {
1716         int     rc = 0;
1717         int     i;
1718
1719         if (console_session.ses_state != LST_SESSION_NONE) {
1720                 /* session exists */
1721                 if (!force) {
1722                         CNETERR("Session %s already exists\n",
1723                                 console_session.ses_name);
1724                         return -EEXIST;
1725                 }
1726
1727                 rc = lstcon_session_end();
1728
1729                 /* lstcon_session_end() only return local error */
1730                 if  (rc != 0)
1731                         return rc;
1732         }
1733
1734         if ((feats & ~LST_FEATS_MASK) != 0) {
1735                 CNETERR("Unknown session features %x\n",
1736                         (feats & ~LST_FEATS_MASK));
1737                 return -EINVAL;
1738         }
1739
1740         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
1741                 LASSERT(list_empty(&console_session.ses_ndl_hash[i]));
1742
1743         lstcon_new_session_id(&console_session.ses_id);
1744
1745         console_session.ses_key     = key;
1746         console_session.ses_state   = LST_SESSION_ACTIVE;
1747         console_session.ses_force   = !!force;
1748         console_session.ses_features = feats;
1749         console_session.ses_feats_updated = 0;
1750         console_session.ses_timeout = (timeout <= 0) ?
1751                                       LST_CONSOLE_TIMEOUT : timeout;
1752         strcpy(console_session.ses_name, name);
1753
1754         rc = lstcon_batch_add(LST_DEFAULT_BATCH);
1755         if (rc != 0)
1756                 return rc;
1757
1758         rc = lstcon_rpc_pinger_start();
1759         if (rc != 0) {
1760                 lstcon_batch_t *bat = NULL;
1761
1762                 lstcon_batch_find(LST_DEFAULT_BATCH, &bat);
1763                 lstcon_batch_destroy(bat);
1764
1765                 return rc;
1766         }
1767
1768         if (copy_to_user(sid_up, &console_session.ses_id,
1769                              sizeof(lst_sid_t)) == 0)
1770                 return rc;
1771
1772         lstcon_session_end();
1773
1774         return -EFAULT;
1775 }
1776
1777 int
1778 lstcon_session_info(lst_sid_t *sid_up, int *key_up, unsigned *featp,
1779                     lstcon_ndlist_ent_t *ndinfo_up, char *name_up, int len)
1780 {
1781         lstcon_ndlist_ent_t *entp;
1782         lstcon_ndlink_t     *ndl;
1783         int               rc = 0;
1784
1785         if (console_session.ses_state != LST_SESSION_ACTIVE)
1786                 return -ESRCH;
1787
1788         LIBCFS_ALLOC(entp, sizeof(*entp));
1789         if (entp == NULL)
1790                 return -ENOMEM;
1791
1792         memset(entp, 0, sizeof(*entp));
1793
1794         list_for_each_entry(ndl, &console_session.ses_ndl_list, ndl_link)
1795                 LST_NODE_STATE_COUNTER(ndl->ndl_node, entp);
1796
1797         if (copy_to_user(sid_up, &console_session.ses_id,
1798                              sizeof(lst_sid_t)) ||
1799             copy_to_user(key_up, &console_session.ses_key,
1800                              sizeof(*key_up)) ||
1801             copy_to_user(featp, &console_session.ses_features,
1802                              sizeof(*featp)) ||
1803             copy_to_user(ndinfo_up, entp, sizeof(*entp)) ||
1804             copy_to_user(name_up, console_session.ses_name, len))
1805                 rc = -EFAULT;
1806
1807         LIBCFS_FREE(entp, sizeof(*entp));
1808
1809         return rc;
1810 }
1811
1812 int
1813 lstcon_session_end(void)
1814 {
1815         lstcon_rpc_trans_t *trans;
1816         lstcon_group_t     *grp;
1817         lstcon_batch_t     *bat;
1818         int              rc = 0;
1819
1820         LASSERT (console_session.ses_state == LST_SESSION_ACTIVE);
1821
1822         rc = lstcon_rpc_trans_ndlist(&console_session.ses_ndl_list,
1823                                      NULL, LST_TRANS_SESEND, NULL,
1824                                      lstcon_sesrpc_condition, &trans);
1825         if (rc != 0) {
1826                 CERROR("Can't create transaction: %d\n", rc);
1827                 return rc;
1828         }
1829
1830         console_session.ses_shutdown = 1;
1831
1832         lstcon_rpc_pinger_stop();
1833
1834         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1835
1836         lstcon_rpc_trans_destroy(trans);
1837         /* User can do nothing even rpc failed, so go on */
1838
1839         /* waiting for orphan rpcs to die */
1840         lstcon_rpc_cleanup_wait();
1841
1842         console_session.ses_id    = LST_INVALID_SID;
1843         console_session.ses_state = LST_SESSION_NONE;
1844         console_session.ses_key   = 0;
1845         console_session.ses_force = 0;
1846         console_session.ses_feats_updated = 0;
1847
1848         /* destroy all batches */
1849         while (!list_empty(&console_session.ses_bat_list)) {
1850                 bat = list_entry(console_session.ses_bat_list.next,
1851                                      lstcon_batch_t, bat_link);
1852
1853                 lstcon_batch_destroy(bat);
1854         }
1855
1856         /* destroy all groups */
1857         while (!list_empty(&console_session.ses_grp_list)) {
1858                 grp = list_entry(console_session.ses_grp_list.next,
1859                                      lstcon_group_t, grp_link);
1860                 LASSERT (grp->grp_ref == 1);
1861
1862                 lstcon_group_put(grp);
1863         }
1864
1865         /* all nodes should be released */
1866         LASSERT (list_empty(&console_session.ses_ndl_list));
1867
1868         console_session.ses_shutdown = 0;
1869         console_session.ses_expired  = 0;
1870
1871         return rc;
1872 }
1873
1874 int
1875 lstcon_session_feats_check(unsigned feats)
1876 {
1877         int rc = 0;
1878
1879         if ((feats & ~LST_FEATS_MASK) != 0) {
1880                 CERROR("Can't support these features: %x\n",
1881                        (feats & ~LST_FEATS_MASK));
1882                 return -EPROTO;
1883         }
1884
1885         spin_lock(&console_session.ses_rpc_lock);
1886
1887         if (!console_session.ses_feats_updated) {
1888                 console_session.ses_feats_updated = 1;
1889                 console_session.ses_features = feats;
1890         }
1891
1892         if (console_session.ses_features != feats)
1893                 rc = -EPROTO;
1894
1895         spin_unlock(&console_session.ses_rpc_lock);
1896
1897         if (rc != 0) {
1898                 CERROR("remote features %x do not match with "
1899                        "session features %x of console\n",
1900                        feats, console_session.ses_features);
1901         }
1902
1903         return rc;
1904 }
1905
1906 static int
1907 lstcon_acceptor_handle (srpc_server_rpc_t *rpc)
1908 {
1909         srpc_msg_t      *rep  = &rpc->srpc_replymsg;
1910         srpc_msg_t      *req  = &rpc->srpc_reqstbuf->buf_msg;
1911         srpc_join_reqst_t *jreq = &req->msg_body.join_reqst;
1912         srpc_join_reply_t *jrep = &rep->msg_body.join_reply;
1913         lstcon_group_t    *grp  = NULL;
1914         lstcon_ndlink_t   *ndl;
1915         int             rc   = 0;
1916
1917         sfw_unpack_message(req);
1918
1919         mutex_lock(&console_session.ses_mutex);
1920
1921         jrep->join_sid = console_session.ses_id;
1922
1923         if (console_session.ses_id.ses_nid == LNET_NID_ANY) {
1924                 jrep->join_status = ESRCH;
1925                 goto out;
1926         }
1927
1928         if (lstcon_session_feats_check(req->msg_ses_feats) != 0) {
1929                 jrep->join_status = EPROTO;
1930                 goto out;
1931         }
1932
1933         if (jreq->join_sid.ses_nid != LNET_NID_ANY &&
1934              !lstcon_session_match(jreq->join_sid)) {
1935                 jrep->join_status = EBUSY;
1936                 goto out;
1937         }
1938
1939         if (lstcon_group_find(jreq->join_group, &grp) != 0) {
1940                 rc = lstcon_group_alloc(jreq->join_group, &grp);
1941                 if (rc != 0) {
1942                         CERROR("Out of memory\n");
1943                         goto out;
1944                 }
1945
1946                 list_add_tail(&grp->grp_link,
1947                                   &console_session.ses_grp_list);
1948                 lstcon_group_addref(grp);
1949         }
1950
1951         if (grp->grp_ref > 2) {
1952                 /* Group in using */
1953                 jrep->join_status = EBUSY;
1954                 goto out;
1955         }
1956
1957         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 0);
1958         if (rc == 0) {
1959                 jrep->join_status = EEXIST;
1960                 goto out;
1961         }
1962
1963         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 1);
1964         if (rc != 0) {
1965                 CERROR("Out of memory\n");
1966                 goto out;
1967         }
1968
1969         ndl->ndl_node->nd_state   = LST_NODE_ACTIVE;
1970         ndl->ndl_node->nd_timeout = console_session.ses_timeout;
1971
1972         if (grp->grp_userland == 0)
1973                 grp->grp_userland = 1;
1974
1975         strcpy(jrep->join_session, console_session.ses_name);
1976         jrep->join_timeout = console_session.ses_timeout;
1977         jrep->join_status  = 0;
1978
1979 out:
1980         rep->msg_ses_feats = console_session.ses_features;
1981         if (grp != NULL)
1982                 lstcon_group_put(grp);
1983
1984         mutex_unlock(&console_session.ses_mutex);
1985
1986         return rc;
1987 }
1988
1989 srpc_service_t lstcon_acceptor_service;
1990 void lstcon_init_acceptor_service(void)
1991 {
1992         /* initialize selftest console acceptor service table */
1993         lstcon_acceptor_service.sv_name    = "join session";
1994         lstcon_acceptor_service.sv_handler = lstcon_acceptor_handle;
1995         lstcon_acceptor_service.sv_id      = SRPC_SERVICE_JOIN;
1996         lstcon_acceptor_service.sv_wi_total = SFW_FRWK_WI_MAX;
1997 }
1998
1999 extern int lstcon_ioctl_entry(unsigned int cmd, struct libcfs_ioctl_data *data);
2000
2001 DECLARE_IOCTL_HANDLER(lstcon_ioctl_handler, lstcon_ioctl_entry);
2002
2003 /* initialize console */
2004 int
2005 lstcon_console_init(void)
2006 {
2007         int     i;
2008         int     rc;
2009
2010         memset(&console_session, 0, sizeof(lstcon_session_t));
2011
2012         console_session.ses_id              = LST_INVALID_SID;
2013         console_session.ses_state           = LST_SESSION_NONE;
2014         console_session.ses_timeout         = 0;
2015         console_session.ses_force           = 0;
2016         console_session.ses_expired         = 0;
2017         console_session.ses_feats_updated   = 0;
2018         console_session.ses_features        = LST_FEATS_MASK;
2019         console_session.ses_laststamp       = cfs_time_current_sec();
2020
2021         mutex_init(&console_session.ses_mutex);
2022
2023         INIT_LIST_HEAD(&console_session.ses_ndl_list);
2024         INIT_LIST_HEAD(&console_session.ses_grp_list);
2025         INIT_LIST_HEAD(&console_session.ses_bat_list);
2026         INIT_LIST_HEAD(&console_session.ses_trans_list);
2027
2028         LIBCFS_ALLOC(console_session.ses_ndl_hash,
2029                      sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2030         if (console_session.ses_ndl_hash == NULL)
2031                 return -ENOMEM;
2032
2033         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
2034                 INIT_LIST_HEAD(&console_session.ses_ndl_hash[i]);
2035
2036
2037         /* initialize acceptor service table */
2038         lstcon_init_acceptor_service();
2039
2040         rc = srpc_add_service(&lstcon_acceptor_service);
2041         LASSERT (rc != -EBUSY);
2042         if (rc != 0) {
2043                 LIBCFS_FREE(console_session.ses_ndl_hash,
2044                             sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2045                 return rc;
2046         }
2047
2048         rc = srpc_service_add_buffers(&lstcon_acceptor_service,
2049                                       lstcon_acceptor_service.sv_wi_total);
2050         if (rc != 0) {
2051                 rc = -ENOMEM;
2052                 goto out;
2053         }
2054
2055         rc = libcfs_register_ioctl(&lstcon_ioctl_handler);
2056
2057         if (rc == 0) {
2058                 lstcon_rpc_module_init();
2059                 return 0;
2060         }
2061
2062 out:
2063         srpc_shutdown_service(&lstcon_acceptor_service);
2064         srpc_remove_service(&lstcon_acceptor_service);
2065
2066         LIBCFS_FREE(console_session.ses_ndl_hash,
2067                     sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2068
2069         srpc_wait_service_shutdown(&lstcon_acceptor_service);
2070
2071         return rc;
2072 }
2073
2074 int
2075 lstcon_console_fini(void)
2076 {
2077         int     i;
2078
2079         libcfs_deregister_ioctl(&lstcon_ioctl_handler);
2080
2081         mutex_lock(&console_session.ses_mutex);
2082
2083         srpc_shutdown_service(&lstcon_acceptor_service);
2084         srpc_remove_service(&lstcon_acceptor_service);
2085
2086         if (console_session.ses_state != LST_SESSION_NONE)
2087                 lstcon_session_end();
2088
2089         lstcon_rpc_module_fini();
2090
2091         mutex_unlock(&console_session.ses_mutex);
2092
2093         LASSERT (list_empty(&console_session.ses_ndl_list));
2094         LASSERT (list_empty(&console_session.ses_grp_list));
2095         LASSERT (list_empty(&console_session.ses_bat_list));
2096         LASSERT (list_empty(&console_session.ses_trans_list));
2097
2098         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
2099                 LASSERT (list_empty(&console_session.ses_ndl_hash[i]));
2100         }
2101
2102         LIBCFS_FREE(console_session.ses_ndl_hash,
2103                     sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2104
2105         srpc_wait_service_shutdown(&lstcon_acceptor_service);
2106
2107         return 0;
2108 }