kernel/livepatch/core.c

   1 /*
   2  * core.c - Kernel Live Patching Core
   3  *
   4  * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>
   5  * Copyright (C) 2014 SUSE
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License
   9  * as published by the Free Software Foundation; either version 2
  10  * of the License, or (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  19  */
  20
  21 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  22
  23 #include <linux/module.h>
  24 #include <linux/kernel.h>
  25 #include <linux/mutex.h>
  26 #include <linux/slab.h>
  27 #include <linux/list.h>
  28 #include <linux/kallsyms.h>
  29 #include <linux/livepatch.h>
  30 #include <linux/elf.h>
  31 #include <linux/moduleloader.h>
  32 #include <linux/completion.h>
  33 #include <asm/cacheflush.h>
  34 #include "core.h"
  35 #include "patch.h"
  36 #include "transition.h"
  37
  38 /*
  39  * klp_mutex is a coarse lock which serializes access to klp data.  All
  40  * accesses to klp-related variables and structures must have mutex protection,
  41  * except within the following functions which carefully avoid the need for it:
  42  *
  43  * - klp_ftrace_handler()
  44  * - klp_update_patch_state()
  45  */
  46 DEFINE_MUTEX(klp_mutex);
  47
  48 /*
  49  * Actively used patches: enabled or in transition. Note that replaced
  50  * or disabled patches are not listed even though the related kernel
  51  * module still can be loaded.
  52  */
  53 LIST_HEAD(klp_patches);
  54
  55 static struct kobject *klp_root_kobj;
  56
  57 static bool klp_is_module(struct klp_object *obj)
  58 {
  59         return obj->name;
  60 }
  61
  62 /* sets obj->mod if object is not vmlinux and module is found */
  63 static void klp_find_object_module(struct klp_object *obj)
  64 {
  65         struct module *mod;
  66
  67         if (!klp_is_module(obj))
  68                 return;
  69
  70         mutex_lock(&module_mutex);
  71         /*
  72          * We do not want to block removal of patched modules and therefore
  73          * we do not take a reference here. The patches are removed by
  74          * klp_module_going() instead.
  75          */
  76         mod = find_module(obj->name);
  77         /*
  78          * Do not mess work of klp_module_coming() and klp_module_going().
  79          * Note that the patch might still be needed before klp_module_going()
  80          * is called. Module functions can be called even in the GOING state
  81          * until mod->exit() finishes. This is especially important for
  82          * patches that modify semantic of the functions.
  83          */
  84         if (mod && mod->klp_alive)
  85                 obj->mod = mod;
  86
  87         mutex_unlock(&module_mutex);
  88 }
  89
  90 static bool klp_initialized(void)
  91 {
  92         return !!klp_root_kobj;
  93 }
  94
  95 static struct klp_func *klp_find_func(struct klp_object *obj,
  96                                       struct klp_func *old_func)
  97 {
  98         struct klp_func *func;
  99
 100         klp_for_each_func(obj, func) {
 101                 if ((strcmp(old_func->old_name, func->old_name) == 0) &&
 102                     (old_func->old_sympos == func->old_sympos)) {
 103                         return func;
 104                 }
 105         }
 106
 107         return NULL;
 108 }
 109
 110 static struct klp_object *klp_find_object(struct klp_patch *patch,
 111                                           struct klp_object *old_obj)
 112 {
 113         struct klp_object *obj;
 114
 115         klp_for_each_object(patch, obj) {
 116                 if (klp_is_module(old_obj)) {
 117                         if (klp_is_module(obj) &&
 118                             strcmp(old_obj->name, obj->name) == 0) {
 119                                 return obj;
 120                         }
 121                 } else if (!klp_is_module(obj)) {
 122                         return obj;
 123                 }
 124         }
 125
 126         return NULL;
 127 }
 128
 129 struct klp_find_arg {
 130         const char *objname;
 131         const char *name;
 132         unsigned long addr;
 133         unsigned long count;
 134         unsigned long pos;
 135 };
 136
 137 static int klp_find_callback(void *data, const char *name,
 138                              struct module *mod, unsigned long addr)
 139 {
 140         struct klp_find_arg *args = data;
 141
 142         if ((mod && !args->objname) || (!mod && args->objname))
 143                 return 0;
 144
 145         if (strcmp(args->name, name))
 146                 return 0;
 147
 148         if (args->objname && strcmp(args->objname, mod->name))
 149                 return 0;
 150
 151         args->addr = addr;
 152         args->count++;
 153
 154         /*
 155          * Finish the search when the symbol is found for the desired position
 156          * or the position is not defined for a non-unique symbol.
 157          */
 158         if ((args->pos && (args->count == args->pos)) ||
 159             (!args->pos && (args->count > 1)))
 160                 return 1;
 161
 162         return 0;
 163 }
 164
 165 static int klp_find_object_symbol(const char *objname, const char *name,
 166                                   unsigned long sympos, unsigned long *addr)
 167 {
 168         struct klp_find_arg args = {
 169                 .objname = objname,
 170                 .name = name,
 171                 .addr = 0,
 172                 .count = 0,
 173                 .pos = sympos,
 174         };
 175
 176         mutex_lock(&module_mutex);
 177         if (objname)
 178                 module_kallsyms_on_each_symbol(klp_find_callback, &args);
 179         else
 180                 kallsyms_on_each_symbol(klp_find_callback, &args);
 181         mutex_unlock(&module_mutex);
 182
 183         /*
 184          * Ensure an address was found. If sympos is 0, ensure symbol is unique;
 185          * otherwise ensure the symbol position count matches sympos.
 186          */
 187         if (args.addr == 0)
 188                 pr_err("symbol '%s' not found in symbol table\n", name);
 189         else if (args.count > 1 && sympos == 0) {
 190                 pr_err("unresolvable ambiguity for symbol '%s' in object '%s'\n",
 191                        name, objname);
 192         } else if (sympos != args.count && sympos > 0) {
 193                 pr_err("symbol position %lu for symbol '%s' in object '%s' not found\n",
 194                        sympos, name, objname ? objname : "vmlinux");
 195         } else {
 196                 *addr = args.addr;
 197                 return 0;
 198         }
 199
 200         *addr = 0;
 201         return -EINVAL;
 202 }
 203
 204 static int klp_resolve_symbols(Elf_Shdr *relasec, struct module *pmod)
 205 {
 206         int i, cnt, vmlinux, ret;
 207         char objname[MODULE_NAME_LEN];
 208         char symname[KSYM_NAME_LEN];
 209         char *strtab = pmod->core_kallsyms.strtab;
 210         Elf_Rela *relas;
 211         Elf_Sym *sym;
 212         unsigned long sympos, addr;
 213
 214         /*
 215          * Since the field widths for objname and symname in the sscanf()
 216          * call are hard-coded and correspond to MODULE_NAME_LEN and
 217          * KSYM_NAME_LEN respectively, we must make sure that MODULE_NAME_LEN
 218          * and KSYM_NAME_LEN have the values we expect them to have.
 219          *
 220          * Because the value of MODULE_NAME_LEN can differ among architectures,
 221          * we use the smallest/strictest upper bound possible (56, based on
 222          * the current definition of MODULE_NAME_LEN) to prevent overflows.
 223          */
 224         BUILD_BUG_ON(MODULE_NAME_LEN < 56 || KSYM_NAME_LEN != 128);
 225
 226         relas = (Elf_Rela *) relasec->sh_addr;
 227         /* For each rela in this klp relocation section */
 228         for (i = 0; i < relasec->sh_size / sizeof(Elf_Rela); i++) {
 229                 sym = pmod->core_kallsyms.symtab + ELF_R_SYM(relas[i].r_info);
 230                 if (sym->st_shndx != SHN_LIVEPATCH) {
 231                         pr_err("symbol %s is not marked as a livepatch symbol\n",
 232                                strtab + sym->st_name);
 233                         return -EINVAL;
 234                 }
 235
 236                 /* Format: .klp.sym.objname.symname,sympos */
 237                 cnt = sscanf(strtab + sym->st_name,
 238                              ".klp.sym.%55[^.].%127[^,],%lu",
 239                              objname, symname, &sympos);
 240                 if (cnt != 3) {
 241                         pr_err("symbol %s has an incorrectly formatted name\n",
 242                                strtab + sym->st_name);
 243                         return -EINVAL;
 244                 }
 245
 246                 /* klp_find_object_symbol() treats a NULL objname as vmlinux */
 247                 vmlinux = !strcmp(objname, "vmlinux");
 248                 ret = klp_find_object_symbol(vmlinux ? NULL : objname,
 249                                              symname, sympos, &addr);
 250                 if (ret)
 251                         return ret;
 252
 253                 sym->st_value = addr;
 254         }
 255
 256         return 0;
 257 }
 258
 259 static int klp_write_object_relocations(struct module *pmod,
 260                                         struct klp_object *obj)
 261 {
 262         int i, cnt, ret = 0;
 263         const char *objname, *secname;
 264         char sec_objname[MODULE_NAME_LEN];
 265         Elf_Shdr *sec;
 266
 267         if (WARN_ON(!klp_is_object_loaded(obj)))
 268                 return -EINVAL;
 269
 270         objname = klp_is_module(obj) ? obj->name : "vmlinux";
 271
 272         /* For each klp relocation section */
 273         for (i = 1; i < pmod->klp_info->hdr.e_shnum; i++) {
 274                 sec = pmod->klp_info->sechdrs + i;
 275                 secname = pmod->klp_info->secstrings + sec->sh_name;
 276                 if (!(sec->sh_flags & SHF_RELA_LIVEPATCH))
 277                         continue;
 278
 279                 /*
 280                  * Format: .klp.rela.sec_objname.section_name
 281                  * See comment in klp_resolve_symbols() for an explanation
 282                  * of the selected field width value.
 283                  */
 284                 cnt = sscanf(secname, ".klp.rela.%55[^.]", sec_objname);
 285                 if (cnt != 1) {
 286                         pr_err("section %s has an incorrectly formatted name\n",
 287                                secname);
 288                         ret = -EINVAL;
 289                         break;
 290                 }
 291
 292                 if (strcmp(objname, sec_objname))
 293                         continue;
 294
 295                 ret = klp_resolve_symbols(sec, pmod);
 296                 if (ret)
 297                         break;
 298
 299                 ret = apply_relocate_add(pmod->klp_info->sechdrs,
 300                                          pmod->core_kallsyms.strtab,
 301                                          pmod->klp_info->symndx, i, pmod);
 302                 if (ret)
 303                         break;
 304         }
 305
 306         return ret;
 307 }
 308
 309 /*
 310  * Sysfs Interface
 311  *
 312  * /sys/kernel/livepatch
 313  * /sys/kernel/livepatch/<patch>
 314  * /sys/kernel/livepatch/<patch>/enabled
 315  * /sys/kernel/livepatch/<patch>/transition
 316  * /sys/kernel/livepatch/<patch>/force
 317  * /sys/kernel/livepatch/<patch>/<object>
 318  * /sys/kernel/livepatch/<patch>/<object>/<function,sympos>
 319  */
 320 static int __klp_disable_patch(struct klp_patch *patch);
 321
 322 static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr,
 323                              const char *buf, size_t count)
 324 {
 325         struct klp_patch *patch;
 326         int ret;
 327         bool enabled;
 328
 329         ret = kstrtobool(buf, &enabled);
 330         if (ret)
 331                 return ret;
 332
 333         patch = container_of(kobj, struct klp_patch, kobj);
 334
 335         mutex_lock(&klp_mutex);
 336
 337         if (patch->enabled == enabled) {
 338                 /* already in requested state */
 339                 ret = -EINVAL;
 340                 goto out;
 341         }
 342
 343         /*
 344          * Allow to reverse a pending transition in both ways. It might be
 345          * necessary to complete the transition without forcing and breaking
 346          * the system integrity.
 347          *
 348          * Do not allow to re-enable a disabled patch.
 349          */
 350         if (patch == klp_transition_patch)
 351                 klp_reverse_transition();
 352         else if (!enabled)
 353                 ret = __klp_disable_patch(patch);
 354         else
 355                 ret = -EINVAL;
 356
 357 out:
 358         mutex_unlock(&klp_mutex);
 359
 360         if (ret)
 361                 return ret;
 362         return count;
 363 }
 364
 365 static ssize_t enabled_show(struct kobject *kobj,
 366                             struct kobj_attribute *attr, char *buf)
 367 {
 368         struct klp_patch *patch;
 369
 370         patch = container_of(kobj, struct klp_patch, kobj);
 371         return snprintf(buf, PAGE_SIZE-1, "%d\n", patch->enabled);
 372 }
 373
 374 static ssize_t transition_show(struct kobject *kobj,
 375                                struct kobj_attribute *attr, char *buf)
 376 {
 377         struct klp_patch *patch;
 378
 379         patch = container_of(kobj, struct klp_patch, kobj);
 380         return snprintf(buf, PAGE_SIZE-1, "%d\n",
 381                         patch == klp_transition_patch);
 382 }
 383
 384 static ssize_t force_store(struct kobject *kobj, struct kobj_attribute *attr,
 385                            const char *buf, size_t count)
 386 {
 387         struct klp_patch *patch;
 388         int ret;
 389         bool val;
 390
 391         ret = kstrtobool(buf, &val);
 392         if (ret)
 393                 return ret;
 394
 395         if (!val)
 396                 return count;
 397
 398         mutex_lock(&klp_mutex);
 399
 400         patch = container_of(kobj, struct klp_patch, kobj);
 401         if (patch != klp_transition_patch) {
 402                 mutex_unlock(&klp_mutex);
 403                 return -EINVAL;
 404         }
 405
 406         klp_force_transition();
 407
 408         mutex_unlock(&klp_mutex);
 409
 410         return count;
 411 }
 412
 413 static struct kobj_attribute enabled_kobj_attr = __ATTR_RW(enabled);
 414 static struct kobj_attribute transition_kobj_attr = __ATTR_RO(transition);
 415 static struct kobj_attribute force_kobj_attr = __ATTR_WO(force);
 416 static struct attribute *klp_patch_attrs[] = {
 417         &enabled_kobj_attr.attr,
 418         &transition_kobj_attr.attr,
 419         &force_kobj_attr.attr,
 420         NULL
 421 };
 422
 423 static void klp_free_object_dynamic(struct klp_object *obj)
 424 {
 425         kfree(obj->name);
 426         kfree(obj);
 427 }
 428
 429 static void klp_init_func_early(struct klp_object *obj,
 430                                 struct klp_func *func);
 431 static void klp_init_object_early(struct klp_patch *patch,
 432                                   struct klp_object *obj);
 433
 434 static struct klp_object *klp_alloc_object_dynamic(const char *name,
 435                                                    struct klp_patch *patch)
 436 {
 437         struct klp_object *obj;
 438
 439         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
 440         if (!obj)
 441                 return NULL;
 442
 443         if (name) {
 444                 obj->name = kstrdup(name, GFP_KERNEL);
 445                 if (!obj->name) {
 446                         kfree(obj);
 447                         return NULL;
 448                 }
 449         }
 450
 451         klp_init_object_early(patch, obj);
 452         obj->dynamic = true;
 453
 454         return obj;
 455 }
 456
 457 static void klp_free_func_nop(struct klp_func *func)
 458 {
 459         kfree(func->old_name);
 460         kfree(func);
 461 }
 462
 463 static struct klp_func *klp_alloc_func_nop(struct klp_func *old_func,
 464                                            struct klp_object *obj)
 465 {
 466         struct klp_func *func;
 467
 468         func = kzalloc(sizeof(*func), GFP_KERNEL);
 469         if (!func)
 470                 return NULL;
 471
 472         if (old_func->old_name) {
 473                 func->old_name = kstrdup(old_func->old_name, GFP_KERNEL);
 474                 if (!func->old_name) {
 475                         kfree(func);
 476                         return NULL;
 477                 }
 478         }
 479
 480         klp_init_func_early(obj, func);
 481         /*
 482          * func->new_func is same as func->old_func. These addresses are
 483          * set when the object is loaded, see klp_init_object_loaded().
 484          */
 485         func->old_sympos = old_func->old_sympos;
 486         func->nop = true;
 487
 488         return func;
 489 }
 490
 491 static int klp_add_object_nops(struct klp_patch *patch,
 492                                struct klp_object *old_obj)
 493 {
 494         struct klp_object *obj;
 495         struct klp_func *func, *old_func;
 496
 497         obj = klp_find_object(patch, old_obj);
 498
 499         if (!obj) {
 500                 obj = klp_alloc_object_dynamic(old_obj->name, patch);
 501                 if (!obj)
 502                         return -ENOMEM;
 503         }
 504
 505         klp_for_each_func(old_obj, old_func) {
 506                 func = klp_find_func(obj, old_func);
 507                 if (func)
 508                         continue;
 509
 510                 func = klp_alloc_func_nop(old_func, obj);
 511                 if (!func)
 512                         return -ENOMEM;
 513         }
 514
 515         return 0;
 516 }
 517
 518 /*
 519  * Add 'nop' functions which simply return to the caller to run
 520  * the original function. The 'nop' functions are added to a
 521  * patch to facilitate a 'replace' mode.
 522  */
 523 static int klp_add_nops(struct klp_patch *patch)
 524 {
 525         struct klp_patch *old_patch;
 526         struct klp_object *old_obj;
 527
 528         klp_for_each_patch(old_patch) {
 529                 klp_for_each_object(old_patch, old_obj) {
 530                         int err;
 531
 532                         err = klp_add_object_nops(patch, old_obj);
 533                         if (err)
 534                                 return err;
 535                 }
 536         }
 537
 538         return 0;
 539 }
 540
 541 static void klp_kobj_release_patch(struct kobject *kobj)
 542 {
 543         struct klp_patch *patch;
 544
 545         patch = container_of(kobj, struct klp_patch, kobj);
 546         complete(&patch->finish);
 547 }
 548
 549 static struct kobj_type klp_ktype_patch = {
 550         .release = klp_kobj_release_patch,
 551         .sysfs_ops = &kobj_sysfs_ops,
 552         .default_attrs = klp_patch_attrs,
 553 };
 554
 555 static void klp_kobj_release_object(struct kobject *kobj)
 556 {
 557         struct klp_object *obj;
 558
 559         obj = container_of(kobj, struct klp_object, kobj);
 560
 561         if (obj->dynamic)
 562                 klp_free_object_dynamic(obj);
 563 }
 564
 565 static struct kobj_type klp_ktype_object = {
 566         .release = klp_kobj_release_object,
 567         .sysfs_ops = &kobj_sysfs_ops,
 568 };
 569
 570 static void klp_kobj_release_func(struct kobject *kobj)
 571 {
 572         struct klp_func *func;
 573
 574         func = container_of(kobj, struct klp_func, kobj);
 575
 576         if (func->nop)
 577                 klp_free_func_nop(func);
 578 }
 579
 580 static struct kobj_type klp_ktype_func = {
 581         .release = klp_kobj_release_func,
 582         .sysfs_ops = &kobj_sysfs_ops,
 583 };
 584
 585 static void __klp_free_funcs(struct klp_object *obj, bool nops_only)
 586 {
 587         struct klp_func *func, *tmp_func;
 588
 589         klp_for_each_func_safe(obj, func, tmp_func) {
 590                 if (nops_only && !func->nop)
 591                         continue;
 592
 593                 list_del(&func->node);
 594                 kobject_put(&func->kobj);
 595         }
 596 }
 597
 598 /* Clean up when a patched object is unloaded */
 599 static void klp_free_object_loaded(struct klp_object *obj)
 600 {
 601         struct klp_func *func;
 602
 603         obj->mod = NULL;
 604
 605         klp_for_each_func(obj, func) {
 606                 func->old_func = NULL;
 607
 608                 if (func->nop)
 609                         func->new_func = NULL;
 610         }
 611 }
 612
 613 static void __klp_free_objects(struct klp_patch *patch, bool nops_only)
 614 {
 615         struct klp_object *obj, *tmp_obj;
 616
 617         klp_for_each_object_safe(patch, obj, tmp_obj) {
 618                 __klp_free_funcs(obj, nops_only);
 619
 620                 if (nops_only && !obj->dynamic)
 621                         continue;
 622
 623                 list_del(&obj->node);
 624                 kobject_put(&obj->kobj);
 625         }
 626 }
 627
 628 static void klp_free_objects(struct klp_patch *patch)
 629 {
 630         __klp_free_objects(patch, false);
 631 }
 632
 633 static void klp_free_objects_dynamic(struct klp_patch *patch)
 634 {
 635         __klp_free_objects(patch, true);
 636 }
 637
 638 /*
 639  * This function implements the free operations that can be called safely
 640  * under klp_mutex.
 641  *
 642  * The operation must be completed by calling klp_free_patch_finish()
 643  * outside klp_mutex.
 644  */
 645 void klp_free_patch_start(struct klp_patch *patch)
 646 {
 647         if (!list_empty(&patch->list))
 648                 list_del(&patch->list);
 649
 650         klp_free_objects(patch);
 651 }
 652
 653 /*
 654  * This function implements the free part that must be called outside
 655  * klp_mutex.
 656  *
 657  * It must be called after klp_free_patch_start(). And it has to be
 658  * the last function accessing the livepatch structures when the patch
 659  * gets disabled.
 660  */
 661 static void klp_free_patch_finish(struct klp_patch *patch)
 662 {
 663         /*
 664          * Avoid deadlock with enabled_store() sysfs callback by
 665          * calling this outside klp_mutex. It is safe because
 666          * this is called when the patch gets disabled and it
 667          * cannot get enabled again.
 668          */
 669         kobject_put(&patch->kobj);
 670         wait_for_completion(&patch->finish);
 671
 672         /* Put the module after the last access to struct klp_patch. */
 673         if (!patch->forced)
 674                 module_put(patch->mod);
 675 }
 676
 677 /*
 678  * The livepatch might be freed from sysfs interface created by the patch.
 679  * This work allows to wait until the interface is destroyed in a separate
 680  * context.
 681  */
 682 static void klp_free_patch_work_fn(struct work_struct *work)
 683 {
 684         struct klp_patch *patch =
 685                 container_of(work, struct klp_patch, free_work);
 686
 687         klp_free_patch_finish(patch);
 688 }
 689
 690 static int klp_init_func(struct klp_object *obj, struct klp_func *func)
 691 {
 692         if (!func->old_name)
 693                 return -EINVAL;
 694
 695         /*
 696          * NOPs get the address later. The patched module must be loaded,
 697          * see klp_init_object_loaded().
 698          */
 699         if (!func->new_func && !func->nop)
 700                 return -EINVAL;
 701
 702         if (strlen(func->old_name) >= KSYM_NAME_LEN)
 703                 return -EINVAL;
 704
 705         INIT_LIST_HEAD(&func->stack_node);
 706         func->patched = false;
 707         func->transition = false;
 708
 709         /* The format for the sysfs directory is <function,sympos> where sympos
 710          * is the nth occurrence of this symbol in kallsyms for the patched
 711          * object. If the user selects 0 for old_sympos, then 1 will be used
 712          * since a unique symbol will be the first occurrence.
 713          */
 714         return kobject_add(&func->kobj, &obj->kobj, "%s,%lu",
 715                            func->old_name,
 716                            func->old_sympos ? func->old_sympos : 1);
 717 }
 718
 719 /* Arches may override this to finish any remaining arch-specific tasks */
 720 void __weak arch_klp_init_object_loaded(struct klp_patch *patch,
 721                                         struct klp_object *obj)
 722 {
 723 }
 724
 725 /* parts of the initialization that is done only when the object is loaded */
 726 static int klp_init_object_loaded(struct klp_patch *patch,
 727                                   struct klp_object *obj)
 728 {
 729         struct klp_func *func;
 730         int ret;
 731
 732         module_disable_ro(patch->mod);
 733         ret = klp_write_object_relocations(patch->mod, obj);
 734         if (ret) {
 735                 module_enable_ro(patch->mod, true);
 736                 return ret;
 737         }
 738
 739         arch_klp_init_object_loaded(patch, obj);
 740         module_enable_ro(patch->mod, true);
 741
 742         klp_for_each_func(obj, func) {
 743                 ret = klp_find_object_symbol(obj->name, func->old_name,
 744                                              func->old_sympos,
 745                                              (unsigned long *)&func->old_func);
 746                 if (ret)
 747                         return ret;
 748
 749                 ret = kallsyms_lookup_size_offset((unsigned long)func->old_func,
 750                                                   &func->old_size, NULL);
 751                 if (!ret) {
 752                         pr_err("kallsyms size lookup failed for '%s'\n",
 753                                func->old_name);
 754                         return -ENOENT;
 755                 }
 756
 757                 if (func->nop)
 758                         func->new_func = func->old_func;
 759
 760                 ret = kallsyms_lookup_size_offset((unsigned long)func->new_func,
 761                                                   &func->new_size, NULL);
 762                 if (!ret) {
 763                         pr_err("kallsyms size lookup failed for '%s' replacement\n",
 764                                func->old_name);
 765                         return -ENOENT;
 766                 }
 767         }
 768
 769         return 0;
 770 }
 771
 772 static int klp_init_object(struct klp_patch *patch, struct klp_object *obj)
 773 {
 774         struct klp_func *func;
 775         int ret;
 776         const char *name;
 777
 778         if (klp_is_module(obj) && strlen(obj->name) >= MODULE_NAME_LEN)
 779                 return -EINVAL;
 780
 781         obj->patched = false;
 782         obj->mod = NULL;
 783
 784         klp_find_object_module(obj);
 785
 786         name = klp_is_module(obj) ? obj->name : "vmlinux";
 787         ret = kobject_add(&obj->kobj, &patch->kobj, "%s", name);
 788         if (ret)
 789                 return ret;
 790
 791         klp_for_each_func(obj, func) {
 792                 ret = klp_init_func(obj, func);
 793                 if (ret)
 794                         return ret;
 795         }
 796
 797         if (klp_is_object_loaded(obj))
 798                 ret = klp_init_object_loaded(patch, obj);
 799
 800         return ret;
 801 }
 802
 803 static void klp_init_func_early(struct klp_object *obj,
 804                                 struct klp_func *func)
 805 {
 806         kobject_init(&func->kobj, &klp_ktype_func);
 807         list_add_tail(&func->node, &obj->func_list);
 808 }
 809
 810 static void klp_init_object_early(struct klp_patch *patch,
 811                                   struct klp_object *obj)
 812 {
 813         INIT_LIST_HEAD(&obj->func_list);
 814         kobject_init(&obj->kobj, &klp_ktype_object);
 815         list_add_tail(&obj->node, &patch->obj_list);
 816 }
 817
 818 static int klp_init_patch_early(struct klp_patch *patch)
 819 {
 820         struct klp_object *obj;
 821         struct klp_func *func;
 822
 823         if (!patch->objs)
 824                 return -EINVAL;
 825
 826         INIT_LIST_HEAD(&patch->list);
 827         INIT_LIST_HEAD(&patch->obj_list);
 828         kobject_init(&patch->kobj, &klp_ktype_patch);
 829         patch->enabled = false;
 830         patch->forced = false;
 831         INIT_WORK(&patch->free_work, klp_free_patch_work_fn);
 832         init_completion(&patch->finish);
 833
 834         klp_for_each_object_static(patch, obj) {
 835                 if (!obj->funcs)
 836                         return -EINVAL;
 837
 838                 klp_init_object_early(patch, obj);
 839
 840                 klp_for_each_func_static(obj, func) {
 841                         klp_init_func_early(obj, func);
 842                 }
 843         }
 844
 845         if (!try_module_get(patch->mod))
 846                 return -ENODEV;
 847
 848         return 0;
 849 }
 850
 851 static int klp_init_patch(struct klp_patch *patch)
 852 {
 853         struct klp_object *obj;
 854         int ret;
 855
 856         ret = kobject_add(&patch->kobj, klp_root_kobj, "%s", patch->mod->name);
 857         if (ret)
 858                 return ret;
 859
 860         if (patch->replace) {
 861                 ret = klp_add_nops(patch);
 862                 if (ret)
 863                         return ret;
 864         }
 865
 866         klp_for_each_object(patch, obj) {
 867                 ret = klp_init_object(patch, obj);
 868                 if (ret)
 869                         return ret;
 870         }
 871
 872         list_add_tail(&patch->list, &klp_patches);
 873
 874         return 0;
 875 }
 876
 877 static int __klp_disable_patch(struct klp_patch *patch)
 878 {
 879         struct klp_object *obj;
 880
 881         if (WARN_ON(!patch->enabled))
 882                 return -EINVAL;
 883
 884         if (klp_transition_patch)
 885                 return -EBUSY;
 886
 887         klp_init_transition(patch, KLP_UNPATCHED);
 888
 889         klp_for_each_object(patch, obj)
 890                 if (obj->patched)
 891                         klp_pre_unpatch_callback(obj);
 892
 893         /*
 894          * Enforce the order of the func->transition writes in
 895          * klp_init_transition() and the TIF_PATCH_PENDING writes in
 896          * klp_start_transition().  In the rare case where klp_ftrace_handler()
 897          * is called shortly after klp_update_patch_state() switches the task,
 898          * this ensures the handler sees that func->transition is set.
 899          */
 900         smp_wmb();
 901
 902         klp_start_transition();
 903         patch->enabled = false;
 904         klp_try_complete_transition();
 905
 906         return 0;
 907 }
 908
 909 static int __klp_enable_patch(struct klp_patch *patch)
 910 {
 911         struct klp_object *obj;
 912         int ret;
 913
 914         if (klp_transition_patch)
 915                 return -EBUSY;
 916
 917         if (WARN_ON(patch->enabled))
 918                 return -EINVAL;
 919
 920         pr_notice("enabling patch '%s'\n", patch->mod->name);
 921
 922         klp_init_transition(patch, KLP_PATCHED);
 923
 924         /*
 925          * Enforce the order of the func->transition writes in
 926          * klp_init_transition() and the ops->func_stack writes in
 927          * klp_patch_object(), so that klp_ftrace_handler() will see the
 928          * func->transition updates before the handler is registered and the
 929          * new funcs become visible to the handler.
 930          */
 931         smp_wmb();
 932
 933         klp_for_each_object(patch, obj) {
 934                 if (!klp_is_object_loaded(obj))
 935                         continue;
 936
 937                 ret = klp_pre_patch_callback(obj);
 938                 if (ret) {
 939                         pr_warn("pre-patch callback failed for object '%s'\n",
 940                                 klp_is_module(obj) ? obj->name : "vmlinux");
 941                         goto err;
 942                 }
 943
 944                 ret = klp_patch_object(obj);
 945                 if (ret) {
 946                         pr_warn("failed to patch object '%s'\n",
 947                                 klp_is_module(obj) ? obj->name : "vmlinux");
 948                         goto err;
 949                 }
 950         }
 951
 952         klp_start_transition();
 953         patch->enabled = true;
 954         klp_try_complete_transition();
 955
 956         return 0;
 957 err:
 958         pr_warn("failed to enable patch '%s'\n", patch->mod->name);
 959
 960         klp_cancel_transition();
 961         return ret;
 962 }
 963
 964 /**
 965  * klp_enable_patch() - enable the livepatch
 966  * @patch:      patch to be enabled
 967  *
 968  * Initializes the data structure associated with the patch, creates the sysfs
 969  * interface, performs the needed symbol lookups and code relocations,
 970  * registers the patched functions with ftrace.
 971  *
 972  * This function is supposed to be called from the livepatch module_init()
 973  * callback.
 974  *
 975  * Return: 0 on success, otherwise error
 976  */
 977 int klp_enable_patch(struct klp_patch *patch)
 978 {
 979         int ret;
 980
 981         if (!patch || !patch->mod)
 982                 return -EINVAL;
 983
 984         if (!is_livepatch_module(patch->mod)) {
 985                 pr_err("module %s is not marked as a livepatch module\n",
 986                        patch->mod->name);
 987                 return -EINVAL;
 988         }
 989
 990         if (!klp_initialized())
 991                 return -ENODEV;
 992
 993         if (!klp_have_reliable_stack()) {
 994                 pr_warn("This architecture doesn't have support for the livepatch consistency model.\n");
 995                 pr_warn("The livepatch transition may never complete.\n");
 996         }
 997
 998         mutex_lock(&klp_mutex);
 999
1000         ret = klp_init_patch_early(patch);
1001         if (ret) {
1002                 mutex_unlock(&klp_mutex);
1003                 return ret;
1004         }
1005
1006         ret = klp_init_patch(patch);
1007         if (ret)
1008                 goto err;
1009
1010         ret = __klp_enable_patch(patch);
1011         if (ret)
1012                 goto err;
1013
1014         mutex_unlock(&klp_mutex);
1015
1016         return 0;
1017
1018 err:
1019         klp_free_patch_start(patch);
1020
1021         mutex_unlock(&klp_mutex);
1022
1023         klp_free_patch_finish(patch);
1024
1025         return ret;
1026 }
1027 EXPORT_SYMBOL_GPL(klp_enable_patch);
1028
1029 /*
1030  * This function removes replaced patches.
1031  *
1032  * We could be pretty aggressive here. It is called in the situation where
1033  * these structures are no longer accessible. All functions are redirected
1034  * by the klp_transition_patch. They use either a new code or they are in
1035  * the original code because of the special nop function patches.
1036  *
1037  * The only exception is when the transition was forced. In this case,
1038  * klp_ftrace_handler() might still see the replaced patch on the stack.
1039  * Fortunately, it is carefully designed to work with removed functions
1040  * thanks to RCU. We only have to keep the patches on the system. Also
1041  * this is handled transparently by patch->module_put.
1042  */
1043 void klp_discard_replaced_patches(struct klp_patch *new_patch)
1044 {
1045         struct klp_patch *old_patch, *tmp_patch;
1046
1047         klp_for_each_patch_safe(old_patch, tmp_patch) {
1048                 if (old_patch == new_patch)
1049                         return;
1050
1051                 old_patch->enabled = false;
1052                 klp_unpatch_objects(old_patch);
1053                 klp_free_patch_start(old_patch);
1054                 schedule_work(&old_patch->free_work);
1055         }
1056 }
1057
1058 /*
1059  * This function removes the dynamically allocated 'nop' functions.
1060  *
1061  * We could be pretty aggressive. NOPs do not change the existing
1062  * behavior except for adding unnecessary delay by the ftrace handler.
1063  *
1064  * It is safe even when the transition was forced. The ftrace handler
1065  * will see a valid ops->func_stack entry thanks to RCU.
1066  *
1067  * We could even free the NOPs structures. They must be the last entry
1068  * in ops->func_stack. Therefore unregister_ftrace_function() is called.
1069  * It does the same as klp_synchronize_transition() to make sure that
1070  * nobody is inside the ftrace handler once the operation finishes.
1071  *
1072  * IMPORTANT: It must be called right after removing the replaced patches!
1073  */
1074 void klp_discard_nops(struct klp_patch *new_patch)
1075 {
1076         klp_unpatch_objects_dynamic(klp_transition_patch);
1077         klp_free_objects_dynamic(klp_transition_patch);
1078 }
1079
1080 /*
1081  * Remove parts of patches that touch a given kernel module. The list of
1082  * patches processed might be limited. When limit is NULL, all patches
1083  * will be handled.
1084  */
1085 static void klp_cleanup_module_patches_limited(struct module *mod,
1086                                                struct klp_patch *limit)
1087 {
1088         struct klp_patch *patch;
1089         struct klp_object *obj;
1090
1091         klp_for_each_patch(patch) {
1092                 if (patch == limit)
1093                         break;
1094
1095                 klp_for_each_object(patch, obj) {
1096                         if (!klp_is_module(obj) || strcmp(obj->name, mod->name))
1097                                 continue;
1098
1099                         if (patch != klp_transition_patch)
1100                                 klp_pre_unpatch_callback(obj);
1101
1102                         pr_notice("reverting patch '%s' on unloading module '%s'\n",
1103                                   patch->mod->name, obj->mod->name);
1104                         klp_unpatch_object(obj);
1105
1106                         klp_post_unpatch_callback(obj);
1107
1108                         klp_free_object_loaded(obj);
1109                         break;
1110                 }
1111         }
1112 }
1113
1114 int klp_module_coming(struct module *mod)
1115 {
1116         int ret;
1117         struct klp_patch *patch;
1118         struct klp_object *obj;
1119
1120         if (WARN_ON(mod->state != MODULE_STATE_COMING))
1121                 return -EINVAL;
1122
1123         mutex_lock(&klp_mutex);
1124         /*
1125          * Each module has to know that klp_module_coming()
1126          * has been called. We never know what module will
1127          * get patched by a new patch.
1128          */
1129         mod->klp_alive = true;
1130
1131         klp_for_each_patch(patch) {
1132                 klp_for_each_object(patch, obj) {
1133                         if (!klp_is_module(obj) || strcmp(obj->name, mod->name))
1134                                 continue;
1135
1136                         obj->mod = mod;
1137
1138                         ret = klp_init_object_loaded(patch, obj);
1139                         if (ret) {
1140                                 pr_warn("failed to initialize patch '%s' for module '%s' (%d)\n",
1141                                         patch->mod->name, obj->mod->name, ret);
1142                                 goto err;
1143                         }
1144
1145                         pr_notice("applying patch '%s' to loading module '%s'\n",
1146                                   patch->mod->name, obj->mod->name);
1147
1148                         ret = klp_pre_patch_callback(obj);
1149                         if (ret) {
1150                                 pr_warn("pre-patch callback failed for object '%s'\n",
1151                                         obj->name);
1152                                 goto err;
1153                         }
1154
1155                         ret = klp_patch_object(obj);
1156                         if (ret) {
1157                                 pr_warn("failed to apply patch '%s' to module '%s' (%d)\n",
1158                                         patch->mod->name, obj->mod->name, ret);
1159
1160                                 klp_post_unpatch_callback(obj);
1161                                 goto err;
1162                         }
1163
1164                         if (patch != klp_transition_patch)
1165                                 klp_post_patch_callback(obj);
1166
1167                         break;
1168                 }
1169         }
1170
1171         mutex_unlock(&klp_mutex);
1172
1173         return 0;
1174
1175 err:
1176         /*
1177          * If a patch is unsuccessfully applied, return
1178          * error to the module loader.
1179          */
1180         pr_warn("patch '%s' failed for module '%s', refusing to load module '%s'\n",
1181                 patch->mod->name, obj->mod->name, obj->mod->name);
1182         mod->klp_alive = false;
1183         klp_cleanup_module_patches_limited(mod, patch);
1184         mutex_unlock(&klp_mutex);
1185
1186         return ret;
1187 }
1188
1189 void klp_module_going(struct module *mod)
1190 {
1191         if (WARN_ON(mod->state != MODULE_STATE_GOING &&
1192                     mod->state != MODULE_STATE_COMING))
1193                 return;
1194
1195         mutex_lock(&klp_mutex);
1196         /*
1197          * Each module has to know that klp_module_going()
1198          * has been called. We never know what module will
1199          * get patched by a new patch.
1200          */
1201         mod->klp_alive = false;
1202
1203         klp_cleanup_module_patches_limited(mod, NULL);
1204
1205         mutex_unlock(&klp_mutex);
1206 }
1207
1208 static int __init klp_init(void)
1209 {
1210         int ret;
1211
1212         ret = klp_check_compiler_support();
1213         if (ret) {
1214                 pr_info("Your compiler is too old; turning off.\n");
1215                 return -EINVAL;
1216         }
1217
1218         klp_root_kobj = kobject_create_and_add("livepatch", kernel_kobj);
1219         if (!klp_root_kobj)
1220                 return -ENOMEM;
1221
1222         return 0;
1223 }
1224
1225 module_init(klp_init);