2 * Copyright (C) 2015 IT University of Copenhagen. All rights reserved.
3 * Initial release: Matias Bjorling <m@bjorling.me>
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License version
7 * 2 as published by the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful, but
10 * WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program; see the file COPYING. If not, write to
16 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139,
21 #include <linux/list.h>
22 #include <linux/types.h>
23 #include <linux/sem.h>
24 #include <linux/bitmap.h>
25 #include <linux/moduleparam.h>
26 #include <linux/miscdevice.h>
27 #include <linux/lightnvm.h>
28 #include <linux/sched/sysctl.h>
30 static LIST_HEAD(nvm_tgt_types);
31 static DECLARE_RWSEM(nvm_tgtt_lock);
32 static LIST_HEAD(nvm_devices);
33 static DECLARE_RWSEM(nvm_lock);
35 /* Map between virtual and physical channel and lun */
43 struct nvm_ch_map *chnls;
48 struct list_head list;
50 sector_t end; /* end is excluded */
53 static struct nvm_target *nvm_find_target(struct nvm_dev *dev, const char *name)
55 struct nvm_target *tgt;
57 list_for_each_entry(tgt, &dev->targets, list)
58 if (!strcmp(name, tgt->disk->disk_name))
64 static int nvm_reserve_luns(struct nvm_dev *dev, int lun_begin, int lun_end)
68 for (i = lun_begin; i <= lun_end; i++) {
69 if (test_and_set_bit(i, dev->lun_map)) {
70 pr_err("nvm: lun %d already allocated\n", i);
77 while (--i > lun_begin)
78 clear_bit(i, dev->lun_map);
83 static void nvm_release_luns_err(struct nvm_dev *dev, int lun_begin,
88 for (i = lun_begin; i <= lun_end; i++)
89 WARN_ON(!test_and_clear_bit(i, dev->lun_map));
92 static void nvm_remove_tgt_dev(struct nvm_tgt_dev *tgt_dev)
94 struct nvm_dev *dev = tgt_dev->parent;
95 struct nvm_dev_map *dev_map = tgt_dev->map;
98 for (i = 0; i < dev_map->nr_chnls; i++) {
99 struct nvm_ch_map *ch_map = &dev_map->chnls[i];
100 int *lun_offs = ch_map->lun_offs;
101 int ch = i + ch_map->ch_off;
103 for (j = 0; j < ch_map->nr_luns; j++) {
104 int lun = j + lun_offs[j];
105 int lunid = (ch * dev->geo.luns_per_chnl) + lun;
107 WARN_ON(!test_and_clear_bit(lunid, dev->lun_map));
110 kfree(ch_map->lun_offs);
113 kfree(dev_map->chnls);
116 kfree(tgt_dev->luns);
120 static struct nvm_tgt_dev *nvm_create_tgt_dev(struct nvm_dev *dev,
121 int lun_begin, int lun_end)
123 struct nvm_tgt_dev *tgt_dev = NULL;
124 struct nvm_dev_map *dev_rmap = dev->rmap;
125 struct nvm_dev_map *dev_map;
126 struct ppa_addr *luns;
127 int nr_luns = lun_end - lun_begin + 1;
128 int luns_left = nr_luns;
129 int nr_chnls = nr_luns / dev->geo.luns_per_chnl;
130 int nr_chnls_mod = nr_luns % dev->geo.luns_per_chnl;
131 int bch = lun_begin / dev->geo.luns_per_chnl;
132 int blun = lun_begin % dev->geo.luns_per_chnl;
134 int lun_balanced = 1;
138 nr_chnls = nr_luns / dev->geo.luns_per_chnl;
139 nr_chnls = (nr_chnls_mod == 0) ? nr_chnls : nr_chnls + 1;
141 dev_map = kmalloc(sizeof(struct nvm_dev_map), GFP_KERNEL);
145 dev_map->chnls = kcalloc(nr_chnls, sizeof(struct nvm_ch_map),
150 luns = kcalloc(nr_luns, sizeof(struct ppa_addr), GFP_KERNEL);
154 prev_nr_luns = (luns_left > dev->geo.luns_per_chnl) ?
155 dev->geo.luns_per_chnl : luns_left;
156 for (i = 0; i < nr_chnls; i++) {
157 struct nvm_ch_map *ch_rmap = &dev_rmap->chnls[i + bch];
158 int *lun_roffs = ch_rmap->lun_offs;
159 struct nvm_ch_map *ch_map = &dev_map->chnls[i];
161 int luns_in_chnl = (luns_left > dev->geo.luns_per_chnl) ?
162 dev->geo.luns_per_chnl : luns_left;
164 if (lun_balanced && prev_nr_luns != luns_in_chnl)
167 ch_map->ch_off = ch_rmap->ch_off = bch;
168 ch_map->nr_luns = luns_in_chnl;
170 lun_offs = kcalloc(luns_in_chnl, sizeof(int), GFP_KERNEL);
174 for (j = 0; j < luns_in_chnl; j++) {
176 luns[lunid].g.ch = i;
177 luns[lunid++].g.lun = j;
180 lun_roffs[j + blun] = blun;
183 ch_map->lun_offs = lun_offs;
185 /* when starting a new channel, lun offset is reset */
187 luns_left -= luns_in_chnl;
190 dev_map->nr_chnls = nr_chnls;
192 tgt_dev = kmalloc(sizeof(struct nvm_tgt_dev), GFP_KERNEL);
196 memcpy(&tgt_dev->geo, &dev->geo, sizeof(struct nvm_geo));
197 /* Target device only owns a portion of the physical device */
198 tgt_dev->geo.nr_chnls = nr_chnls;
199 tgt_dev->geo.nr_luns = nr_luns;
200 tgt_dev->geo.luns_per_chnl = (lun_balanced) ? prev_nr_luns : -1;
201 tgt_dev->total_secs = nr_luns * tgt_dev->geo.sec_per_lun;
203 tgt_dev->map = dev_map;
204 tgt_dev->luns = luns;
205 memcpy(&tgt_dev->identity, &dev->identity, sizeof(struct nvm_id));
207 tgt_dev->parent = dev;
212 kfree(dev_map->chnls[i].lun_offs);
215 kfree(dev_map->chnls);
222 static const struct block_device_operations nvm_fops = {
223 .owner = THIS_MODULE,
226 static int nvm_create_tgt(struct nvm_dev *dev, struct nvm_ioctl_create *create)
228 struct nvm_ioctl_create_simple *s = &create->conf.s;
229 struct request_queue *tqueue;
230 struct gendisk *tdisk;
231 struct nvm_tgt_type *tt;
232 struct nvm_target *t;
233 struct nvm_tgt_dev *tgt_dev;
236 tt = nvm_find_target_type(create->tgttype, 1);
238 pr_err("nvm: target type %s not found\n", create->tgttype);
242 mutex_lock(&dev->mlock);
243 t = nvm_find_target(dev, create->tgtname);
245 pr_err("nvm: target name already exists.\n");
246 mutex_unlock(&dev->mlock);
249 mutex_unlock(&dev->mlock);
251 if (nvm_reserve_luns(dev, s->lun_begin, s->lun_end))
254 t = kmalloc(sizeof(struct nvm_target), GFP_KERNEL);
258 tgt_dev = nvm_create_tgt_dev(dev, s->lun_begin, s->lun_end);
260 pr_err("nvm: could not create target device\n");
264 tqueue = blk_alloc_queue_node(GFP_KERNEL, dev->q->node);
267 blk_queue_make_request(tqueue, tt->make_rq);
269 tdisk = alloc_disk(0);
273 sprintf(tdisk->disk_name, "%s", create->tgtname);
274 tdisk->flags = GENHD_FL_EXT_DEVT;
276 tdisk->first_minor = 0;
277 tdisk->fops = &nvm_fops;
278 tdisk->queue = tqueue;
280 targetdata = tt->init(tgt_dev, tdisk);
281 if (IS_ERR(targetdata))
284 tdisk->private_data = targetdata;
285 tqueue->queuedata = targetdata;
287 blk_queue_max_hw_sectors(tqueue, 8 * dev->ops->max_phys_sect);
289 set_capacity(tdisk, tt->capacity(targetdata));
296 mutex_lock(&dev->mlock);
297 list_add_tail(&t->list, &dev->targets);
298 mutex_unlock(&dev->mlock);
304 blk_cleanup_queue(tqueue);
310 nvm_release_luns_err(dev, s->lun_begin, s->lun_end);
314 static void __nvm_remove_target(struct nvm_target *t)
316 struct nvm_tgt_type *tt = t->type;
317 struct gendisk *tdisk = t->disk;
318 struct request_queue *q = tdisk->queue;
321 blk_cleanup_queue(q);
324 tt->exit(tdisk->private_data);
326 nvm_remove_tgt_dev(t->dev);
334 * nvm_remove_tgt - Removes a target from the media manager
336 * @remove: ioctl structure with target name to remove.
343 static int nvm_remove_tgt(struct nvm_dev *dev, struct nvm_ioctl_remove *remove)
345 struct nvm_target *t;
347 mutex_lock(&dev->mlock);
348 t = nvm_find_target(dev, remove->tgtname);
350 mutex_unlock(&dev->mlock);
353 __nvm_remove_target(t);
354 mutex_unlock(&dev->mlock);
359 static int nvm_register_map(struct nvm_dev *dev)
361 struct nvm_dev_map *rmap;
364 rmap = kmalloc(sizeof(struct nvm_dev_map), GFP_KERNEL);
368 rmap->chnls = kcalloc(dev->geo.nr_chnls, sizeof(struct nvm_ch_map),
373 for (i = 0; i < dev->geo.nr_chnls; i++) {
374 struct nvm_ch_map *ch_rmap;
376 int luns_in_chnl = dev->geo.luns_per_chnl;
378 ch_rmap = &rmap->chnls[i];
380 ch_rmap->ch_off = -1;
381 ch_rmap->nr_luns = luns_in_chnl;
383 lun_roffs = kcalloc(luns_in_chnl, sizeof(int), GFP_KERNEL);
387 for (j = 0; j < luns_in_chnl; j++)
390 ch_rmap->lun_offs = lun_roffs;
398 kfree(rmap->chnls[i].lun_offs);
405 static void nvm_map_to_dev(struct nvm_tgt_dev *tgt_dev, struct ppa_addr *p)
407 struct nvm_dev_map *dev_map = tgt_dev->map;
408 struct nvm_ch_map *ch_map = &dev_map->chnls[p->g.ch];
409 int lun_off = ch_map->lun_offs[p->g.lun];
411 p->g.ch += ch_map->ch_off;
415 static void nvm_map_to_tgt(struct nvm_tgt_dev *tgt_dev, struct ppa_addr *p)
417 struct nvm_dev *dev = tgt_dev->parent;
418 struct nvm_dev_map *dev_rmap = dev->rmap;
419 struct nvm_ch_map *ch_rmap = &dev_rmap->chnls[p->g.ch];
420 int lun_roff = ch_rmap->lun_offs[p->g.lun];
422 p->g.ch -= ch_rmap->ch_off;
423 p->g.lun -= lun_roff;
426 static void nvm_ppa_tgt_to_dev(struct nvm_tgt_dev *tgt_dev,
427 struct ppa_addr *ppa_list, int nr_ppas)
431 for (i = 0; i < nr_ppas; i++) {
432 nvm_map_to_dev(tgt_dev, &ppa_list[i]);
433 ppa_list[i] = generic_to_dev_addr(tgt_dev, ppa_list[i]);
437 static void nvm_ppa_dev_to_tgt(struct nvm_tgt_dev *tgt_dev,
438 struct ppa_addr *ppa_list, int nr_ppas)
442 for (i = 0; i < nr_ppas; i++) {
443 ppa_list[i] = dev_to_generic_addr(tgt_dev, ppa_list[i]);
444 nvm_map_to_tgt(tgt_dev, &ppa_list[i]);
448 static void nvm_rq_tgt_to_dev(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd)
450 if (rqd->nr_ppas == 1) {
451 nvm_ppa_tgt_to_dev(tgt_dev, &rqd->ppa_addr, 1);
455 nvm_ppa_tgt_to_dev(tgt_dev, rqd->ppa_list, rqd->nr_ppas);
458 static void nvm_rq_dev_to_tgt(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd)
460 if (rqd->nr_ppas == 1) {
461 nvm_ppa_dev_to_tgt(tgt_dev, &rqd->ppa_addr, 1);
465 nvm_ppa_dev_to_tgt(tgt_dev, rqd->ppa_list, rqd->nr_ppas);
468 void nvm_part_to_tgt(struct nvm_dev *dev, sector_t *entries,
471 struct nvm_geo *geo = &dev->geo;
472 struct nvm_dev_map *dev_rmap = dev->rmap;
475 for (i = 0; i < len; i++) {
476 struct nvm_ch_map *ch_rmap;
478 struct ppa_addr gaddr;
479 u64 pba = le64_to_cpu(entries[i]);
486 gaddr = linear_to_generic_addr(geo, pba);
487 ch_rmap = &dev_rmap->chnls[gaddr.g.ch];
488 lun_roffs = ch_rmap->lun_offs;
490 off = gaddr.g.ch * geo->luns_per_chnl + gaddr.g.lun;
492 diff = ((ch_rmap->ch_off * geo->luns_per_chnl) +
493 (lun_roffs[gaddr.g.lun])) * geo->sec_per_lun;
495 entries[i] -= cpu_to_le64(diff);
498 EXPORT_SYMBOL(nvm_part_to_tgt);
500 struct nvm_tgt_type *nvm_find_target_type(const char *name, int lock)
502 struct nvm_tgt_type *tmp, *tt = NULL;
505 down_write(&nvm_tgtt_lock);
507 list_for_each_entry(tmp, &nvm_tgt_types, list)
508 if (!strcmp(name, tmp->name)) {
514 up_write(&nvm_tgtt_lock);
517 EXPORT_SYMBOL(nvm_find_target_type);
519 int nvm_register_tgt_type(struct nvm_tgt_type *tt)
523 down_write(&nvm_tgtt_lock);
524 if (nvm_find_target_type(tt->name, 0))
527 list_add(&tt->list, &nvm_tgt_types);
528 up_write(&nvm_tgtt_lock);
532 EXPORT_SYMBOL(nvm_register_tgt_type);
534 void nvm_unregister_tgt_type(struct nvm_tgt_type *tt)
539 down_write(&nvm_lock);
543 EXPORT_SYMBOL(nvm_unregister_tgt_type);
545 void *nvm_dev_dma_alloc(struct nvm_dev *dev, gfp_t mem_flags,
546 dma_addr_t *dma_handler)
548 return dev->ops->dev_dma_alloc(dev, dev->dma_pool, mem_flags,
551 EXPORT_SYMBOL(nvm_dev_dma_alloc);
553 void nvm_dev_dma_free(struct nvm_dev *dev, void *addr, dma_addr_t dma_handler)
555 dev->ops->dev_dma_free(dev->dma_pool, addr, dma_handler);
557 EXPORT_SYMBOL(nvm_dev_dma_free);
559 static struct nvm_dev *nvm_find_nvm_dev(const char *name)
563 list_for_each_entry(dev, &nvm_devices, devices)
564 if (!strcmp(name, dev->name))
570 int nvm_set_tgt_bb_tbl(struct nvm_tgt_dev *tgt_dev, struct ppa_addr *ppas,
571 int nr_ppas, int type)
573 struct nvm_dev *dev = tgt_dev->parent;
577 if (nr_ppas > dev->ops->max_phys_sect) {
578 pr_err("nvm: unable to update all blocks atomically\n");
582 memset(&rqd, 0, sizeof(struct nvm_rq));
584 nvm_set_rqd_ppalist(dev, &rqd, ppas, nr_ppas, 1);
585 nvm_rq_tgt_to_dev(tgt_dev, &rqd);
587 ret = dev->ops->set_bb_tbl(dev, &rqd.ppa_addr, rqd.nr_ppas, type);
588 nvm_free_rqd_ppalist(dev, &rqd);
590 pr_err("nvm: failed bb mark\n");
596 EXPORT_SYMBOL(nvm_set_tgt_bb_tbl);
598 int nvm_max_phys_sects(struct nvm_tgt_dev *tgt_dev)
600 struct nvm_dev *dev = tgt_dev->parent;
602 return dev->ops->max_phys_sect;
604 EXPORT_SYMBOL(nvm_max_phys_sects);
606 int nvm_submit_io(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd)
608 struct nvm_dev *dev = tgt_dev->parent;
610 if (!dev->ops->submit_io)
613 nvm_rq_tgt_to_dev(tgt_dev, rqd);
616 return dev->ops->submit_io(dev, rqd);
618 EXPORT_SYMBOL(nvm_submit_io);
620 int nvm_erase_blk(struct nvm_tgt_dev *tgt_dev, struct ppa_addr *ppas, int flags)
622 struct nvm_dev *dev = tgt_dev->parent;
626 if (!dev->ops->erase_block)
629 nvm_map_to_dev(tgt_dev, ppas);
631 memset(&rqd, 0, sizeof(struct nvm_rq));
633 ret = nvm_set_rqd_ppalist(dev, &rqd, ppas, 1, 1);
637 nvm_rq_tgt_to_dev(tgt_dev, &rqd);
641 ret = dev->ops->erase_block(dev, &rqd);
643 nvm_free_rqd_ppalist(dev, &rqd);
647 EXPORT_SYMBOL(nvm_erase_blk);
649 int nvm_get_l2p_tbl(struct nvm_tgt_dev *tgt_dev, u64 slba, u32 nlb,
650 nvm_l2p_update_fn *update_l2p, void *priv)
652 struct nvm_dev *dev = tgt_dev->parent;
654 if (!dev->ops->get_l2p_tbl)
657 return dev->ops->get_l2p_tbl(dev, slba, nlb, update_l2p, priv);
659 EXPORT_SYMBOL(nvm_get_l2p_tbl);
661 int nvm_get_area(struct nvm_tgt_dev *tgt_dev, sector_t *lba, sector_t len)
663 struct nvm_dev *dev = tgt_dev->parent;
664 struct nvm_geo *geo = &dev->geo;
665 struct nvm_area *area, *prev, *next;
667 sector_t max_sectors = (geo->sec_size * dev->total_secs) >> 9;
669 if (len > max_sectors)
672 area = kmalloc(sizeof(struct nvm_area), GFP_KERNEL);
678 spin_lock(&dev->lock);
679 list_for_each_entry(next, &dev->area_list, list) {
680 if (begin + len > next->begin) {
688 if ((begin + len) > max_sectors) {
689 spin_unlock(&dev->lock);
694 area->begin = *lba = begin;
695 area->end = begin + len;
697 if (prev) /* insert into sorted order */
698 list_add(&area->list, &prev->list);
700 list_add(&area->list, &dev->area_list);
701 spin_unlock(&dev->lock);
705 EXPORT_SYMBOL(nvm_get_area);
707 void nvm_put_area(struct nvm_tgt_dev *tgt_dev, sector_t begin)
709 struct nvm_dev *dev = tgt_dev->parent;
710 struct nvm_area *area;
712 spin_lock(&dev->lock);
713 list_for_each_entry(area, &dev->area_list, list) {
714 if (area->begin != begin)
717 list_del(&area->list);
718 spin_unlock(&dev->lock);
722 spin_unlock(&dev->lock);
724 EXPORT_SYMBOL(nvm_put_area);
726 int nvm_set_rqd_ppalist(struct nvm_dev *dev, struct nvm_rq *rqd,
727 const struct ppa_addr *ppas, int nr_ppas, int vblk)
729 struct nvm_geo *geo = &dev->geo;
730 int i, plane_cnt, pl_idx;
733 if ((!vblk || geo->plane_mode == NVM_PLANE_SINGLE) && nr_ppas == 1) {
734 rqd->nr_ppas = nr_ppas;
735 rqd->ppa_addr = ppas[0];
740 rqd->nr_ppas = nr_ppas;
741 rqd->ppa_list = nvm_dev_dma_alloc(dev, GFP_KERNEL, &rqd->dma_ppa_list);
742 if (!rqd->ppa_list) {
743 pr_err("nvm: failed to allocate dma memory\n");
748 for (i = 0; i < nr_ppas; i++)
749 rqd->ppa_list[i] = ppas[i];
751 plane_cnt = geo->plane_mode;
752 rqd->nr_ppas *= plane_cnt;
754 for (i = 0; i < nr_ppas; i++) {
755 for (pl_idx = 0; pl_idx < plane_cnt; pl_idx++) {
758 rqd->ppa_list[(pl_idx * nr_ppas) + i] = ppa;
765 EXPORT_SYMBOL(nvm_set_rqd_ppalist);
767 void nvm_free_rqd_ppalist(struct nvm_dev *dev, struct nvm_rq *rqd)
772 nvm_dev_dma_free(dev, rqd->ppa_list, rqd->dma_ppa_list);
774 EXPORT_SYMBOL(nvm_free_rqd_ppalist);
776 void nvm_end_io(struct nvm_rq *rqd, int error)
778 struct nvm_tgt_dev *tgt_dev = rqd->dev;
779 struct nvm_tgt_instance *ins = rqd->ins;
781 /* Convert address space */
783 nvm_rq_dev_to_tgt(tgt_dev, rqd);
786 ins->tt->end_io(rqd);
788 EXPORT_SYMBOL(nvm_end_io);
791 * folds a bad block list from its plane representation to its virtual
792 * block representation. The fold is done in place and reduced size is
795 * If any of the planes status are bad or grown bad block, the virtual block
796 * is marked bad. If not bad, the first plane state acts as the block state.
798 int nvm_bb_tbl_fold(struct nvm_dev *dev, u8 *blks, int nr_blks)
800 struct nvm_geo *geo = &dev->geo;
801 int blk, offset, pl, blktype;
803 if (nr_blks != geo->blks_per_lun * geo->plane_mode)
806 for (blk = 0; blk < geo->blks_per_lun; blk++) {
807 offset = blk * geo->plane_mode;
808 blktype = blks[offset];
810 /* Bad blocks on any planes take precedence over other types */
811 for (pl = 0; pl < geo->plane_mode; pl++) {
812 if (blks[offset + pl] &
813 (NVM_BLK_T_BAD|NVM_BLK_T_GRWN_BAD)) {
814 blktype = blks[offset + pl];
822 return geo->blks_per_lun;
824 EXPORT_SYMBOL(nvm_bb_tbl_fold);
826 int nvm_get_tgt_bb_tbl(struct nvm_tgt_dev *tgt_dev, struct ppa_addr ppa,
829 struct nvm_dev *dev = tgt_dev->parent;
831 nvm_ppa_tgt_to_dev(tgt_dev, &ppa, 1);
833 return dev->ops->get_bb_tbl(dev, ppa, blks);
835 EXPORT_SYMBOL(nvm_get_tgt_bb_tbl);
837 static int nvm_init_slc_tbl(struct nvm_dev *dev, struct nvm_id_group *grp)
839 struct nvm_geo *geo = &dev->geo;
842 dev->lps_per_blk = geo->pgs_per_blk;
843 dev->lptbl = kcalloc(dev->lps_per_blk, sizeof(int), GFP_KERNEL);
847 /* Just a linear array */
848 for (i = 0; i < dev->lps_per_blk; i++)
854 static int nvm_init_mlc_tbl(struct nvm_dev *dev, struct nvm_id_group *grp)
857 struct nvm_id_lp_mlc *mlc = &grp->lptbl.mlc;
862 dev->lps_per_blk = mlc->num_pairs;
863 dev->lptbl = kcalloc(dev->lps_per_blk, sizeof(int), GFP_KERNEL);
867 /* The lower page table encoding consists of a list of bytes, where each
868 * has a lower and an upper half. The first half byte maintains the
869 * increment value and every value after is an offset added to the
870 * previous incrementation value
872 dev->lptbl[0] = mlc->pairs[0] & 0xF;
873 for (i = 1; i < dev->lps_per_blk; i++) {
874 p = mlc->pairs[i >> 1];
875 if (i & 0x1) /* upper */
876 dev->lptbl[i] = dev->lptbl[i - 1] + ((p & 0xF0) >> 4);
878 dev->lptbl[i] = dev->lptbl[i - 1] + (p & 0xF);
884 static int nvm_core_init(struct nvm_dev *dev)
886 struct nvm_id *id = &dev->identity;
887 struct nvm_id_group *grp = &id->groups[0];
888 struct nvm_geo *geo = &dev->geo;
891 /* Whole device values */
892 geo->nr_chnls = grp->num_ch;
893 geo->luns_per_chnl = grp->num_lun;
895 /* Generic device values */
896 geo->pgs_per_blk = grp->num_pg;
897 geo->blks_per_lun = grp->num_blk;
898 geo->nr_planes = grp->num_pln;
899 geo->fpg_size = grp->fpg_sz;
900 geo->pfpg_size = grp->fpg_sz * grp->num_pln;
901 geo->sec_size = grp->csecs;
902 geo->oob_size = grp->sos;
903 geo->sec_per_pg = grp->fpg_sz / grp->csecs;
904 geo->mccap = grp->mccap;
905 memcpy(&geo->ppaf, &id->ppaf, sizeof(struct nvm_addr_format));
907 geo->plane_mode = NVM_PLANE_SINGLE;
908 geo->max_rq_size = dev->ops->max_phys_sect * geo->sec_size;
910 if (grp->mpos & 0x020202)
911 geo->plane_mode = NVM_PLANE_DOUBLE;
912 if (grp->mpos & 0x040404)
913 geo->plane_mode = NVM_PLANE_QUAD;
915 if (grp->mtype != 0) {
916 pr_err("nvm: memory type not supported\n");
920 /* calculated values */
921 geo->sec_per_pl = geo->sec_per_pg * geo->nr_planes;
922 geo->sec_per_blk = geo->sec_per_pl * geo->pgs_per_blk;
923 geo->sec_per_lun = geo->sec_per_blk * geo->blks_per_lun;
924 geo->nr_luns = geo->luns_per_chnl * geo->nr_chnls;
926 dev->total_secs = geo->nr_luns * geo->sec_per_lun;
927 dev->lun_map = kcalloc(BITS_TO_LONGS(geo->nr_luns),
928 sizeof(unsigned long), GFP_KERNEL);
932 switch (grp->fmtype) {
933 case NVM_ID_FMTYPE_SLC:
934 if (nvm_init_slc_tbl(dev, grp)) {
939 case NVM_ID_FMTYPE_MLC:
940 if (nvm_init_mlc_tbl(dev, grp)) {
946 pr_err("nvm: flash type not supported\n");
951 INIT_LIST_HEAD(&dev->area_list);
952 INIT_LIST_HEAD(&dev->targets);
953 mutex_init(&dev->mlock);
954 spin_lock_init(&dev->lock);
956 ret = nvm_register_map(dev);
960 blk_queue_logical_block_size(dev->q, geo->sec_size);
967 void nvm_free(struct nvm_dev *dev)
973 dev->ops->destroy_dma_pool(dev->dma_pool);
981 static int nvm_init(struct nvm_dev *dev)
983 struct nvm_geo *geo = &dev->geo;
986 if (dev->ops->identity(dev, &dev->identity)) {
987 pr_err("nvm: device could not be identified\n");
991 pr_debug("nvm: ver:%x nvm_vendor:%x groups:%u\n",
992 dev->identity.ver_id, dev->identity.vmnt,
993 dev->identity.cgrps);
995 if (dev->identity.ver_id != 1) {
996 pr_err("nvm: device not supported by kernel.");
1000 if (dev->identity.cgrps != 1) {
1001 pr_err("nvm: only one group configuration supported.");
1005 ret = nvm_core_init(dev);
1007 pr_err("nvm: could not initialize core structures.\n");
1011 pr_info("nvm: registered %s [%u/%u/%u/%u/%u/%u]\n",
1012 dev->name, geo->sec_per_pg, geo->nr_planes,
1013 geo->pgs_per_blk, geo->blks_per_lun,
1014 geo->nr_luns, geo->nr_chnls);
1017 pr_err("nvm: failed to initialize nvm\n");
1021 struct nvm_dev *nvm_alloc_dev(int node)
1023 return kzalloc_node(sizeof(struct nvm_dev), GFP_KERNEL, node);
1025 EXPORT_SYMBOL(nvm_alloc_dev);
1027 int nvm_register(struct nvm_dev *dev)
1031 if (!dev->q || !dev->ops)
1034 if (dev->ops->max_phys_sect > 256) {
1035 pr_info("nvm: max sectors supported is 256.\n");
1039 if (dev->ops->max_phys_sect > 1) {
1040 dev->dma_pool = dev->ops->create_dma_pool(dev, "ppalist");
1041 if (!dev->dma_pool) {
1042 pr_err("nvm: could not create dma pool\n");
1047 ret = nvm_init(dev);
1051 /* register device with a supported media manager */
1052 down_write(&nvm_lock);
1053 list_add(&dev->devices, &nvm_devices);
1054 up_write(&nvm_lock);
1058 dev->ops->destroy_dma_pool(dev->dma_pool);
1061 EXPORT_SYMBOL(nvm_register);
1063 void nvm_unregister(struct nvm_dev *dev)
1065 struct nvm_target *t, *tmp;
1067 mutex_lock(&dev->mlock);
1068 list_for_each_entry_safe(t, tmp, &dev->targets, list) {
1069 if (t->dev->parent != dev)
1071 __nvm_remove_target(t);
1073 mutex_unlock(&dev->mlock);
1075 down_write(&nvm_lock);
1076 list_del(&dev->devices);
1077 up_write(&nvm_lock);
1081 EXPORT_SYMBOL(nvm_unregister);
1083 static int __nvm_configure_create(struct nvm_ioctl_create *create)
1085 struct nvm_dev *dev;
1086 struct nvm_ioctl_create_simple *s;
1088 down_write(&nvm_lock);
1089 dev = nvm_find_nvm_dev(create->dev);
1090 up_write(&nvm_lock);
1093 pr_err("nvm: device not found\n");
1097 if (create->conf.type != NVM_CONFIG_TYPE_SIMPLE) {
1098 pr_err("nvm: config type not valid\n");
1101 s = &create->conf.s;
1103 if (s->lun_begin > s->lun_end || s->lun_end > dev->geo.nr_luns) {
1104 pr_err("nvm: lun out of bound (%u:%u > %u)\n",
1105 s->lun_begin, s->lun_end, dev->geo.nr_luns);
1109 return nvm_create_tgt(dev, create);
1112 static long nvm_ioctl_info(struct file *file, void __user *arg)
1114 struct nvm_ioctl_info *info;
1115 struct nvm_tgt_type *tt;
1118 if (!capable(CAP_SYS_ADMIN))
1121 info = memdup_user(arg, sizeof(struct nvm_ioctl_info));
1125 info->version[0] = NVM_VERSION_MAJOR;
1126 info->version[1] = NVM_VERSION_MINOR;
1127 info->version[2] = NVM_VERSION_PATCH;
1129 down_write(&nvm_lock);
1130 list_for_each_entry(tt, &nvm_tgt_types, list) {
1131 struct nvm_ioctl_info_tgt *tgt = &info->tgts[tgt_iter];
1133 tgt->version[0] = tt->version[0];
1134 tgt->version[1] = tt->version[1];
1135 tgt->version[2] = tt->version[2];
1136 strncpy(tgt->tgtname, tt->name, NVM_TTYPE_NAME_MAX);
1141 info->tgtsize = tgt_iter;
1142 up_write(&nvm_lock);
1144 if (copy_to_user(arg, info, sizeof(struct nvm_ioctl_info))) {
1153 static long nvm_ioctl_get_devices(struct file *file, void __user *arg)
1155 struct nvm_ioctl_get_devices *devices;
1156 struct nvm_dev *dev;
1159 if (!capable(CAP_SYS_ADMIN))
1162 devices = kzalloc(sizeof(struct nvm_ioctl_get_devices), GFP_KERNEL);
1166 down_write(&nvm_lock);
1167 list_for_each_entry(dev, &nvm_devices, devices) {
1168 struct nvm_ioctl_device_info *info = &devices->info[i];
1170 sprintf(info->devname, "%s", dev->name);
1172 /* kept for compatibility */
1173 info->bmversion[0] = 1;
1174 info->bmversion[1] = 0;
1175 info->bmversion[2] = 0;
1176 sprintf(info->bmname, "%s", "gennvm");
1180 pr_err("nvm: max 31 devices can be reported.\n");
1184 up_write(&nvm_lock);
1186 devices->nr_devices = i;
1188 if (copy_to_user(arg, devices,
1189 sizeof(struct nvm_ioctl_get_devices))) {
1198 static long nvm_ioctl_dev_create(struct file *file, void __user *arg)
1200 struct nvm_ioctl_create create;
1202 if (!capable(CAP_SYS_ADMIN))
1205 if (copy_from_user(&create, arg, sizeof(struct nvm_ioctl_create)))
1208 create.dev[DISK_NAME_LEN - 1] = '\0';
1209 create.tgttype[NVM_TTYPE_NAME_MAX - 1] = '\0';
1210 create.tgtname[DISK_NAME_LEN - 1] = '\0';
1212 if (create.flags != 0) {
1213 pr_err("nvm: no flags supported\n");
1217 return __nvm_configure_create(&create);
1220 static long nvm_ioctl_dev_remove(struct file *file, void __user *arg)
1222 struct nvm_ioctl_remove remove;
1223 struct nvm_dev *dev;
1226 if (!capable(CAP_SYS_ADMIN))
1229 if (copy_from_user(&remove, arg, sizeof(struct nvm_ioctl_remove)))
1232 remove.tgtname[DISK_NAME_LEN - 1] = '\0';
1234 if (remove.flags != 0) {
1235 pr_err("nvm: no flags supported\n");
1239 list_for_each_entry(dev, &nvm_devices, devices) {
1240 ret = nvm_remove_tgt(dev, &remove);
1248 /* kept for compatibility reasons */
1249 static long nvm_ioctl_dev_init(struct file *file, void __user *arg)
1251 struct nvm_ioctl_dev_init init;
1253 if (!capable(CAP_SYS_ADMIN))
1256 if (copy_from_user(&init, arg, sizeof(struct nvm_ioctl_dev_init)))
1259 if (init.flags != 0) {
1260 pr_err("nvm: no flags supported\n");
1267 /* Kept for compatibility reasons */
1268 static long nvm_ioctl_dev_factory(struct file *file, void __user *arg)
1270 struct nvm_ioctl_dev_factory fact;
1272 if (!capable(CAP_SYS_ADMIN))
1275 if (copy_from_user(&fact, arg, sizeof(struct nvm_ioctl_dev_factory)))
1278 fact.dev[DISK_NAME_LEN - 1] = '\0';
1280 if (fact.flags & ~(NVM_FACTORY_NR_BITS - 1))
1286 static long nvm_ctl_ioctl(struct file *file, uint cmd, unsigned long arg)
1288 void __user *argp = (void __user *)arg;
1292 return nvm_ioctl_info(file, argp);
1293 case NVM_GET_DEVICES:
1294 return nvm_ioctl_get_devices(file, argp);
1295 case NVM_DEV_CREATE:
1296 return nvm_ioctl_dev_create(file, argp);
1297 case NVM_DEV_REMOVE:
1298 return nvm_ioctl_dev_remove(file, argp);
1300 return nvm_ioctl_dev_init(file, argp);
1301 case NVM_DEV_FACTORY:
1302 return nvm_ioctl_dev_factory(file, argp);
1307 static const struct file_operations _ctl_fops = {
1308 .open = nonseekable_open,
1309 .unlocked_ioctl = nvm_ctl_ioctl,
1310 .owner = THIS_MODULE,
1311 .llseek = noop_llseek,
1314 static struct miscdevice _nvm_misc = {
1315 .minor = MISC_DYNAMIC_MINOR,
1317 .nodename = "lightnvm/control",
1320 builtin_misc_device(_nvm_misc);