1
2
3
4
5
6
7
8
9#include <linux/spinlock.h>
10#include <linux/module.h>
11#include <linux/kernel.h>
12#include <linux/timer.h>
13#include <linux/parser.h>
14#include <linux/vmalloc.h>
15#include <linux/uio_driver.h>
16#include <linux/xarray.h>
17#include <linux/stringify.h>
18#include <linux/bitops.h>
19#include <linux/highmem.h>
20#include <linux/configfs.h>
21#include <linux/mutex.h>
22#include <linux/workqueue.h>
23#include <net/genetlink.h>
24#include <scsi/scsi_common.h>
25#include <scsi/scsi_proto.h>
26#include <target/target_core_base.h>
27#include <target/target_core_fabric.h>
28#include <target/target_core_backend.h>
29
30#include <linux/target_core_user.h>
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61#define TCMU_TIME_OUT (30 * MSEC_PER_SEC)
62
63
64#define MB_CMDR_SIZE (8 * 1024 * 1024)
65
66#define CMDR_OFF sizeof(struct tcmu_mailbox)
67#define CMDR_SIZE (MB_CMDR_SIZE - CMDR_OFF)
68
69
70
71
72
73#define DATA_PAGES_PER_BLK_DEF 1
74#define DATA_AREA_PAGES_DEF (256 * 1024)
75
76#define TCMU_MBS_TO_PAGES(_mbs) ((size_t)_mbs << (20 - PAGE_SHIFT))
77#define TCMU_PAGES_TO_MBS(_pages) (_pages >> (20 - PAGE_SHIFT))
78
79
80
81
82
83#define TCMU_GLOBAL_MAX_PAGES_DEF (512 * 1024)
84
85static u8 tcmu_kern_cmd_reply_supported;
86static u8 tcmu_netlink_blocked;
87
88static struct device *tcmu_root_device;
89
90struct tcmu_hba {
91 u32 host_id;
92};
93
94#define TCMU_CONFIG_LEN 256
95
96static DEFINE_MUTEX(tcmu_nl_cmd_mutex);
97static LIST_HEAD(tcmu_nl_cmd_list);
98
99struct tcmu_dev;
100
101struct tcmu_nl_cmd {
102
103 struct completion complete;
104 struct list_head nl_list;
105 struct tcmu_dev *udev;
106 int cmd;
107 int status;
108};
109
110struct tcmu_dev {
111 struct list_head node;
112 struct kref kref;
113
114 struct se_device se_dev;
115 struct se_dev_plug se_plug;
116
117 char *name;
118 struct se_hba *hba;
119
120#define TCMU_DEV_BIT_OPEN 0
121#define TCMU_DEV_BIT_BROKEN 1
122#define TCMU_DEV_BIT_BLOCKED 2
123#define TCMU_DEV_BIT_TMR_NOTIFY 3
124#define TCMU_DEV_BIT_PLUGGED 4
125 unsigned long flags;
126
127 struct uio_info uio_info;
128
129 struct inode *inode;
130
131 uint64_t dev_size;
132
133 struct tcmu_mailbox *mb_addr;
134 void *cmdr;
135 u32 cmdr_size;
136 u32 cmdr_last_cleaned;
137
138
139 size_t data_off;
140 int data_area_mb;
141 uint32_t max_blocks;
142 size_t mmap_pages;
143
144 struct mutex cmdr_lock;
145 struct list_head qfull_queue;
146 struct list_head tmr_queue;
147
148 uint32_t dbi_max;
149 uint32_t dbi_thresh;
150 unsigned long *data_bitmap;
151 struct xarray data_pages;
152 uint32_t data_pages_per_blk;
153 uint32_t data_blk_size;
154
155 struct xarray commands;
156
157 struct timer_list cmd_timer;
158 unsigned int cmd_time_out;
159 struct list_head inflight_queue;
160
161 struct timer_list qfull_timer;
162 int qfull_time_out;
163
164 struct list_head timedout_entry;
165
166 struct tcmu_nl_cmd curr_nl_cmd;
167
168 char dev_config[TCMU_CONFIG_LEN];
169
170 int nl_reply_supported;
171};
172
173#define TCMU_DEV(_se_dev) container_of(_se_dev, struct tcmu_dev, se_dev)
174
175struct tcmu_cmd {
176 struct se_cmd *se_cmd;
177 struct tcmu_dev *tcmu_dev;
178 struct list_head queue_entry;
179
180 uint16_t cmd_id;
181
182
183
184 uint32_t dbi_cnt;
185 uint32_t dbi_bidi_cnt;
186 uint32_t dbi_cur;
187 uint32_t *dbi;
188
189 uint32_t data_len_bidi;
190
191 unsigned long deadline;
192
193#define TCMU_CMD_BIT_EXPIRED 0
194 unsigned long flags;
195};
196
197struct tcmu_tmr {
198 struct list_head queue_entry;
199
200 uint8_t tmr_type;
201 uint32_t tmr_cmd_cnt;
202 int16_t tmr_cmd_ids[];
203};
204
205
206
207
208
209
210
211
212
213
214
215static DEFINE_MUTEX(root_udev_mutex);
216static LIST_HEAD(root_udev);
217
218static DEFINE_SPINLOCK(timed_out_udevs_lock);
219static LIST_HEAD(timed_out_udevs);
220
221static struct kmem_cache *tcmu_cmd_cache;
222
223static atomic_t global_page_count = ATOMIC_INIT(0);
224static struct delayed_work tcmu_unmap_work;
225static int tcmu_global_max_pages = TCMU_GLOBAL_MAX_PAGES_DEF;
226
227static int tcmu_set_global_max_data_area(const char *str,
228 const struct kernel_param *kp)
229{
230 int ret, max_area_mb;
231
232 ret = kstrtoint(str, 10, &max_area_mb);
233 if (ret)
234 return -EINVAL;
235
236 if (max_area_mb <= 0) {
237 pr_err("global_max_data_area must be larger than 0.\n");
238 return -EINVAL;
239 }
240
241 tcmu_global_max_pages = TCMU_MBS_TO_PAGES(max_area_mb);
242 if (atomic_read(&global_page_count) > tcmu_global_max_pages)
243 schedule_delayed_work(&tcmu_unmap_work, 0);
244 else
245 cancel_delayed_work_sync(&tcmu_unmap_work);
246
247 return 0;
248}
249
250static int tcmu_get_global_max_data_area(char *buffer,
251 const struct kernel_param *kp)
252{
253 return sprintf(buffer, "%d\n", TCMU_PAGES_TO_MBS(tcmu_global_max_pages));
254}
255
256static const struct kernel_param_ops tcmu_global_max_data_area_op = {
257 .set = tcmu_set_global_max_data_area,
258 .get = tcmu_get_global_max_data_area,
259};
260
261module_param_cb(global_max_data_area_mb, &tcmu_global_max_data_area_op, NULL,
262 S_IWUSR | S_IRUGO);
263MODULE_PARM_DESC(global_max_data_area_mb,
264 "Max MBs allowed to be allocated to all the tcmu device's "
265 "data areas.");
266
267static int tcmu_get_block_netlink(char *buffer,
268 const struct kernel_param *kp)
269{
270 return sprintf(buffer, "%s\n", tcmu_netlink_blocked ?
271 "blocked" : "unblocked");
272}
273
274static int tcmu_set_block_netlink(const char *str,
275 const struct kernel_param *kp)
276{
277 int ret;
278 u8 val;
279
280 ret = kstrtou8(str, 0, &val);
281 if (ret < 0)
282 return ret;
283
284 if (val > 1) {
285 pr_err("Invalid block netlink value %u\n", val);
286 return -EINVAL;
287 }
288
289 tcmu_netlink_blocked = val;
290 return 0;
291}
292
293static const struct kernel_param_ops tcmu_block_netlink_op = {
294 .set = tcmu_set_block_netlink,
295 .get = tcmu_get_block_netlink,
296};
297
298module_param_cb(block_netlink, &tcmu_block_netlink_op, NULL, S_IWUSR | S_IRUGO);
299MODULE_PARM_DESC(block_netlink, "Block new netlink commands.");
300
301static int tcmu_fail_netlink_cmd(struct tcmu_nl_cmd *nl_cmd)
302{
303 struct tcmu_dev *udev = nl_cmd->udev;
304
305 if (!tcmu_netlink_blocked) {
306 pr_err("Could not reset device's netlink interface. Netlink is not blocked.\n");
307 return -EBUSY;
308 }
309
310 if (nl_cmd->cmd != TCMU_CMD_UNSPEC) {
311 pr_debug("Aborting nl cmd %d on %s\n", nl_cmd->cmd, udev->name);
312 nl_cmd->status = -EINTR;
313 list_del(&nl_cmd->nl_list);
314 complete(&nl_cmd->complete);
315 }
316 return 0;
317}
318
319static int tcmu_set_reset_netlink(const char *str,
320 const struct kernel_param *kp)
321{
322 struct tcmu_nl_cmd *nl_cmd, *tmp_cmd;
323 int ret;
324 u8 val;
325
326 ret = kstrtou8(str, 0, &val);
327 if (ret < 0)
328 return ret;
329
330 if (val != 1) {
331 pr_err("Invalid reset netlink value %u\n", val);
332 return -EINVAL;
333 }
334
335 mutex_lock(&tcmu_nl_cmd_mutex);
336 list_for_each_entry_safe(nl_cmd, tmp_cmd, &tcmu_nl_cmd_list, nl_list) {
337 ret = tcmu_fail_netlink_cmd(nl_cmd);
338 if (ret)
339 break;
340 }
341 mutex_unlock(&tcmu_nl_cmd_mutex);
342
343 return ret;
344}
345
346static const struct kernel_param_ops tcmu_reset_netlink_op = {
347 .set = tcmu_set_reset_netlink,
348};
349
350module_param_cb(reset_netlink, &tcmu_reset_netlink_op, NULL, S_IWUSR);
351MODULE_PARM_DESC(reset_netlink, "Reset netlink commands.");
352
353
354enum tcmu_multicast_groups {
355 TCMU_MCGRP_CONFIG,
356};
357
358static const struct genl_multicast_group tcmu_mcgrps[] = {
359 [TCMU_MCGRP_CONFIG] = { .name = "config", },
360};
361
362static struct nla_policy tcmu_attr_policy[TCMU_ATTR_MAX+1] = {
363 [TCMU_ATTR_DEVICE] = { .type = NLA_STRING },
364 [TCMU_ATTR_MINOR] = { .type = NLA_U32 },
365 [TCMU_ATTR_CMD_STATUS] = { .type = NLA_S32 },
366 [TCMU_ATTR_DEVICE_ID] = { .type = NLA_U32 },
367 [TCMU_ATTR_SUPP_KERN_CMD_REPLY] = { .type = NLA_U8 },
368};
369
370static int tcmu_genl_cmd_done(struct genl_info *info, int completed_cmd)
371{
372 struct tcmu_dev *udev = NULL;
373 struct tcmu_nl_cmd *nl_cmd;
374 int dev_id, rc, ret = 0;
375
376 if (!info->attrs[TCMU_ATTR_CMD_STATUS] ||
377 !info->attrs[TCMU_ATTR_DEVICE_ID]) {
378 printk(KERN_ERR "TCMU_ATTR_CMD_STATUS or TCMU_ATTR_DEVICE_ID not set, doing nothing\n");
379 return -EINVAL;
380 }
381
382 dev_id = nla_get_u32(info->attrs[TCMU_ATTR_DEVICE_ID]);
383 rc = nla_get_s32(info->attrs[TCMU_ATTR_CMD_STATUS]);
384
385 mutex_lock(&tcmu_nl_cmd_mutex);
386 list_for_each_entry(nl_cmd, &tcmu_nl_cmd_list, nl_list) {
387 if (nl_cmd->udev->se_dev.dev_index == dev_id) {
388 udev = nl_cmd->udev;
389 break;
390 }
391 }
392
393 if (!udev) {
394 pr_err("tcmu nl cmd %u/%d completion could not find device with dev id %u.\n",
395 completed_cmd, rc, dev_id);
396 ret = -ENODEV;
397 goto unlock;
398 }
399 list_del(&nl_cmd->nl_list);
400
401 pr_debug("%s genl cmd done got id %d curr %d done %d rc %d stat %d\n",
402 udev->name, dev_id, nl_cmd->cmd, completed_cmd, rc,
403 nl_cmd->status);
404
405 if (nl_cmd->cmd != completed_cmd) {
406 pr_err("Mismatched commands on %s (Expecting reply for %d. Current %d).\n",
407 udev->name, completed_cmd, nl_cmd->cmd);
408 ret = -EINVAL;
409 goto unlock;
410 }
411
412 nl_cmd->status = rc;
413 complete(&nl_cmd->complete);
414unlock:
415 mutex_unlock(&tcmu_nl_cmd_mutex);
416 return ret;
417}
418
419static int tcmu_genl_rm_dev_done(struct sk_buff *skb, struct genl_info *info)
420{
421 return tcmu_genl_cmd_done(info, TCMU_CMD_REMOVED_DEVICE);
422}
423
424static int tcmu_genl_add_dev_done(struct sk_buff *skb, struct genl_info *info)
425{
426 return tcmu_genl_cmd_done(info, TCMU_CMD_ADDED_DEVICE);
427}
428
429static int tcmu_genl_reconfig_dev_done(struct sk_buff *skb,
430 struct genl_info *info)
431{
432 return tcmu_genl_cmd_done(info, TCMU_CMD_RECONFIG_DEVICE);
433}
434
435static int tcmu_genl_set_features(struct sk_buff *skb, struct genl_info *info)
436{
437 if (info->attrs[TCMU_ATTR_SUPP_KERN_CMD_REPLY]) {
438 tcmu_kern_cmd_reply_supported =
439 nla_get_u8(info->attrs[TCMU_ATTR_SUPP_KERN_CMD_REPLY]);
440 printk(KERN_INFO "tcmu daemon: command reply support %u.\n",
441 tcmu_kern_cmd_reply_supported);
442 }
443
444 return 0;
445}
446
447static const struct genl_small_ops tcmu_genl_ops[] = {
448 {
449 .cmd = TCMU_CMD_SET_FEATURES,
450 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
451 .flags = GENL_ADMIN_PERM,
452 .doit = tcmu_genl_set_features,
453 },
454 {
455 .cmd = TCMU_CMD_ADDED_DEVICE_DONE,
456 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
457 .flags = GENL_ADMIN_PERM,
458 .doit = tcmu_genl_add_dev_done,
459 },
460 {
461 .cmd = TCMU_CMD_REMOVED_DEVICE_DONE,
462 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
463 .flags = GENL_ADMIN_PERM,
464 .doit = tcmu_genl_rm_dev_done,
465 },
466 {
467 .cmd = TCMU_CMD_RECONFIG_DEVICE_DONE,
468 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
469 .flags = GENL_ADMIN_PERM,
470 .doit = tcmu_genl_reconfig_dev_done,
471 },
472};
473
474
475static struct genl_family tcmu_genl_family __ro_after_init = {
476 .module = THIS_MODULE,
477 .hdrsize = 0,
478 .name = "TCM-USER",
479 .version = 2,
480 .maxattr = TCMU_ATTR_MAX,
481 .policy = tcmu_attr_policy,
482 .mcgrps = tcmu_mcgrps,
483 .n_mcgrps = ARRAY_SIZE(tcmu_mcgrps),
484 .netnsok = true,
485 .small_ops = tcmu_genl_ops,
486 .n_small_ops = ARRAY_SIZE(tcmu_genl_ops),
487};
488
489#define tcmu_cmd_set_dbi_cur(cmd, index) ((cmd)->dbi_cur = (index))
490#define tcmu_cmd_reset_dbi_cur(cmd) tcmu_cmd_set_dbi_cur(cmd, 0)
491#define tcmu_cmd_set_dbi(cmd, index) ((cmd)->dbi[(cmd)->dbi_cur++] = (index))
492#define tcmu_cmd_get_dbi(cmd) ((cmd)->dbi[(cmd)->dbi_cur++])
493
494static void tcmu_cmd_free_data(struct tcmu_cmd *tcmu_cmd, uint32_t len)
495{
496 struct tcmu_dev *udev = tcmu_cmd->tcmu_dev;
497 uint32_t i;
498
499 for (i = 0; i < len; i++)
500 clear_bit(tcmu_cmd->dbi[i], udev->data_bitmap);
501}
502
503static inline int tcmu_get_empty_block(struct tcmu_dev *udev,
504 struct tcmu_cmd *tcmu_cmd,
505 int prev_dbi, int length, int *iov_cnt)
506{
507 XA_STATE(xas, &udev->data_pages, 0);
508 struct page *page;
509 int i, cnt, dbi, dpi;
510 int page_cnt = DIV_ROUND_UP(length, PAGE_SIZE);
511
512 dbi = find_first_zero_bit(udev->data_bitmap, udev->dbi_thresh);
513 if (dbi == udev->dbi_thresh)
514 return -1;
515
516 dpi = dbi * udev->data_pages_per_blk;
517
518 xas_set(&xas, dpi);
519 rcu_read_lock();
520 for (cnt = 0; xas_next(&xas) && cnt < page_cnt;)
521 cnt++;
522 rcu_read_unlock();
523
524 for (i = cnt; i < page_cnt; i++) {
525
526 page = alloc_page(GFP_NOIO);
527 if (!page)
528 break;
529
530 if (xa_store(&udev->data_pages, dpi + i, page, GFP_NOIO)) {
531 __free_page(page);
532 break;
533 }
534 }
535 if (atomic_add_return(i - cnt, &global_page_count) >
536 tcmu_global_max_pages)
537 schedule_delayed_work(&tcmu_unmap_work, 0);
538
539 if (i && dbi > udev->dbi_max)
540 udev->dbi_max = dbi;
541
542 set_bit(dbi, udev->data_bitmap);
543 tcmu_cmd_set_dbi(tcmu_cmd, dbi);
544
545 if (dbi != prev_dbi + 1)
546 *iov_cnt += 1;
547
548 return i == page_cnt ? dbi : -1;
549}
550
551static int tcmu_get_empty_blocks(struct tcmu_dev *udev,
552 struct tcmu_cmd *tcmu_cmd, int length)
553{
554
555 int dbi = -2;
556 int blk_data_len, iov_cnt = 0;
557 uint32_t blk_size = udev->data_blk_size;
558
559 for (; length > 0; length -= blk_size) {
560 blk_data_len = min_t(uint32_t, length, blk_size);
561 dbi = tcmu_get_empty_block(udev, tcmu_cmd, dbi, blk_data_len,
562 &iov_cnt);
563 if (dbi < 0)
564 return -1;
565 }
566 return iov_cnt;
567}
568
569static inline void tcmu_free_cmd(struct tcmu_cmd *tcmu_cmd)
570{
571 kfree(tcmu_cmd->dbi);
572 kmem_cache_free(tcmu_cmd_cache, tcmu_cmd);
573}
574
575static inline void tcmu_cmd_set_block_cnts(struct tcmu_cmd *cmd)
576{
577 int i, len;
578 struct se_cmd *se_cmd = cmd->se_cmd;
579 uint32_t blk_size = cmd->tcmu_dev->data_blk_size;
580
581 cmd->dbi_cnt = DIV_ROUND_UP(se_cmd->data_length, blk_size);
582
583 if (se_cmd->se_cmd_flags & SCF_BIDI) {
584 BUG_ON(!(se_cmd->t_bidi_data_sg && se_cmd->t_bidi_data_nents));
585 for (i = 0, len = 0; i < se_cmd->t_bidi_data_nents; i++)
586 len += se_cmd->t_bidi_data_sg[i].length;
587 cmd->dbi_bidi_cnt = DIV_ROUND_UP(len, blk_size);
588 cmd->dbi_cnt += cmd->dbi_bidi_cnt;
589 cmd->data_len_bidi = len;
590 }
591}
592
593static int new_block_to_iov(struct tcmu_dev *udev, struct tcmu_cmd *cmd,
594 struct iovec **iov, int prev_dbi, int len)
595{
596
597 int dbi = tcmu_cmd_get_dbi(cmd);
598
599
600 len = min_t(int, len, udev->data_blk_size);
601
602
603
604
605
606 if (dbi != prev_dbi + 1) {
607
608 if (prev_dbi >= 0)
609 (*iov)++;
610
611 (*iov)->iov_base = (void __user *)
612 (udev->data_off + dbi * udev->data_blk_size);
613 }
614 (*iov)->iov_len += len;
615
616 return dbi;
617}
618
619static void tcmu_setup_iovs(struct tcmu_dev *udev, struct tcmu_cmd *cmd,
620 struct iovec **iov, int data_length)
621{
622
623 int dbi = -2;
624
625
626 for (; data_length > 0; data_length -= udev->data_blk_size)
627 dbi = new_block_to_iov(udev, cmd, iov, dbi, data_length);
628}
629
630static struct tcmu_cmd *tcmu_alloc_cmd(struct se_cmd *se_cmd)
631{
632 struct se_device *se_dev = se_cmd->se_dev;
633 struct tcmu_dev *udev = TCMU_DEV(se_dev);
634 struct tcmu_cmd *tcmu_cmd;
635
636 tcmu_cmd = kmem_cache_zalloc(tcmu_cmd_cache, GFP_NOIO);
637 if (!tcmu_cmd)
638 return NULL;
639
640 INIT_LIST_HEAD(&tcmu_cmd->queue_entry);
641 tcmu_cmd->se_cmd = se_cmd;
642 tcmu_cmd->tcmu_dev = udev;
643
644 tcmu_cmd_set_block_cnts(tcmu_cmd);
645 tcmu_cmd->dbi = kcalloc(tcmu_cmd->dbi_cnt, sizeof(uint32_t),
646 GFP_NOIO);
647 if (!tcmu_cmd->dbi) {
648 kmem_cache_free(tcmu_cmd_cache, tcmu_cmd);
649 return NULL;
650 }
651
652 return tcmu_cmd;
653}
654
655static inline void tcmu_flush_dcache_range(void *vaddr, size_t size)
656{
657 unsigned long offset = offset_in_page(vaddr);
658 void *start = vaddr - offset;
659
660 size = round_up(size+offset, PAGE_SIZE);
661
662 while (size) {
663 flush_dcache_page(vmalloc_to_page(start));
664 start += PAGE_SIZE;
665 size -= PAGE_SIZE;
666 }
667}
668
669
670
671
672
673static inline size_t spc_used(size_t head, size_t tail, size_t size)
674{
675 int diff = head - tail;
676
677 if (diff >= 0)
678 return diff;
679 else
680 return size + diff;
681}
682
683static inline size_t spc_free(size_t head, size_t tail, size_t size)
684{
685
686 return (size - spc_used(head, tail, size) - 1);
687}
688
689static inline size_t head_to_end(size_t head, size_t size)
690{
691 return size - head;
692}
693
694#define UPDATE_HEAD(head, used, size) smp_store_release(&head, ((head % size) + used) % size)
695
696#define TCMU_SG_TO_DATA_AREA 1
697#define TCMU_DATA_AREA_TO_SG 2
698
699static inline void tcmu_copy_data(struct tcmu_dev *udev,
700 struct tcmu_cmd *tcmu_cmd, uint32_t direction,
701 struct scatterlist *sg, unsigned int sg_nents,
702 struct iovec **iov, size_t data_len)
703{
704
705 int dbi = -2;
706 size_t page_remaining, cp_len;
707 int page_cnt, page_inx, dpi;
708 struct sg_mapping_iter sg_iter;
709 unsigned int sg_flags;
710 struct page *page;
711 void *data_page_start, *data_addr;
712
713 if (direction == TCMU_SG_TO_DATA_AREA)
714 sg_flags = SG_MITER_ATOMIC | SG_MITER_FROM_SG;
715 else
716 sg_flags = SG_MITER_ATOMIC | SG_MITER_TO_SG;
717 sg_miter_start(&sg_iter, sg, sg_nents, sg_flags);
718
719 while (data_len) {
720 if (direction == TCMU_SG_TO_DATA_AREA)
721 dbi = new_block_to_iov(udev, tcmu_cmd, iov, dbi,
722 data_len);
723 else
724 dbi = tcmu_cmd_get_dbi(tcmu_cmd);
725
726 page_cnt = DIV_ROUND_UP(data_len, PAGE_SIZE);
727 if (page_cnt > udev->data_pages_per_blk)
728 page_cnt = udev->data_pages_per_blk;
729
730 dpi = dbi * udev->data_pages_per_blk;
731 for (page_inx = 0; page_inx < page_cnt && data_len;
732 page_inx++, dpi++) {
733 page = xa_load(&udev->data_pages, dpi);
734
735 if (direction == TCMU_DATA_AREA_TO_SG)
736 flush_dcache_page(page);
737 data_page_start = kmap_atomic(page);
738 page_remaining = PAGE_SIZE;
739
740 while (page_remaining && data_len) {
741 if (!sg_miter_next(&sg_iter)) {
742
743 data_len = 0;
744 pr_debug("%s: aborting data copy due to exhausted sg_list\n",
745 __func__);
746 break;
747 }
748 cp_len = min3(sg_iter.length, page_remaining,
749 data_len);
750
751 data_addr = data_page_start +
752 PAGE_SIZE - page_remaining;
753 if (direction == TCMU_SG_TO_DATA_AREA)
754 memcpy(data_addr, sg_iter.addr, cp_len);
755 else
756 memcpy(sg_iter.addr, data_addr, cp_len);
757
758 data_len -= cp_len;
759 page_remaining -= cp_len;
760 sg_iter.consumed = cp_len;
761 }
762 sg_miter_stop(&sg_iter);
763
764 kunmap_atomic(data_page_start);
765 if (direction == TCMU_SG_TO_DATA_AREA)
766 flush_dcache_page(page);
767 }
768 }
769}
770
771static void scatter_data_area(struct tcmu_dev *udev, struct tcmu_cmd *tcmu_cmd,
772 struct iovec **iov)
773{
774 struct se_cmd *se_cmd = tcmu_cmd->se_cmd;
775
776 tcmu_copy_data(udev, tcmu_cmd, TCMU_SG_TO_DATA_AREA, se_cmd->t_data_sg,
777 se_cmd->t_data_nents, iov, se_cmd->data_length);
778}
779
780static void gather_data_area(struct tcmu_dev *udev, struct tcmu_cmd *tcmu_cmd,
781 bool bidi, uint32_t read_len)
782{
783 struct se_cmd *se_cmd = tcmu_cmd->se_cmd;
784 struct scatterlist *data_sg;
785 unsigned int data_nents;
786
787 if (!bidi) {
788 data_sg = se_cmd->t_data_sg;
789 data_nents = se_cmd->t_data_nents;
790 } else {
791
792
793
794
795
796 tcmu_cmd_set_dbi_cur(tcmu_cmd,
797 tcmu_cmd->dbi_cnt - tcmu_cmd->dbi_bidi_cnt);
798
799 data_sg = se_cmd->t_bidi_data_sg;
800 data_nents = se_cmd->t_bidi_data_nents;
801 }
802
803 tcmu_copy_data(udev, tcmu_cmd, TCMU_DATA_AREA_TO_SG, data_sg,
804 data_nents, NULL, read_len);
805}
806
807static inline size_t spc_bitmap_free(unsigned long *bitmap, uint32_t thresh)
808{
809 return thresh - bitmap_weight(bitmap, thresh);
810}
811
812
813
814
815
816
817static bool is_ring_space_avail(struct tcmu_dev *udev, size_t cmd_size)
818{
819 struct tcmu_mailbox *mb = udev->mb_addr;
820 size_t space, cmd_needed;
821 u32 cmd_head;
822
823 tcmu_flush_dcache_range(mb, sizeof(*mb));
824
825 cmd_head = mb->cmd_head % udev->cmdr_size;
826
827
828
829
830
831 if (head_to_end(cmd_head, udev->cmdr_size) >= cmd_size)
832 cmd_needed = cmd_size;
833 else
834 cmd_needed = cmd_size + head_to_end(cmd_head, udev->cmdr_size);
835
836 space = spc_free(cmd_head, udev->cmdr_last_cleaned, udev->cmdr_size);
837 if (space < cmd_needed) {
838 pr_debug("no cmd space: %u %u %u\n", cmd_head,
839 udev->cmdr_last_cleaned, udev->cmdr_size);
840 return false;
841 }
842 return true;
843}
844
845
846
847
848
849
850
851static int tcmu_alloc_data_space(struct tcmu_dev *udev, struct tcmu_cmd *cmd,
852 int *iov_bidi_cnt)
853{
854 int space, iov_cnt = 0, ret = 0;
855
856 if (!cmd->dbi_cnt)
857 goto wr_iov_cnts;
858
859
860 space = spc_bitmap_free(udev->data_bitmap, udev->dbi_thresh);
861 if (space < cmd->dbi_cnt) {
862 unsigned long blocks_left =
863 (udev->max_blocks - udev->dbi_thresh) + space;
864
865 if (blocks_left < cmd->dbi_cnt) {
866 pr_debug("no data space: only %lu available, but ask for %u\n",
867 blocks_left * udev->data_blk_size,
868 cmd->dbi_cnt * udev->data_blk_size);
869 return -1;
870 }
871
872 udev->dbi_thresh += cmd->dbi_cnt;
873 if (udev->dbi_thresh > udev->max_blocks)
874 udev->dbi_thresh = udev->max_blocks;
875 }
876
877 iov_cnt = tcmu_get_empty_blocks(udev, cmd, cmd->se_cmd->data_length);
878 if (iov_cnt < 0)
879 return -1;
880
881 if (cmd->dbi_bidi_cnt) {
882 ret = tcmu_get_empty_blocks(udev, cmd, cmd->data_len_bidi);
883 if (ret < 0)
884 return -1;
885 }
886wr_iov_cnts:
887 *iov_bidi_cnt = ret;
888 return iov_cnt + ret;
889}
890
891static inline size_t tcmu_cmd_get_base_cmd_size(size_t iov_cnt)
892{
893 return max(offsetof(struct tcmu_cmd_entry, req.iov[iov_cnt]),
894 sizeof(struct tcmu_cmd_entry));
895}
896
897static inline size_t tcmu_cmd_get_cmd_size(struct tcmu_cmd *tcmu_cmd,
898 size_t base_command_size)
899{
900 struct se_cmd *se_cmd = tcmu_cmd->se_cmd;
901 size_t command_size;
902
903 command_size = base_command_size +
904 round_up(scsi_command_size(se_cmd->t_task_cdb),
905 TCMU_OP_ALIGN_SIZE);
906
907 WARN_ON(command_size & (TCMU_OP_ALIGN_SIZE-1));
908
909 return command_size;
910}
911
912static void tcmu_setup_cmd_timer(struct tcmu_cmd *tcmu_cmd, unsigned int tmo,
913 struct timer_list *timer)
914{
915 if (!tmo)
916 return;
917
918 tcmu_cmd->deadline = round_jiffies_up(jiffies + msecs_to_jiffies(tmo));
919 if (!timer_pending(timer))
920 mod_timer(timer, tcmu_cmd->deadline);
921
922 pr_debug("Timeout set up for cmd %p, dev = %s, tmo = %lu\n", tcmu_cmd,
923 tcmu_cmd->tcmu_dev->name, tmo / MSEC_PER_SEC);
924}
925
926static int add_to_qfull_queue(struct tcmu_cmd *tcmu_cmd)
927{
928 struct tcmu_dev *udev = tcmu_cmd->tcmu_dev;
929 unsigned int tmo;
930
931
932
933
934
935 if (!udev->qfull_time_out)
936 return -ETIMEDOUT;
937 else if (udev->qfull_time_out > 0)
938 tmo = udev->qfull_time_out;
939 else if (udev->cmd_time_out)
940 tmo = udev->cmd_time_out;
941 else
942 tmo = TCMU_TIME_OUT;
943
944 tcmu_setup_cmd_timer(tcmu_cmd, tmo, &udev->qfull_timer);
945
946 list_add_tail(&tcmu_cmd->queue_entry, &udev->qfull_queue);
947 pr_debug("adding cmd %p on dev %s to ring space wait queue\n",
948 tcmu_cmd, udev->name);
949 return 0;
950}
951
952static uint32_t ring_insert_padding(struct tcmu_dev *udev, size_t cmd_size)
953{
954 struct tcmu_cmd_entry_hdr *hdr;
955 struct tcmu_mailbox *mb = udev->mb_addr;
956 uint32_t cmd_head = mb->cmd_head % udev->cmdr_size;
957
958
959 if (head_to_end(cmd_head, udev->cmdr_size) < cmd_size) {
960 size_t pad_size = head_to_end(cmd_head, udev->cmdr_size);
961
962 hdr = udev->cmdr + cmd_head;
963 tcmu_hdr_set_op(&hdr->len_op, TCMU_OP_PAD);
964 tcmu_hdr_set_len(&hdr->len_op, pad_size);
965 hdr->cmd_id = 0;
966 hdr->kflags = 0;
967 hdr->uflags = 0;
968 tcmu_flush_dcache_range(hdr, sizeof(*hdr));
969
970 UPDATE_HEAD(mb->cmd_head, pad_size, udev->cmdr_size);
971 tcmu_flush_dcache_range(mb, sizeof(*mb));
972
973 cmd_head = mb->cmd_head % udev->cmdr_size;
974 WARN_ON(cmd_head != 0);
975 }
976
977 return cmd_head;
978}
979
980static void tcmu_unplug_device(struct se_dev_plug *se_plug)
981{
982 struct se_device *se_dev = se_plug->se_dev;
983 struct tcmu_dev *udev = TCMU_DEV(se_dev);
984
985 clear_bit(TCMU_DEV_BIT_PLUGGED, &udev->flags);
986 uio_event_notify(&udev->uio_info);
987}
988
989static struct se_dev_plug *tcmu_plug_device(struct se_device *se_dev)
990{
991 struct tcmu_dev *udev = TCMU_DEV(se_dev);
992
993 if (!test_and_set_bit(TCMU_DEV_BIT_PLUGGED, &udev->flags))
994 return &udev->se_plug;
995
996 return NULL;
997}
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009static int queue_cmd_ring(struct tcmu_cmd *tcmu_cmd, sense_reason_t *scsi_err)
1010{
1011 struct tcmu_dev *udev = tcmu_cmd->tcmu_dev;
1012 struct se_cmd *se_cmd = tcmu_cmd->se_cmd;
1013 size_t base_command_size, command_size;
1014 struct tcmu_mailbox *mb = udev->mb_addr;
1015 struct tcmu_cmd_entry *entry;
1016 struct iovec *iov;
1017 int iov_cnt, iov_bidi_cnt;
1018 uint32_t cmd_id, cmd_head;
1019 uint64_t cdb_off;
1020 uint32_t blk_size = udev->data_blk_size;
1021
1022 size_t data_length = (size_t)tcmu_cmd->dbi_cnt * blk_size;
1023
1024 *scsi_err = TCM_NO_SENSE;
1025
1026 if (test_bit(TCMU_DEV_BIT_BLOCKED, &udev->flags)) {
1027 *scsi_err = TCM_LUN_BUSY;
1028 return -1;
1029 }
1030
1031 if (test_bit(TCMU_DEV_BIT_BROKEN, &udev->flags)) {
1032 *scsi_err = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
1033 return -1;
1034 }
1035
1036 if (!list_empty(&udev->qfull_queue))
1037 goto queue;
1038
1039 if (data_length > (size_t)udev->max_blocks * blk_size) {
1040 pr_warn("TCMU: Request of size %zu is too big for %zu data area\n",
1041 data_length, (size_t)udev->max_blocks * blk_size);
1042 *scsi_err = TCM_INVALID_CDB_FIELD;
1043 return -1;
1044 }
1045
1046 iov_cnt = tcmu_alloc_data_space(udev, tcmu_cmd, &iov_bidi_cnt);
1047 if (iov_cnt < 0)
1048 goto free_and_queue;
1049
1050
1051
1052
1053
1054 base_command_size = tcmu_cmd_get_base_cmd_size(iov_cnt);
1055 command_size = tcmu_cmd_get_cmd_size(tcmu_cmd, base_command_size);
1056
1057 if (command_size > (udev->cmdr_size / 2)) {
1058 pr_warn("TCMU: Request of size %zu is too big for %u cmd ring\n",
1059 command_size, udev->cmdr_size);
1060 tcmu_cmd_free_data(tcmu_cmd, tcmu_cmd->dbi_cur);
1061 *scsi_err = TCM_INVALID_CDB_FIELD;
1062 return -1;
1063 }
1064
1065 if (!is_ring_space_avail(udev, command_size))
1066
1067
1068
1069
1070 goto free_and_queue;
1071
1072 if (xa_alloc(&udev->commands, &cmd_id, tcmu_cmd, XA_LIMIT(1, 0xffff),
1073 GFP_NOWAIT) < 0) {
1074 pr_err("tcmu: Could not allocate cmd id.\n");
1075
1076 tcmu_cmd_free_data(tcmu_cmd, tcmu_cmd->dbi_cnt);
1077 *scsi_err = TCM_OUT_OF_RESOURCES;
1078 return -1;
1079 }
1080 tcmu_cmd->cmd_id = cmd_id;
1081
1082 pr_debug("allocated cmd id %u for cmd %p dev %s\n", tcmu_cmd->cmd_id,
1083 tcmu_cmd, udev->name);
1084
1085 cmd_head = ring_insert_padding(udev, command_size);
1086
1087 entry = udev->cmdr + cmd_head;
1088 memset(entry, 0, command_size);
1089 tcmu_hdr_set_op(&entry->hdr.len_op, TCMU_OP_CMD);
1090
1091
1092 tcmu_cmd_reset_dbi_cur(tcmu_cmd);
1093 iov = &entry->req.iov[0];
1094
1095 if (se_cmd->data_direction == DMA_TO_DEVICE ||
1096 se_cmd->se_cmd_flags & SCF_BIDI)
1097 scatter_data_area(udev, tcmu_cmd, &iov);
1098 else
1099 tcmu_setup_iovs(udev, tcmu_cmd, &iov, se_cmd->data_length);
1100
1101 entry->req.iov_cnt = iov_cnt - iov_bidi_cnt;
1102
1103
1104 if (se_cmd->se_cmd_flags & SCF_BIDI) {
1105 iov++;
1106 tcmu_setup_iovs(udev, tcmu_cmd, &iov, tcmu_cmd->data_len_bidi);
1107 entry->req.iov_bidi_cnt = iov_bidi_cnt;
1108 }
1109
1110 tcmu_setup_cmd_timer(tcmu_cmd, udev->cmd_time_out, &udev->cmd_timer);
1111
1112 entry->hdr.cmd_id = tcmu_cmd->cmd_id;
1113
1114 tcmu_hdr_set_len(&entry->hdr.len_op, command_size);
1115
1116
1117 cdb_off = CMDR_OFF + cmd_head + base_command_size;
1118 memcpy((void *) mb + cdb_off, se_cmd->t_task_cdb, scsi_command_size(se_cmd->t_task_cdb));
1119 entry->req.cdb_off = cdb_off;
1120 tcmu_flush_dcache_range(entry, command_size);
1121
1122 UPDATE_HEAD(mb->cmd_head, command_size, udev->cmdr_size);
1123 tcmu_flush_dcache_range(mb, sizeof(*mb));
1124
1125 list_add_tail(&tcmu_cmd->queue_entry, &udev->inflight_queue);
1126
1127 if (!test_bit(TCMU_DEV_BIT_PLUGGED, &udev->flags))
1128 uio_event_notify(&udev->uio_info);
1129
1130 return 0;
1131
1132free_and_queue:
1133 tcmu_cmd_free_data(tcmu_cmd, tcmu_cmd->dbi_cur);
1134 tcmu_cmd_reset_dbi_cur(tcmu_cmd);
1135
1136queue:
1137 if (add_to_qfull_queue(tcmu_cmd)) {
1138 *scsi_err = TCM_OUT_OF_RESOURCES;
1139 return -1;
1140 }
1141
1142 return 1;
1143}
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154static int
1155queue_tmr_ring(struct tcmu_dev *udev, struct tcmu_tmr *tmr)
1156{
1157 struct tcmu_tmr_entry *entry;
1158 int cmd_size;
1159 int id_list_sz;
1160 struct tcmu_mailbox *mb = udev->mb_addr;
1161 uint32_t cmd_head;
1162
1163 if (test_bit(TCMU_DEV_BIT_BROKEN, &udev->flags))
1164 goto out_free;
1165
1166 id_list_sz = sizeof(tmr->tmr_cmd_ids[0]) * tmr->tmr_cmd_cnt;
1167 cmd_size = round_up(sizeof(*entry) + id_list_sz, TCMU_OP_ALIGN_SIZE);
1168
1169 if (!list_empty(&udev->tmr_queue) ||
1170 !is_ring_space_avail(udev, cmd_size)) {
1171 list_add_tail(&tmr->queue_entry, &udev->tmr_queue);
1172 pr_debug("adding tmr %p on dev %s to TMR ring space wait queue\n",
1173 tmr, udev->name);
1174 return 1;
1175 }
1176
1177 cmd_head = ring_insert_padding(udev, cmd_size);
1178
1179 entry = udev->cmdr + cmd_head;
1180 memset(entry, 0, cmd_size);
1181 tcmu_hdr_set_op(&entry->hdr.len_op, TCMU_OP_TMR);
1182 tcmu_hdr_set_len(&entry->hdr.len_op, cmd_size);
1183 entry->tmr_type = tmr->tmr_type;
1184 entry->cmd_cnt = tmr->tmr_cmd_cnt;
1185 memcpy(&entry->cmd_ids[0], &tmr->tmr_cmd_ids[0], id_list_sz);
1186 tcmu_flush_dcache_range(entry, cmd_size);
1187
1188 UPDATE_HEAD(mb->cmd_head, cmd_size, udev->cmdr_size);
1189 tcmu_flush_dcache_range(mb, sizeof(*mb));
1190
1191 uio_event_notify(&udev->uio_info);
1192
1193out_free:
1194 kfree(tmr);
1195
1196 return 0;
1197}
1198
1199static sense_reason_t
1200tcmu_queue_cmd(struct se_cmd *se_cmd)
1201{
1202 struct se_device *se_dev = se_cmd->se_dev;
1203 struct tcmu_dev *udev = TCMU_DEV(se_dev);
1204 struct tcmu_cmd *tcmu_cmd;
1205 sense_reason_t scsi_ret = TCM_CHECK_CONDITION_ABORT_CMD;
1206 int ret = -1;
1207
1208 tcmu_cmd = tcmu_alloc_cmd(se_cmd);
1209 if (!tcmu_cmd)
1210 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
1211
1212 mutex_lock(&udev->cmdr_lock);
1213 if (!(se_cmd->transport_state & CMD_T_ABORTED))
1214 ret = queue_cmd_ring(tcmu_cmd, &scsi_ret);
1215 if (ret < 0)
1216 tcmu_free_cmd(tcmu_cmd);
1217 else
1218 se_cmd->priv = tcmu_cmd;
1219 mutex_unlock(&udev->cmdr_lock);
1220 return scsi_ret;
1221}
1222
1223static void tcmu_set_next_deadline(struct list_head *queue,
1224 struct timer_list *timer)
1225{
1226 struct tcmu_cmd *cmd;
1227
1228 if (!list_empty(queue)) {
1229 cmd = list_first_entry(queue, struct tcmu_cmd, queue_entry);
1230 mod_timer(timer, cmd->deadline);
1231 } else
1232 del_timer(timer);
1233}
1234
1235static int
1236tcmu_tmr_type(enum tcm_tmreq_table tmf)
1237{
1238 switch (tmf) {
1239 case TMR_ABORT_TASK: return TCMU_TMR_ABORT_TASK;
1240 case TMR_ABORT_TASK_SET: return TCMU_TMR_ABORT_TASK_SET;
1241 case TMR_CLEAR_ACA: return TCMU_TMR_CLEAR_ACA;
1242 case TMR_CLEAR_TASK_SET: return TCMU_TMR_CLEAR_TASK_SET;
1243 case TMR_LUN_RESET: return TCMU_TMR_LUN_RESET;
1244 case TMR_TARGET_WARM_RESET: return TCMU_TMR_TARGET_WARM_RESET;
1245 case TMR_TARGET_COLD_RESET: return TCMU_TMR_TARGET_COLD_RESET;
1246 case TMR_LUN_RESET_PRO: return TCMU_TMR_LUN_RESET_PRO;
1247 default: return TCMU_TMR_UNKNOWN;
1248 }
1249}
1250
1251static void
1252tcmu_tmr_notify(struct se_device *se_dev, enum tcm_tmreq_table tmf,
1253 struct list_head *cmd_list)
1254{
1255 int i = 0, cmd_cnt = 0;
1256 bool unqueued = false;
1257 uint16_t *cmd_ids = NULL;
1258 struct tcmu_cmd *cmd;
1259 struct se_cmd *se_cmd;
1260 struct tcmu_tmr *tmr;
1261 struct tcmu_dev *udev = TCMU_DEV(se_dev);
1262
1263 mutex_lock(&udev->cmdr_lock);
1264
1265
1266 list_for_each_entry(se_cmd, cmd_list, state_list) {
1267 i++;
1268 if (!se_cmd->priv)
1269 continue;
1270 cmd = se_cmd->priv;
1271
1272 if (cmd->cmd_id) {
1273 cmd_cnt++;
1274 continue;
1275 }
1276 pr_debug("Removing aborted command %p from queue on dev %s.\n",
1277 cmd, udev->name);
1278
1279 list_del_init(&cmd->queue_entry);
1280 tcmu_free_cmd(cmd);
1281 se_cmd->priv = NULL;
1282 target_complete_cmd(se_cmd, SAM_STAT_TASK_ABORTED);
1283 unqueued = true;
1284 }
1285 if (unqueued)
1286 tcmu_set_next_deadline(&udev->qfull_queue, &udev->qfull_timer);
1287
1288 if (!test_bit(TCMU_DEV_BIT_TMR_NOTIFY, &udev->flags))
1289 goto unlock;
1290
1291 pr_debug("TMR event %d on dev %s, aborted cmds %d, afflicted cmd_ids %d\n",
1292 tcmu_tmr_type(tmf), udev->name, i, cmd_cnt);
1293
1294 tmr = kmalloc(sizeof(*tmr) + cmd_cnt * sizeof(*cmd_ids), GFP_NOIO);
1295 if (!tmr)
1296 goto unlock;
1297
1298 tmr->tmr_type = tcmu_tmr_type(tmf);
1299 tmr->tmr_cmd_cnt = cmd_cnt;
1300
1301 if (cmd_cnt != 0) {
1302 cmd_cnt = 0;
1303 list_for_each_entry(se_cmd, cmd_list, state_list) {
1304 if (!se_cmd->priv)
1305 continue;
1306 cmd = se_cmd->priv;
1307 if (cmd->cmd_id)
1308 tmr->tmr_cmd_ids[cmd_cnt++] = cmd->cmd_id;
1309 }
1310 }
1311
1312 queue_tmr_ring(udev, tmr);
1313
1314unlock:
1315 mutex_unlock(&udev->cmdr_lock);
1316}
1317
1318static void tcmu_handle_completion(struct tcmu_cmd *cmd, struct tcmu_cmd_entry *entry)
1319{
1320 struct se_cmd *se_cmd = cmd->se_cmd;
1321 struct tcmu_dev *udev = cmd->tcmu_dev;
1322 bool read_len_valid = false;
1323 uint32_t read_len;
1324
1325
1326
1327
1328
1329 if (test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags)) {
1330 WARN_ON_ONCE(se_cmd);
1331 goto out;
1332 }
1333
1334 list_del_init(&cmd->queue_entry);
1335
1336 tcmu_cmd_reset_dbi_cur(cmd);
1337
1338 if (entry->hdr.uflags & TCMU_UFLAG_UNKNOWN_OP) {
1339 pr_warn("TCMU: Userspace set UNKNOWN_OP flag on se_cmd %p\n",
1340 cmd->se_cmd);
1341 entry->rsp.scsi_status = SAM_STAT_CHECK_CONDITION;
1342 goto done;
1343 }
1344
1345 read_len = se_cmd->data_length;
1346 if (se_cmd->data_direction == DMA_FROM_DEVICE &&
1347 (entry->hdr.uflags & TCMU_UFLAG_READ_LEN) && entry->rsp.read_len) {
1348 read_len_valid = true;
1349 if (entry->rsp.read_len < read_len)
1350 read_len = entry->rsp.read_len;
1351 }
1352
1353 if (entry->rsp.scsi_status == SAM_STAT_CHECK_CONDITION) {
1354 transport_copy_sense_to_cmd(se_cmd, entry->rsp.sense_buffer);
1355 if (!read_len_valid )
1356 goto done;
1357 else
1358 se_cmd->se_cmd_flags |= SCF_TREAT_READ_AS_NORMAL;
1359 }
1360 if (se_cmd->se_cmd_flags & SCF_BIDI) {
1361
1362 gather_data_area(udev, cmd, true, read_len);
1363 } else if (se_cmd->data_direction == DMA_FROM_DEVICE) {
1364 gather_data_area(udev, cmd, false, read_len);
1365 } else if (se_cmd->data_direction == DMA_TO_DEVICE) {
1366
1367 } else if (se_cmd->data_direction != DMA_NONE) {
1368 pr_warn("TCMU: data direction was %d!\n",
1369 se_cmd->data_direction);
1370 }
1371
1372done:
1373 se_cmd->priv = NULL;
1374 if (read_len_valid) {
1375 pr_debug("read_len = %d\n", read_len);
1376 target_complete_cmd_with_length(cmd->se_cmd,
1377 entry->rsp.scsi_status, read_len);
1378 } else
1379 target_complete_cmd(cmd->se_cmd, entry->rsp.scsi_status);
1380
1381out:
1382 tcmu_cmd_free_data(cmd, cmd->dbi_cnt);
1383 tcmu_free_cmd(cmd);
1384}
1385
1386static int tcmu_run_tmr_queue(struct tcmu_dev *udev)
1387{
1388 struct tcmu_tmr *tmr, *tmp;
1389 LIST_HEAD(tmrs);
1390
1391 if (list_empty(&udev->tmr_queue))
1392 return 1;
1393
1394 pr_debug("running %s's tmr queue\n", udev->name);
1395
1396 list_splice_init(&udev->tmr_queue, &tmrs);
1397
1398 list_for_each_entry_safe(tmr, tmp, &tmrs, queue_entry) {
1399 list_del_init(&tmr->queue_entry);
1400
1401 pr_debug("removing tmr %p on dev %s from queue\n",
1402 tmr, udev->name);
1403
1404 if (queue_tmr_ring(udev, tmr)) {
1405 pr_debug("ran out of space during tmr queue run\n");
1406
1407
1408
1409
1410 list_splice_tail(&tmrs, &udev->tmr_queue);
1411 return 0;
1412 }
1413 }
1414
1415 return 1;
1416}
1417
1418static bool tcmu_handle_completions(struct tcmu_dev *udev)
1419{
1420 struct tcmu_mailbox *mb;
1421 struct tcmu_cmd *cmd;
1422 bool free_space = false;
1423
1424 if (test_bit(TCMU_DEV_BIT_BROKEN, &udev->flags)) {
1425 pr_err("ring broken, not handling completions\n");
1426 return false;
1427 }
1428
1429 mb = udev->mb_addr;
1430 tcmu_flush_dcache_range(mb, sizeof(*mb));
1431
1432 while (udev->cmdr_last_cleaned != READ_ONCE(mb->cmd_tail)) {
1433
1434 struct tcmu_cmd_entry *entry = udev->cmdr + udev->cmdr_last_cleaned;
1435
1436
1437
1438
1439
1440 size_t ring_left = head_to_end(udev->cmdr_last_cleaned,
1441 udev->cmdr_size);
1442 tcmu_flush_dcache_range(entry, ring_left < sizeof(*entry) ?
1443 ring_left : sizeof(*entry));
1444
1445 free_space = true;
1446
1447 if (tcmu_hdr_get_op(entry->hdr.len_op) == TCMU_OP_PAD ||
1448 tcmu_hdr_get_op(entry->hdr.len_op) == TCMU_OP_TMR) {
1449 UPDATE_HEAD(udev->cmdr_last_cleaned,
1450 tcmu_hdr_get_len(entry->hdr.len_op),
1451 udev->cmdr_size);
1452 continue;
1453 }
1454 WARN_ON(tcmu_hdr_get_op(entry->hdr.len_op) != TCMU_OP_CMD);
1455
1456 cmd = xa_erase(&udev->commands, entry->hdr.cmd_id);
1457 if (!cmd) {
1458 pr_err("cmd_id %u not found, ring is broken\n",
1459 entry->hdr.cmd_id);
1460 set_bit(TCMU_DEV_BIT_BROKEN, &udev->flags);
1461 return false;
1462 }
1463
1464 tcmu_handle_completion(cmd, entry);
1465
1466 UPDATE_HEAD(udev->cmdr_last_cleaned,
1467 tcmu_hdr_get_len(entry->hdr.len_op),
1468 udev->cmdr_size);
1469 }
1470 if (free_space)
1471 free_space = tcmu_run_tmr_queue(udev);
1472
1473 if (atomic_read(&global_page_count) > tcmu_global_max_pages &&
1474 xa_empty(&udev->commands) && list_empty(&udev->qfull_queue)) {
1475
1476
1477
1478
1479 schedule_delayed_work(&tcmu_unmap_work, 0);
1480 }
1481 if (udev->cmd_time_out)
1482 tcmu_set_next_deadline(&udev->inflight_queue, &udev->cmd_timer);
1483
1484 return free_space;
1485}
1486
1487static void tcmu_check_expired_ring_cmd(struct tcmu_cmd *cmd)
1488{
1489 struct se_cmd *se_cmd;
1490
1491 if (!time_after_eq(jiffies, cmd->deadline))
1492 return;
1493
1494 set_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags);
1495 list_del_init(&cmd->queue_entry);
1496 se_cmd = cmd->se_cmd;
1497 se_cmd->priv = NULL;
1498 cmd->se_cmd = NULL;
1499
1500 pr_debug("Timing out inflight cmd %u on dev %s.\n",
1501 cmd->cmd_id, cmd->tcmu_dev->name);
1502
1503 target_complete_cmd(se_cmd, SAM_STAT_CHECK_CONDITION);
1504}
1505
1506static void tcmu_check_expired_queue_cmd(struct tcmu_cmd *cmd)
1507{
1508 struct se_cmd *se_cmd;
1509
1510 if (!time_after_eq(jiffies, cmd->deadline))
1511 return;
1512
1513 pr_debug("Timing out queued cmd %p on dev %s.\n",
1514 cmd, cmd->tcmu_dev->name);
1515
1516 list_del_init(&cmd->queue_entry);
1517 se_cmd = cmd->se_cmd;
1518 tcmu_free_cmd(cmd);
1519
1520 se_cmd->priv = NULL;
1521 target_complete_cmd(se_cmd, SAM_STAT_TASK_SET_FULL);
1522}
1523
1524static void tcmu_device_timedout(struct tcmu_dev *udev)
1525{
1526 spin_lock(&timed_out_udevs_lock);
1527 if (list_empty(&udev->timedout_entry))
1528 list_add_tail(&udev->timedout_entry, &timed_out_udevs);
1529 spin_unlock(&timed_out_udevs_lock);
1530
1531 schedule_delayed_work(&tcmu_unmap_work, 0);
1532}
1533
1534static void tcmu_cmd_timedout(struct timer_list *t)
1535{
1536 struct tcmu_dev *udev = from_timer(udev, t, cmd_timer);
1537
1538 pr_debug("%s cmd timeout has expired\n", udev->name);
1539 tcmu_device_timedout(udev);
1540}
1541
1542static void tcmu_qfull_timedout(struct timer_list *t)
1543{
1544 struct tcmu_dev *udev = from_timer(udev, t, qfull_timer);
1545
1546 pr_debug("%s qfull timeout has expired\n", udev->name);
1547 tcmu_device_timedout(udev);
1548}
1549
1550static int tcmu_attach_hba(struct se_hba *hba, u32 host_id)
1551{
1552 struct tcmu_hba *tcmu_hba;
1553
1554 tcmu_hba = kzalloc(sizeof(struct tcmu_hba), GFP_KERNEL);
1555 if (!tcmu_hba)
1556 return -ENOMEM;
1557
1558 tcmu_hba->host_id = host_id;
1559 hba->hba_ptr = tcmu_hba;
1560
1561 return 0;
1562}
1563
1564static void tcmu_detach_hba(struct se_hba *hba)
1565{
1566 kfree(hba->hba_ptr);
1567 hba->hba_ptr = NULL;
1568}
1569
1570static struct se_device *tcmu_alloc_device(struct se_hba *hba, const char *name)
1571{
1572 struct tcmu_dev *udev;
1573
1574 udev = kzalloc(sizeof(struct tcmu_dev), GFP_KERNEL);
1575 if (!udev)
1576 return NULL;
1577 kref_init(&udev->kref);
1578
1579 udev->name = kstrdup(name, GFP_KERNEL);
1580 if (!udev->name) {
1581 kfree(udev);
1582 return NULL;
1583 }
1584
1585 udev->hba = hba;
1586 udev->cmd_time_out = TCMU_TIME_OUT;
1587 udev->qfull_time_out = -1;
1588
1589 udev->data_pages_per_blk = DATA_PAGES_PER_BLK_DEF;
1590 udev->max_blocks = DATA_AREA_PAGES_DEF / udev->data_pages_per_blk;
1591 udev->data_area_mb = TCMU_PAGES_TO_MBS(DATA_AREA_PAGES_DEF);
1592
1593 mutex_init(&udev->cmdr_lock);
1594
1595 INIT_LIST_HEAD(&udev->node);
1596 INIT_LIST_HEAD(&udev->timedout_entry);
1597 INIT_LIST_HEAD(&udev->qfull_queue);
1598 INIT_LIST_HEAD(&udev->tmr_queue);
1599 INIT_LIST_HEAD(&udev->inflight_queue);
1600 xa_init_flags(&udev->commands, XA_FLAGS_ALLOC1);
1601
1602 timer_setup(&udev->qfull_timer, tcmu_qfull_timedout, 0);
1603 timer_setup(&udev->cmd_timer, tcmu_cmd_timedout, 0);
1604
1605 xa_init(&udev->data_pages);
1606
1607 return &udev->se_dev;
1608}
1609
1610static void tcmu_dev_call_rcu(struct rcu_head *p)
1611{
1612 struct se_device *dev = container_of(p, struct se_device, rcu_head);
1613 struct tcmu_dev *udev = TCMU_DEV(dev);
1614
1615 kfree(udev->uio_info.name);
1616 kfree(udev->name);
1617 kfree(udev);
1618}
1619
1620static int tcmu_check_and_free_pending_cmd(struct tcmu_cmd *cmd)
1621{
1622 if (test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags)) {
1623 kmem_cache_free(tcmu_cmd_cache, cmd);
1624 return 0;
1625 }
1626 return -EINVAL;
1627}
1628
1629static u32 tcmu_blocks_release(struct tcmu_dev *udev, unsigned long first,
1630 unsigned long last)
1631{
1632 XA_STATE(xas, &udev->data_pages, first * udev->data_pages_per_blk);
1633 struct page *page;
1634 u32 pages_freed = 0;
1635
1636 xas_lock(&xas);
1637 xas_for_each(&xas, page, (last + 1) * udev->data_pages_per_blk - 1) {
1638 xas_store(&xas, NULL);
1639 __free_page(page);
1640 pages_freed++;
1641 }
1642 xas_unlock(&xas);
1643
1644 atomic_sub(pages_freed, &global_page_count);
1645
1646 return pages_freed;
1647}
1648
1649static void tcmu_remove_all_queued_tmr(struct tcmu_dev *udev)
1650{
1651 struct tcmu_tmr *tmr, *tmp;
1652
1653 list_for_each_entry_safe(tmr, tmp, &udev->tmr_queue, queue_entry) {
1654 list_del_init(&tmr->queue_entry);
1655 kfree(tmr);
1656 }
1657}
1658
1659static void tcmu_dev_kref_release(struct kref *kref)
1660{
1661 struct tcmu_dev *udev = container_of(kref, struct tcmu_dev, kref);
1662 struct se_device *dev = &udev->se_dev;
1663 struct tcmu_cmd *cmd;
1664 bool all_expired = true;
1665 unsigned long i;
1666
1667 vfree(udev->mb_addr);
1668 udev->mb_addr = NULL;
1669
1670 spin_lock_bh(&timed_out_udevs_lock);
1671 if (!list_empty(&udev->timedout_entry))
1672 list_del(&udev->timedout_entry);
1673 spin_unlock_bh(&timed_out_udevs_lock);
1674
1675
1676 mutex_lock(&udev->cmdr_lock);
1677 xa_for_each(&udev->commands, i, cmd) {
1678 if (tcmu_check_and_free_pending_cmd(cmd) != 0)
1679 all_expired = false;
1680 }
1681
1682 tcmu_remove_all_queued_tmr(udev);
1683 if (!list_empty(&udev->qfull_queue))
1684 all_expired = false;
1685 xa_destroy(&udev->commands);
1686 WARN_ON(!all_expired);
1687
1688 tcmu_blocks_release(udev, 0, udev->dbi_max);
1689 bitmap_free(udev->data_bitmap);
1690 mutex_unlock(&udev->cmdr_lock);
1691
1692 pr_debug("dev_kref_release\n");
1693
1694 call_rcu(&dev->rcu_head, tcmu_dev_call_rcu);
1695}
1696
1697static void run_qfull_queue(struct tcmu_dev *udev, bool fail)
1698{
1699 struct tcmu_cmd *tcmu_cmd, *tmp_cmd;
1700 LIST_HEAD(cmds);
1701 sense_reason_t scsi_ret;
1702 int ret;
1703
1704 if (list_empty(&udev->qfull_queue))
1705 return;
1706
1707 pr_debug("running %s's cmdr queue forcefail %d\n", udev->name, fail);
1708
1709 list_splice_init(&udev->qfull_queue, &cmds);
1710
1711 list_for_each_entry_safe(tcmu_cmd, tmp_cmd, &cmds, queue_entry) {
1712 list_del_init(&tcmu_cmd->queue_entry);
1713
1714 pr_debug("removing cmd %p on dev %s from queue\n",
1715 tcmu_cmd, udev->name);
1716
1717 if (fail) {
1718
1719
1720
1721
1722
1723
1724
1725 tcmu_cmd->se_cmd->priv = NULL;
1726 target_complete_cmd(tcmu_cmd->se_cmd, SAM_STAT_BUSY);
1727 tcmu_free_cmd(tcmu_cmd);
1728 continue;
1729 }
1730
1731 ret = queue_cmd_ring(tcmu_cmd, &scsi_ret);
1732 if (ret < 0) {
1733 pr_debug("cmd %p on dev %s failed with %u\n",
1734 tcmu_cmd, udev->name, scsi_ret);
1735
1736
1737
1738
1739 tcmu_cmd->se_cmd->priv = NULL;
1740 target_complete_cmd(tcmu_cmd->se_cmd,
1741 SAM_STAT_CHECK_CONDITION);
1742 tcmu_free_cmd(tcmu_cmd);
1743 } else if (ret > 0) {
1744 pr_debug("ran out of space during cmdr queue run\n");
1745
1746
1747
1748
1749 list_splice_tail(&cmds, &udev->qfull_queue);
1750 break;
1751 }
1752 }
1753
1754 tcmu_set_next_deadline(&udev->qfull_queue, &udev->qfull_timer);
1755}
1756
1757static int tcmu_irqcontrol(struct uio_info *info, s32 irq_on)
1758{
1759 struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info);
1760
1761 mutex_lock(&udev->cmdr_lock);
1762 if (tcmu_handle_completions(udev))
1763 run_qfull_queue(udev, false);
1764 mutex_unlock(&udev->cmdr_lock);
1765
1766 return 0;
1767}
1768
1769
1770
1771
1772
1773static int tcmu_find_mem_index(struct vm_area_struct *vma)
1774{
1775 struct tcmu_dev *udev = vma->vm_private_data;
1776 struct uio_info *info = &udev->uio_info;
1777
1778 if (vma->vm_pgoff < MAX_UIO_MAPS) {
1779 if (info->mem[vma->vm_pgoff].size == 0)
1780 return -1;
1781 return (int)vma->vm_pgoff;
1782 }
1783 return -1;
1784}
1785
1786static struct page *tcmu_try_get_data_page(struct tcmu_dev *udev, uint32_t dpi)
1787{
1788 struct page *page;
1789
1790 mutex_lock(&udev->cmdr_lock);
1791 page = xa_load(&udev->data_pages, dpi);
1792 if (likely(page)) {
1793 mutex_unlock(&udev->cmdr_lock);
1794 return page;
1795 }
1796
1797
1798
1799
1800
1801 pr_err("Invalid addr to data page mapping (dpi %u) on device %s\n",
1802 dpi, udev->name);
1803 mutex_unlock(&udev->cmdr_lock);
1804
1805 return NULL;
1806}
1807
1808static void tcmu_vma_open(struct vm_area_struct *vma)
1809{
1810 struct tcmu_dev *udev = vma->vm_private_data;
1811
1812 pr_debug("vma_open\n");
1813
1814 kref_get(&udev->kref);
1815}
1816
1817static void tcmu_vma_close(struct vm_area_struct *vma)
1818{
1819 struct tcmu_dev *udev = vma->vm_private_data;
1820
1821 pr_debug("vma_close\n");
1822
1823
1824 kref_put(&udev->kref, tcmu_dev_kref_release);
1825}
1826
1827static vm_fault_t tcmu_vma_fault(struct vm_fault *vmf)
1828{
1829 struct tcmu_dev *udev = vmf->vma->vm_private_data;
1830 struct uio_info *info = &udev->uio_info;
1831 struct page *page;
1832 unsigned long offset;
1833 void *addr;
1834
1835 int mi = tcmu_find_mem_index(vmf->vma);
1836 if (mi < 0)
1837 return VM_FAULT_SIGBUS;
1838
1839
1840
1841
1842
1843 offset = (vmf->pgoff - mi) << PAGE_SHIFT;
1844
1845 if (offset < udev->data_off) {
1846
1847 addr = (void *)(unsigned long)info->mem[mi].addr + offset;
1848 page = vmalloc_to_page(addr);
1849 } else {
1850 uint32_t dpi;
1851
1852
1853 dpi = (offset - udev->data_off) / PAGE_SIZE;
1854 page = tcmu_try_get_data_page(udev, dpi);
1855 if (!page)
1856 return VM_FAULT_SIGBUS;
1857 }
1858
1859 get_page(page);
1860 vmf->page = page;
1861 return 0;
1862}
1863
1864static const struct vm_operations_struct tcmu_vm_ops = {
1865 .open = tcmu_vma_open,
1866 .close = tcmu_vma_close,
1867 .fault = tcmu_vma_fault,
1868};
1869
1870static int tcmu_mmap(struct uio_info *info, struct vm_area_struct *vma)
1871{
1872 struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info);
1873
1874 vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP;
1875 vma->vm_ops = &tcmu_vm_ops;
1876
1877 vma->vm_private_data = udev;
1878
1879
1880 if (vma_pages(vma) != udev->mmap_pages)
1881 return -EINVAL;
1882
1883 tcmu_vma_open(vma);
1884
1885 return 0;
1886}
1887
1888static int tcmu_open(struct uio_info *info, struct inode *inode)
1889{
1890 struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info);
1891
1892
1893 if (test_and_set_bit(TCMU_DEV_BIT_OPEN, &udev->flags))
1894 return -EBUSY;
1895
1896 udev->inode = inode;
1897
1898 pr_debug("open\n");
1899
1900 return 0;
1901}
1902
1903static int tcmu_release(struct uio_info *info, struct inode *inode)
1904{
1905 struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info);
1906
1907 clear_bit(TCMU_DEV_BIT_OPEN, &udev->flags);
1908
1909 pr_debug("close\n");
1910
1911 return 0;
1912}
1913
1914static int tcmu_init_genl_cmd_reply(struct tcmu_dev *udev, int cmd)
1915{
1916 struct tcmu_nl_cmd *nl_cmd = &udev->curr_nl_cmd;
1917
1918 if (!tcmu_kern_cmd_reply_supported)
1919 return 0;
1920
1921 if (udev->nl_reply_supported <= 0)
1922 return 0;
1923
1924 mutex_lock(&tcmu_nl_cmd_mutex);
1925
1926 if (tcmu_netlink_blocked) {
1927 mutex_unlock(&tcmu_nl_cmd_mutex);
1928 pr_warn("Failing nl cmd %d on %s. Interface is blocked.\n", cmd,
1929 udev->name);
1930 return -EAGAIN;
1931 }
1932
1933 if (nl_cmd->cmd != TCMU_CMD_UNSPEC) {
1934 mutex_unlock(&tcmu_nl_cmd_mutex);
1935 pr_warn("netlink cmd %d already executing on %s\n",
1936 nl_cmd->cmd, udev->name);
1937 return -EBUSY;
1938 }
1939
1940 memset(nl_cmd, 0, sizeof(*nl_cmd));
1941 nl_cmd->cmd = cmd;
1942 nl_cmd->udev = udev;
1943 init_completion(&nl_cmd->complete);
1944 INIT_LIST_HEAD(&nl_cmd->nl_list);
1945
1946 list_add_tail(&nl_cmd->nl_list, &tcmu_nl_cmd_list);
1947
1948 mutex_unlock(&tcmu_nl_cmd_mutex);
1949 return 0;
1950}
1951
1952static void tcmu_destroy_genl_cmd_reply(struct tcmu_dev *udev)
1953{
1954 struct tcmu_nl_cmd *nl_cmd = &udev->curr_nl_cmd;
1955
1956 if (!tcmu_kern_cmd_reply_supported)
1957 return;
1958
1959 if (udev->nl_reply_supported <= 0)
1960 return;
1961
1962 mutex_lock(&tcmu_nl_cmd_mutex);
1963
1964 list_del(&nl_cmd->nl_list);
1965 memset(nl_cmd, 0, sizeof(*nl_cmd));
1966
1967 mutex_unlock(&tcmu_nl_cmd_mutex);
1968}
1969
1970static int tcmu_wait_genl_cmd_reply(struct tcmu_dev *udev)
1971{
1972 struct tcmu_nl_cmd *nl_cmd = &udev->curr_nl_cmd;
1973 int ret;
1974
1975 if (!tcmu_kern_cmd_reply_supported)
1976 return 0;
1977
1978 if (udev->nl_reply_supported <= 0)
1979 return 0;
1980
1981 pr_debug("sleeping for nl reply\n");
1982 wait_for_completion(&nl_cmd->complete);
1983
1984 mutex_lock(&tcmu_nl_cmd_mutex);
1985 nl_cmd->cmd = TCMU_CMD_UNSPEC;
1986 ret = nl_cmd->status;
1987 mutex_unlock(&tcmu_nl_cmd_mutex);
1988
1989 return ret;
1990}
1991
1992static int tcmu_netlink_event_init(struct tcmu_dev *udev,
1993 enum tcmu_genl_cmd cmd,
1994 struct sk_buff **buf, void **hdr)
1995{
1996 struct sk_buff *skb;
1997 void *msg_header;
1998 int ret = -ENOMEM;
1999
2000 skb = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
2001 if (!skb)
2002 return ret;
2003
2004 msg_header = genlmsg_put(skb, 0, 0, &tcmu_genl_family, 0, cmd);
2005 if (!msg_header)
2006 goto free_skb;
2007
2008 ret = nla_put_string(skb, TCMU_ATTR_DEVICE, udev->uio_info.name);
2009 if (ret < 0)
2010 goto free_skb;
2011
2012 ret = nla_put_u32(skb, TCMU_ATTR_MINOR, udev->uio_info.uio_dev->minor);
2013 if (ret < 0)
2014 goto free_skb;
2015
2016 ret = nla_put_u32(skb, TCMU_ATTR_DEVICE_ID, udev->se_dev.dev_index);
2017 if (ret < 0)
2018 goto free_skb;
2019
2020 *buf = skb;
2021 *hdr = msg_header;
2022 return ret;
2023
2024free_skb:
2025 nlmsg_free(skb);
2026 return ret;
2027}
2028
2029static int tcmu_netlink_event_send(struct tcmu_dev *udev,
2030 enum tcmu_genl_cmd cmd,
2031 struct sk_buff *skb, void *msg_header)
2032{
2033 int ret;
2034
2035 genlmsg_end(skb, msg_header);
2036
2037 ret = tcmu_init_genl_cmd_reply(udev, cmd);
2038 if (ret) {
2039 nlmsg_free(skb);
2040 return ret;
2041 }
2042
2043 ret = genlmsg_multicast_allns(&tcmu_genl_family, skb, 0,
2044 TCMU_MCGRP_CONFIG, GFP_KERNEL);
2045
2046
2047 if (ret == 0 ||
2048 (ret == -ESRCH && cmd == TCMU_CMD_ADDED_DEVICE))
2049 return tcmu_wait_genl_cmd_reply(udev);
2050 else
2051 tcmu_destroy_genl_cmd_reply(udev);
2052
2053 return ret;
2054}
2055
2056static int tcmu_send_dev_add_event(struct tcmu_dev *udev)
2057{
2058 struct sk_buff *skb = NULL;
2059 void *msg_header = NULL;
2060 int ret = 0;
2061
2062 ret = tcmu_netlink_event_init(udev, TCMU_CMD_ADDED_DEVICE, &skb,
2063 &msg_header);
2064 if (ret < 0)
2065 return ret;
2066 return tcmu_netlink_event_send(udev, TCMU_CMD_ADDED_DEVICE, skb,
2067 msg_header);
2068}
2069
2070static int tcmu_send_dev_remove_event(struct tcmu_dev *udev)
2071{
2072 struct sk_buff *skb = NULL;
2073 void *msg_header = NULL;
2074 int ret = 0;
2075
2076 ret = tcmu_netlink_event_init(udev, TCMU_CMD_REMOVED_DEVICE,
2077 &skb, &msg_header);
2078 if (ret < 0)
2079 return ret;
2080 return tcmu_netlink_event_send(udev, TCMU_CMD_REMOVED_DEVICE,
2081 skb, msg_header);
2082}
2083
2084static int tcmu_update_uio_info(struct tcmu_dev *udev)
2085{
2086 struct tcmu_hba *hba = udev->hba->hba_ptr;
2087 struct uio_info *info;
2088 char *str;
2089
2090 info = &udev->uio_info;
2091
2092 if (udev->dev_config[0])
2093 str = kasprintf(GFP_KERNEL, "tcm-user/%u/%s/%s", hba->host_id,
2094 udev->name, udev->dev_config);
2095 else
2096 str = kasprintf(GFP_KERNEL, "tcm-user/%u/%s", hba->host_id,
2097 udev->name);
2098 if (!str)
2099 return -ENOMEM;
2100
2101
2102 kfree(info->name);
2103 info->name = str;
2104
2105 return 0;
2106}
2107
2108static int tcmu_configure_device(struct se_device *dev)
2109{
2110 struct tcmu_dev *udev = TCMU_DEV(dev);
2111 struct uio_info *info;
2112 struct tcmu_mailbox *mb;
2113 size_t data_size;
2114 int ret = 0;
2115
2116 ret = tcmu_update_uio_info(udev);
2117 if (ret)
2118 return ret;
2119
2120 info = &udev->uio_info;
2121
2122 mutex_lock(&udev->cmdr_lock);
2123 udev->data_bitmap = bitmap_zalloc(udev->max_blocks, GFP_KERNEL);
2124 mutex_unlock(&udev->cmdr_lock);
2125 if (!udev->data_bitmap) {
2126 ret = -ENOMEM;
2127 goto err_bitmap_alloc;
2128 }
2129
2130 mb = vzalloc(MB_CMDR_SIZE);
2131 if (!mb) {
2132 ret = -ENOMEM;
2133 goto err_vzalloc;
2134 }
2135
2136
2137 udev->mb_addr = mb;
2138 udev->cmdr = (void *)mb + CMDR_OFF;
2139 udev->cmdr_size = CMDR_SIZE;
2140 udev->data_off = MB_CMDR_SIZE;
2141 data_size = TCMU_MBS_TO_PAGES(udev->data_area_mb) << PAGE_SHIFT;
2142 udev->mmap_pages = (data_size + MB_CMDR_SIZE) >> PAGE_SHIFT;
2143 udev->data_blk_size = udev->data_pages_per_blk * PAGE_SIZE;
2144 udev->dbi_thresh = 0;
2145
2146
2147 mb->version = TCMU_MAILBOX_VERSION;
2148 mb->flags = TCMU_MAILBOX_FLAG_CAP_OOOC |
2149 TCMU_MAILBOX_FLAG_CAP_READ_LEN |
2150 TCMU_MAILBOX_FLAG_CAP_TMR;
2151 mb->cmdr_off = CMDR_OFF;
2152 mb->cmdr_size = udev->cmdr_size;
2153
2154 WARN_ON(!PAGE_ALIGNED(udev->data_off));
2155 WARN_ON(data_size % PAGE_SIZE);
2156
2157 info->version = __stringify(TCMU_MAILBOX_VERSION);
2158
2159 info->mem[0].name = "tcm-user command & data buffer";
2160 info->mem[0].addr = (phys_addr_t)(uintptr_t)udev->mb_addr;
2161 info->mem[0].size = data_size + MB_CMDR_SIZE;
2162 info->mem[0].memtype = UIO_MEM_NONE;
2163
2164 info->irqcontrol = tcmu_irqcontrol;
2165 info->irq = UIO_IRQ_CUSTOM;
2166
2167 info->mmap = tcmu_mmap;
2168 info->open = tcmu_open;
2169 info->release = tcmu_release;
2170
2171 ret = uio_register_device(tcmu_root_device, info);
2172 if (ret)
2173 goto err_register;
2174
2175
2176 if (dev->dev_attrib.hw_block_size == 0)
2177 dev->dev_attrib.hw_block_size = 512;
2178
2179 if (!dev->dev_attrib.hw_max_sectors)
2180 dev->dev_attrib.hw_max_sectors = 128;
2181 if (!dev->dev_attrib.emulate_write_cache)
2182 dev->dev_attrib.emulate_write_cache = 0;
2183 dev->dev_attrib.hw_queue_depth = 128;
2184
2185
2186
2187
2188 if (udev->nl_reply_supported >= 0)
2189 udev->nl_reply_supported = tcmu_kern_cmd_reply_supported;
2190
2191
2192
2193
2194
2195 kref_get(&udev->kref);
2196
2197 ret = tcmu_send_dev_add_event(udev);
2198 if (ret)
2199 goto err_netlink;
2200
2201 mutex_lock(&root_udev_mutex);
2202 list_add(&udev->node, &root_udev);
2203 mutex_unlock(&root_udev_mutex);
2204
2205 return 0;
2206
2207err_netlink:
2208 kref_put(&udev->kref, tcmu_dev_kref_release);
2209 uio_unregister_device(&udev->uio_info);
2210err_register:
2211 vfree(udev->mb_addr);
2212 udev->mb_addr = NULL;
2213err_vzalloc:
2214 bitmap_free(udev->data_bitmap);
2215 udev->data_bitmap = NULL;
2216err_bitmap_alloc:
2217 kfree(info->name);
2218 info->name = NULL;
2219
2220 return ret;
2221}
2222
2223static void tcmu_free_device(struct se_device *dev)
2224{
2225 struct tcmu_dev *udev = TCMU_DEV(dev);
2226
2227
2228 kref_put(&udev->kref, tcmu_dev_kref_release);
2229}
2230
2231static void tcmu_destroy_device(struct se_device *dev)
2232{
2233 struct tcmu_dev *udev = TCMU_DEV(dev);
2234
2235 del_timer_sync(&udev->cmd_timer);
2236 del_timer_sync(&udev->qfull_timer);
2237
2238 mutex_lock(&root_udev_mutex);
2239 list_del(&udev->node);
2240 mutex_unlock(&root_udev_mutex);
2241
2242 tcmu_send_dev_remove_event(udev);
2243
2244 uio_unregister_device(&udev->uio_info);
2245
2246
2247 kref_put(&udev->kref, tcmu_dev_kref_release);
2248}
2249
2250static void tcmu_unblock_dev(struct tcmu_dev *udev)
2251{
2252 mutex_lock(&udev->cmdr_lock);
2253 clear_bit(TCMU_DEV_BIT_BLOCKED, &udev->flags);
2254 mutex_unlock(&udev->cmdr_lock);
2255}
2256
2257static void tcmu_block_dev(struct tcmu_dev *udev)
2258{
2259 mutex_lock(&udev->cmdr_lock);
2260
2261 if (test_and_set_bit(TCMU_DEV_BIT_BLOCKED, &udev->flags))
2262 goto unlock;
2263
2264
2265 tcmu_handle_completions(udev);
2266
2267 run_qfull_queue(udev, true);
2268
2269unlock:
2270 mutex_unlock(&udev->cmdr_lock);
2271}
2272
2273static void tcmu_reset_ring(struct tcmu_dev *udev, u8 err_level)
2274{
2275 struct tcmu_mailbox *mb;
2276 struct tcmu_cmd *cmd;
2277 unsigned long i;
2278
2279 mutex_lock(&udev->cmdr_lock);
2280
2281 xa_for_each(&udev->commands, i, cmd) {
2282 pr_debug("removing cmd %u on dev %s from ring (is expired %d)\n",
2283 cmd->cmd_id, udev->name,
2284 test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags));
2285
2286 xa_erase(&udev->commands, i);
2287 if (!test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags)) {
2288 WARN_ON(!cmd->se_cmd);
2289 list_del_init(&cmd->queue_entry);
2290 cmd->se_cmd->priv = NULL;
2291 if (err_level == 1) {
2292
2293
2294
2295
2296 target_complete_cmd(cmd->se_cmd, SAM_STAT_BUSY);
2297 } else {
2298
2299 target_complete_cmd(cmd->se_cmd,
2300 SAM_STAT_CHECK_CONDITION);
2301 }
2302 }
2303 tcmu_cmd_free_data(cmd, cmd->dbi_cnt);
2304 tcmu_free_cmd(cmd);
2305 }
2306
2307 mb = udev->mb_addr;
2308 tcmu_flush_dcache_range(mb, sizeof(*mb));
2309 pr_debug("mb last %u head %u tail %u\n", udev->cmdr_last_cleaned,
2310 mb->cmd_tail, mb->cmd_head);
2311
2312 udev->cmdr_last_cleaned = 0;
2313 mb->cmd_tail = 0;
2314 mb->cmd_head = 0;
2315 tcmu_flush_dcache_range(mb, sizeof(*mb));
2316 clear_bit(TCMU_DEV_BIT_BROKEN, &udev->flags);
2317
2318 del_timer(&udev->cmd_timer);
2319
2320
2321
2322
2323
2324
2325
2326
2327 tcmu_remove_all_queued_tmr(udev);
2328
2329 run_qfull_queue(udev, false);
2330
2331 mutex_unlock(&udev->cmdr_lock);
2332}
2333
2334enum {
2335 Opt_dev_config, Opt_dev_size, Opt_hw_block_size, Opt_hw_max_sectors,
2336 Opt_nl_reply_supported, Opt_max_data_area_mb, Opt_data_pages_per_blk,
2337 Opt_err,
2338};
2339
2340static match_table_t tokens = {
2341 {Opt_dev_config, "dev_config=%s"},
2342 {Opt_dev_size, "dev_size=%s"},
2343 {Opt_hw_block_size, "hw_block_size=%d"},
2344 {Opt_hw_max_sectors, "hw_max_sectors=%d"},
2345 {Opt_nl_reply_supported, "nl_reply_supported=%d"},
2346 {Opt_max_data_area_mb, "max_data_area_mb=%d"},
2347 {Opt_data_pages_per_blk, "data_pages_per_blk=%d"},
2348 {Opt_err, NULL}
2349};
2350
2351static int tcmu_set_dev_attrib(substring_t *arg, u32 *dev_attrib)
2352{
2353 int val, ret;
2354
2355 ret = match_int(arg, &val);
2356 if (ret < 0) {
2357 pr_err("match_int() failed for dev attrib. Error %d.\n",
2358 ret);
2359 return ret;
2360 }
2361
2362 if (val <= 0) {
2363 pr_err("Invalid dev attrib value %d. Must be greater than zero.\n",
2364 val);
2365 return -EINVAL;
2366 }
2367 *dev_attrib = val;
2368 return 0;
2369}
2370
2371static int tcmu_set_max_blocks_param(struct tcmu_dev *udev, substring_t *arg)
2372{
2373 int val, ret;
2374 uint32_t pages_per_blk = udev->data_pages_per_blk;
2375
2376 ret = match_int(arg, &val);
2377 if (ret < 0) {
2378 pr_err("match_int() failed for max_data_area_mb=. Error %d.\n",
2379 ret);
2380 return ret;
2381 }
2382 if (val <= 0) {
2383 pr_err("Invalid max_data_area %d.\n", val);
2384 return -EINVAL;
2385 }
2386 if (val > TCMU_PAGES_TO_MBS(tcmu_global_max_pages)) {
2387 pr_err("%d is too large. Adjusting max_data_area_mb to global limit of %u\n",
2388 val, TCMU_PAGES_TO_MBS(tcmu_global_max_pages));
2389 val = TCMU_PAGES_TO_MBS(tcmu_global_max_pages);
2390 }
2391 if (TCMU_MBS_TO_PAGES(val) < pages_per_blk) {
2392 pr_err("Invalid max_data_area %d (%zu pages): smaller than data_pages_per_blk (%u pages).\n",
2393 val, TCMU_MBS_TO_PAGES(val), pages_per_blk);
2394 return -EINVAL;
2395 }
2396
2397 mutex_lock(&udev->cmdr_lock);
2398 if (udev->data_bitmap) {
2399 pr_err("Cannot set max_data_area_mb after it has been enabled.\n");
2400 ret = -EINVAL;
2401 goto unlock;
2402 }
2403
2404 udev->data_area_mb = val;
2405 udev->max_blocks = TCMU_MBS_TO_PAGES(val) / pages_per_blk;
2406
2407unlock:
2408 mutex_unlock(&udev->cmdr_lock);
2409 return ret;
2410}
2411
2412static int tcmu_set_data_pages_per_blk(struct tcmu_dev *udev, substring_t *arg)
2413{
2414 int val, ret;
2415
2416 ret = match_int(arg, &val);
2417 if (ret < 0) {
2418 pr_err("match_int() failed for data_pages_per_blk=. Error %d.\n",
2419 ret);
2420 return ret;
2421 }
2422
2423 if (val > TCMU_MBS_TO_PAGES(udev->data_area_mb)) {
2424 pr_err("Invalid data_pages_per_blk %d: greater than max_data_area_mb %d -> %zd pages).\n",
2425 val, udev->data_area_mb,
2426 TCMU_MBS_TO_PAGES(udev->data_area_mb));
2427 return -EINVAL;
2428 }
2429
2430 mutex_lock(&udev->cmdr_lock);
2431 if (udev->data_bitmap) {
2432 pr_err("Cannot set data_pages_per_blk after it has been enabled.\n");
2433 ret = -EINVAL;
2434 goto unlock;
2435 }
2436
2437 udev->data_pages_per_blk = val;
2438 udev->max_blocks = TCMU_MBS_TO_PAGES(udev->data_area_mb) / val;
2439
2440unlock:
2441 mutex_unlock(&udev->cmdr_lock);
2442 return ret;
2443}
2444
2445static ssize_t tcmu_set_configfs_dev_params(struct se_device *dev,
2446 const char *page, ssize_t count)
2447{
2448 struct tcmu_dev *udev = TCMU_DEV(dev);
2449 char *orig, *ptr, *opts;
2450 substring_t args[MAX_OPT_ARGS];
2451 int ret = 0, token;
2452
2453 opts = kstrdup(page, GFP_KERNEL);
2454 if (!opts)
2455 return -ENOMEM;
2456
2457 orig = opts;
2458
2459 while ((ptr = strsep(&opts, ",\n")) != NULL) {
2460 if (!*ptr)
2461 continue;
2462
2463 token = match_token(ptr, tokens, args);
2464 switch (token) {
2465 case Opt_dev_config:
2466 if (match_strlcpy(udev->dev_config, &args[0],
2467 TCMU_CONFIG_LEN) == 0) {
2468 ret = -EINVAL;
2469 break;
2470 }
2471 pr_debug("TCMU: Referencing Path: %s\n", udev->dev_config);
2472 break;
2473 case Opt_dev_size:
2474 ret = match_u64(&args[0], &udev->dev_size);
2475 if (ret < 0)
2476 pr_err("match_u64() failed for dev_size=. Error %d.\n",
2477 ret);
2478 break;
2479 case Opt_hw_block_size:
2480 ret = tcmu_set_dev_attrib(&args[0],
2481 &(dev->dev_attrib.hw_block_size));
2482 break;
2483 case Opt_hw_max_sectors:
2484 ret = tcmu_set_dev_attrib(&args[0],
2485 &(dev->dev_attrib.hw_max_sectors));
2486 break;
2487 case Opt_nl_reply_supported:
2488 ret = match_int(&args[0], &udev->nl_reply_supported);
2489 if (ret < 0)
2490 pr_err("match_int() failed for nl_reply_supported=. Error %d.\n",
2491 ret);
2492 break;
2493 case Opt_max_data_area_mb:
2494 ret = tcmu_set_max_blocks_param(udev, &args[0]);
2495 break;
2496 case Opt_data_pages_per_blk:
2497 ret = tcmu_set_data_pages_per_blk(udev, &args[0]);
2498 break;
2499 default:
2500 break;
2501 }
2502
2503 if (ret)
2504 break;
2505 }
2506
2507 kfree(orig);
2508 return (!ret) ? count : ret;
2509}
2510
2511static ssize_t tcmu_show_configfs_dev_params(struct se_device *dev, char *b)
2512{
2513 struct tcmu_dev *udev = TCMU_DEV(dev);
2514 ssize_t bl = 0;
2515
2516 bl = sprintf(b + bl, "Config: %s ",
2517 udev->dev_config[0] ? udev->dev_config : "NULL");
2518 bl += sprintf(b + bl, "Size: %llu ", udev->dev_size);
2519 bl += sprintf(b + bl, "MaxDataAreaMB: %u ", udev->data_area_mb);
2520 bl += sprintf(b + bl, "DataPagesPerBlk: %u\n", udev->data_pages_per_blk);
2521
2522 return bl;
2523}
2524
2525static sector_t tcmu_get_blocks(struct se_device *dev)
2526{
2527 struct tcmu_dev *udev = TCMU_DEV(dev);
2528
2529 return div_u64(udev->dev_size - dev->dev_attrib.block_size,
2530 dev->dev_attrib.block_size);
2531}
2532
2533static sense_reason_t
2534tcmu_parse_cdb(struct se_cmd *cmd)
2535{
2536 return passthrough_parse_cdb(cmd, tcmu_queue_cmd);
2537}
2538
2539static ssize_t tcmu_cmd_time_out_show(struct config_item *item, char *page)
2540{
2541 struct se_dev_attrib *da = container_of(to_config_group(item),
2542 struct se_dev_attrib, da_group);
2543 struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
2544
2545 return snprintf(page, PAGE_SIZE, "%lu\n", udev->cmd_time_out / MSEC_PER_SEC);
2546}
2547
2548static ssize_t tcmu_cmd_time_out_store(struct config_item *item, const char *page,
2549 size_t count)
2550{
2551 struct se_dev_attrib *da = container_of(to_config_group(item),
2552 struct se_dev_attrib, da_group);
2553 struct tcmu_dev *udev = container_of(da->da_dev,
2554 struct tcmu_dev, se_dev);
2555 u32 val;
2556 int ret;
2557
2558 if (da->da_dev->export_count) {
2559 pr_err("Unable to set tcmu cmd_time_out while exports exist\n");
2560 return -EINVAL;
2561 }
2562
2563 ret = kstrtou32(page, 0, &val);
2564 if (ret < 0)
2565 return ret;
2566
2567 udev->cmd_time_out = val * MSEC_PER_SEC;
2568 return count;
2569}
2570CONFIGFS_ATTR(tcmu_, cmd_time_out);
2571
2572static ssize_t tcmu_qfull_time_out_show(struct config_item *item, char *page)
2573{
2574 struct se_dev_attrib *da = container_of(to_config_group(item),
2575 struct se_dev_attrib, da_group);
2576 struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
2577
2578 return snprintf(page, PAGE_SIZE, "%ld\n", udev->qfull_time_out <= 0 ?
2579 udev->qfull_time_out :
2580 udev->qfull_time_out / MSEC_PER_SEC);
2581}
2582
2583static ssize_t tcmu_qfull_time_out_store(struct config_item *item,
2584 const char *page, size_t count)
2585{
2586 struct se_dev_attrib *da = container_of(to_config_group(item),
2587 struct se_dev_attrib, da_group);
2588 struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
2589 s32 val;
2590 int ret;
2591
2592 ret = kstrtos32(page, 0, &val);
2593 if (ret < 0)
2594 return ret;
2595
2596 if (val >= 0) {
2597 udev->qfull_time_out = val * MSEC_PER_SEC;
2598 } else if (val == -1) {
2599 udev->qfull_time_out = val;
2600 } else {
2601 printk(KERN_ERR "Invalid qfull timeout value %d\n", val);
2602 return -EINVAL;
2603 }
2604 return count;
2605}
2606CONFIGFS_ATTR(tcmu_, qfull_time_out);
2607
2608static ssize_t tcmu_max_data_area_mb_show(struct config_item *item, char *page)
2609{
2610 struct se_dev_attrib *da = container_of(to_config_group(item),
2611 struct se_dev_attrib, da_group);
2612 struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
2613
2614 return snprintf(page, PAGE_SIZE, "%u\n", udev->data_area_mb);
2615}
2616CONFIGFS_ATTR_RO(tcmu_, max_data_area_mb);
2617
2618static ssize_t tcmu_data_pages_per_blk_show(struct config_item *item,
2619 char *page)
2620{
2621 struct se_dev_attrib *da = container_of(to_config_group(item),
2622 struct se_dev_attrib, da_group);
2623 struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
2624
2625 return snprintf(page, PAGE_SIZE, "%u\n", udev->data_pages_per_blk);
2626}
2627CONFIGFS_ATTR_RO(tcmu_, data_pages_per_blk);
2628
2629static ssize_t tcmu_dev_config_show(struct config_item *item, char *page)
2630{
2631 struct se_dev_attrib *da = container_of(to_config_group(item),
2632 struct se_dev_attrib, da_group);
2633 struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
2634
2635 return snprintf(page, PAGE_SIZE, "%s\n", udev->dev_config);
2636}
2637
2638static int tcmu_send_dev_config_event(struct tcmu_dev *udev,
2639 const char *reconfig_data)
2640{
2641 struct sk_buff *skb = NULL;
2642 void *msg_header = NULL;
2643 int ret = 0;
2644
2645 ret = tcmu_netlink_event_init(udev, TCMU_CMD_RECONFIG_DEVICE,
2646 &skb, &msg_header);
2647 if (ret < 0)
2648 return ret;
2649 ret = nla_put_string(skb, TCMU_ATTR_DEV_CFG, reconfig_data);
2650 if (ret < 0) {
2651 nlmsg_free(skb);
2652 return ret;
2653 }
2654 return tcmu_netlink_event_send(udev, TCMU_CMD_RECONFIG_DEVICE,
2655 skb, msg_header);
2656}
2657
2658
2659static ssize_t tcmu_dev_config_store(struct config_item *item, const char *page,
2660 size_t count)
2661{
2662 struct se_dev_attrib *da = container_of(to_config_group(item),
2663 struct se_dev_attrib, da_group);
2664 struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
2665 int ret, len;
2666
2667 len = strlen(page);
2668 if (!len || len > TCMU_CONFIG_LEN - 1)
2669 return -EINVAL;
2670
2671
2672 if (target_dev_configured(&udev->se_dev)) {
2673 ret = tcmu_send_dev_config_event(udev, page);
2674 if (ret) {
2675 pr_err("Unable to reconfigure device\n");
2676 return ret;
2677 }
2678 strlcpy(udev->dev_config, page, TCMU_CONFIG_LEN);
2679
2680 ret = tcmu_update_uio_info(udev);
2681 if (ret)
2682 return ret;
2683 return count;
2684 }
2685 strlcpy(udev->dev_config, page, TCMU_CONFIG_LEN);
2686
2687 return count;
2688}
2689CONFIGFS_ATTR(tcmu_, dev_config);
2690
2691static ssize_t tcmu_dev_size_show(struct config_item *item, char *page)
2692{
2693 struct se_dev_attrib *da = container_of(to_config_group(item),
2694 struct se_dev_attrib, da_group);
2695 struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
2696
2697 return snprintf(page, PAGE_SIZE, "%llu\n", udev->dev_size);
2698}
2699
2700static int tcmu_send_dev_size_event(struct tcmu_dev *udev, u64 size)
2701{
2702 struct sk_buff *skb = NULL;
2703 void *msg_header = NULL;
2704 int ret = 0;
2705
2706 ret = tcmu_netlink_event_init(udev, TCMU_CMD_RECONFIG_DEVICE,
2707 &skb, &msg_header);
2708 if (ret < 0)
2709 return ret;
2710 ret = nla_put_u64_64bit(skb, TCMU_ATTR_DEV_SIZE,
2711 size, TCMU_ATTR_PAD);
2712 if (ret < 0) {
2713 nlmsg_free(skb);
2714 return ret;
2715 }
2716 return tcmu_netlink_event_send(udev, TCMU_CMD_RECONFIG_DEVICE,
2717 skb, msg_header);
2718}
2719
2720static ssize_t tcmu_dev_size_store(struct config_item *item, const char *page,
2721 size_t count)
2722{
2723 struct se_dev_attrib *da = container_of(to_config_group(item),
2724 struct se_dev_attrib, da_group);
2725 struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
2726 u64 val;
2727 int ret;
2728
2729 ret = kstrtou64(page, 0, &val);
2730 if (ret < 0)
2731 return ret;
2732
2733
2734 if (target_dev_configured(&udev->se_dev)) {
2735 ret = tcmu_send_dev_size_event(udev, val);
2736 if (ret) {
2737 pr_err("Unable to reconfigure device\n");
2738 return ret;
2739 }
2740 }
2741 udev->dev_size = val;
2742 return count;
2743}
2744CONFIGFS_ATTR(tcmu_, dev_size);
2745
2746static ssize_t tcmu_nl_reply_supported_show(struct config_item *item,
2747 char *page)
2748{
2749 struct se_dev_attrib *da = container_of(to_config_group(item),
2750 struct se_dev_attrib, da_group);
2751 struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
2752
2753 return snprintf(page, PAGE_SIZE, "%d\n", udev->nl_reply_supported);
2754}
2755
2756static ssize_t tcmu_nl_reply_supported_store(struct config_item *item,
2757 const char *page, size_t count)
2758{
2759 struct se_dev_attrib *da = container_of(to_config_group(item),
2760 struct se_dev_attrib, da_group);
2761 struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
2762 s8 val;
2763 int ret;
2764
2765 ret = kstrtos8(page, 0, &val);
2766 if (ret < 0)
2767 return ret;
2768
2769 udev->nl_reply_supported = val;
2770 return count;
2771}
2772CONFIGFS_ATTR(tcmu_, nl_reply_supported);
2773
2774static ssize_t tcmu_emulate_write_cache_show(struct config_item *item,
2775 char *page)
2776{
2777 struct se_dev_attrib *da = container_of(to_config_group(item),
2778 struct se_dev_attrib, da_group);
2779
2780 return snprintf(page, PAGE_SIZE, "%i\n", da->emulate_write_cache);
2781}
2782
2783static int tcmu_send_emulate_write_cache(struct tcmu_dev *udev, u8 val)
2784{
2785 struct sk_buff *skb = NULL;
2786 void *msg_header = NULL;
2787 int ret = 0;
2788
2789 ret = tcmu_netlink_event_init(udev, TCMU_CMD_RECONFIG_DEVICE,
2790 &skb, &msg_header);
2791 if (ret < 0)
2792 return ret;
2793 ret = nla_put_u8(skb, TCMU_ATTR_WRITECACHE, val);
2794 if (ret < 0) {
2795 nlmsg_free(skb);
2796 return ret;
2797 }
2798 return tcmu_netlink_event_send(udev, TCMU_CMD_RECONFIG_DEVICE,
2799 skb, msg_header);
2800}
2801
2802static ssize_t tcmu_emulate_write_cache_store(struct config_item *item,
2803 const char *page, size_t count)
2804{
2805 struct se_dev_attrib *da = container_of(to_config_group(item),
2806 struct se_dev_attrib, da_group);
2807 struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
2808 u8 val;
2809 int ret;
2810
2811 ret = kstrtou8(page, 0, &val);
2812 if (ret < 0)
2813 return ret;
2814
2815
2816 if (target_dev_configured(&udev->se_dev)) {
2817 ret = tcmu_send_emulate_write_cache(udev, val);
2818 if (ret) {
2819 pr_err("Unable to reconfigure device\n");
2820 return ret;
2821 }
2822 }
2823
2824 da->emulate_write_cache = val;
2825 return count;
2826}
2827CONFIGFS_ATTR(tcmu_, emulate_write_cache);
2828
2829static ssize_t tcmu_tmr_notification_show(struct config_item *item, char *page)
2830{
2831 struct se_dev_attrib *da = container_of(to_config_group(item),
2832 struct se_dev_attrib, da_group);
2833 struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
2834
2835 return snprintf(page, PAGE_SIZE, "%i\n",
2836 test_bit(TCMU_DEV_BIT_TMR_NOTIFY, &udev->flags));
2837}
2838
2839static ssize_t tcmu_tmr_notification_store(struct config_item *item,
2840 const char *page, size_t count)
2841{
2842 struct se_dev_attrib *da = container_of(to_config_group(item),
2843 struct se_dev_attrib, da_group);
2844 struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
2845 u8 val;
2846 int ret;
2847
2848 ret = kstrtou8(page, 0, &val);
2849 if (ret < 0)
2850 return ret;
2851 if (val > 1)
2852 return -EINVAL;
2853
2854 if (val)
2855 set_bit(TCMU_DEV_BIT_TMR_NOTIFY, &udev->flags);
2856 else
2857 clear_bit(TCMU_DEV_BIT_TMR_NOTIFY, &udev->flags);
2858 return count;
2859}
2860CONFIGFS_ATTR(tcmu_, tmr_notification);
2861
2862static ssize_t tcmu_block_dev_show(struct config_item *item, char *page)
2863{
2864 struct se_device *se_dev = container_of(to_config_group(item),
2865 struct se_device,
2866 dev_action_group);
2867 struct tcmu_dev *udev = TCMU_DEV(se_dev);
2868
2869 if (test_bit(TCMU_DEV_BIT_BLOCKED, &udev->flags))
2870 return snprintf(page, PAGE_SIZE, "%s\n", "blocked");
2871 else
2872 return snprintf(page, PAGE_SIZE, "%s\n", "unblocked");
2873}
2874
2875static ssize_t tcmu_block_dev_store(struct config_item *item, const char *page,
2876 size_t count)
2877{
2878 struct se_device *se_dev = container_of(to_config_group(item),
2879 struct se_device,
2880 dev_action_group);
2881 struct tcmu_dev *udev = TCMU_DEV(se_dev);
2882 u8 val;
2883 int ret;
2884
2885 if (!target_dev_configured(&udev->se_dev)) {
2886 pr_err("Device is not configured.\n");
2887 return -EINVAL;
2888 }
2889
2890 ret = kstrtou8(page, 0, &val);
2891 if (ret < 0)
2892 return ret;
2893
2894 if (val > 1) {
2895 pr_err("Invalid block value %d\n", val);
2896 return -EINVAL;
2897 }
2898
2899 if (!val)
2900 tcmu_unblock_dev(udev);
2901 else
2902 tcmu_block_dev(udev);
2903 return count;
2904}
2905CONFIGFS_ATTR(tcmu_, block_dev);
2906
2907static ssize_t tcmu_reset_ring_store(struct config_item *item, const char *page,
2908 size_t count)
2909{
2910 struct se_device *se_dev = container_of(to_config_group(item),
2911 struct se_device,
2912 dev_action_group);
2913 struct tcmu_dev *udev = TCMU_DEV(se_dev);
2914 u8 val;
2915 int ret;
2916
2917 if (!target_dev_configured(&udev->se_dev)) {
2918 pr_err("Device is not configured.\n");
2919 return -EINVAL;
2920 }
2921
2922 ret = kstrtou8(page, 0, &val);
2923 if (ret < 0)
2924 return ret;
2925
2926 if (val != 1 && val != 2) {
2927 pr_err("Invalid reset ring value %d\n", val);
2928 return -EINVAL;
2929 }
2930
2931 tcmu_reset_ring(udev, val);
2932 return count;
2933}
2934CONFIGFS_ATTR_WO(tcmu_, reset_ring);
2935
2936static struct configfs_attribute *tcmu_attrib_attrs[] = {
2937 &tcmu_attr_cmd_time_out,
2938 &tcmu_attr_qfull_time_out,
2939 &tcmu_attr_max_data_area_mb,
2940 &tcmu_attr_data_pages_per_blk,
2941 &tcmu_attr_dev_config,
2942 &tcmu_attr_dev_size,
2943 &tcmu_attr_emulate_write_cache,
2944 &tcmu_attr_tmr_notification,
2945 &tcmu_attr_nl_reply_supported,
2946 NULL,
2947};
2948
2949static struct configfs_attribute **tcmu_attrs;
2950
2951static struct configfs_attribute *tcmu_action_attrs[] = {
2952 &tcmu_attr_block_dev,
2953 &tcmu_attr_reset_ring,
2954 NULL,
2955};
2956
2957static struct target_backend_ops tcmu_ops = {
2958 .name = "user",
2959 .owner = THIS_MODULE,
2960 .transport_flags_default = TRANSPORT_FLAG_PASSTHROUGH,
2961 .transport_flags_changeable = TRANSPORT_FLAG_PASSTHROUGH_PGR |
2962 TRANSPORT_FLAG_PASSTHROUGH_ALUA,
2963 .attach_hba = tcmu_attach_hba,
2964 .detach_hba = tcmu_detach_hba,
2965 .alloc_device = tcmu_alloc_device,
2966 .configure_device = tcmu_configure_device,
2967 .destroy_device = tcmu_destroy_device,
2968 .free_device = tcmu_free_device,
2969 .unplug_device = tcmu_unplug_device,
2970 .plug_device = tcmu_plug_device,
2971 .parse_cdb = tcmu_parse_cdb,
2972 .tmr_notify = tcmu_tmr_notify,
2973 .set_configfs_dev_params = tcmu_set_configfs_dev_params,
2974 .show_configfs_dev_params = tcmu_show_configfs_dev_params,
2975 .get_device_type = sbc_get_device_type,
2976 .get_blocks = tcmu_get_blocks,
2977 .tb_dev_action_attrs = tcmu_action_attrs,
2978};
2979
2980static void find_free_blocks(void)
2981{
2982 struct tcmu_dev *udev;
2983 loff_t off;
2984 u32 pages_freed, total_pages_freed = 0;
2985 u32 start, end, block, total_blocks_freed = 0;
2986
2987 if (atomic_read(&global_page_count) <= tcmu_global_max_pages)
2988 return;
2989
2990 mutex_lock(&root_udev_mutex);
2991 list_for_each_entry(udev, &root_udev, node) {
2992 mutex_lock(&udev->cmdr_lock);
2993
2994 if (!target_dev_configured(&udev->se_dev)) {
2995 mutex_unlock(&udev->cmdr_lock);
2996 continue;
2997 }
2998
2999
3000 if (tcmu_handle_completions(udev))
3001 run_qfull_queue(udev, false);
3002
3003
3004 if (!udev->dbi_thresh) {
3005 mutex_unlock(&udev->cmdr_lock);
3006 continue;
3007 }
3008
3009 end = udev->dbi_max + 1;
3010 block = find_last_bit(udev->data_bitmap, end);
3011 if (block == udev->dbi_max) {
3012
3013
3014
3015
3016 mutex_unlock(&udev->cmdr_lock);
3017 continue;
3018 } else if (block == end) {
3019
3020 udev->dbi_thresh = start = 0;
3021 udev->dbi_max = 0;
3022 } else {
3023 udev->dbi_thresh = start = block + 1;
3024 udev->dbi_max = block;
3025 }
3026
3027
3028 off = udev->data_off + (loff_t)start * udev->data_blk_size;
3029 unmap_mapping_range(udev->inode->i_mapping, off, 0, 1);
3030
3031
3032 pages_freed = tcmu_blocks_release(udev, start, end - 1);
3033 mutex_unlock(&udev->cmdr_lock);
3034
3035 total_pages_freed += pages_freed;
3036 total_blocks_freed += end - start;
3037 pr_debug("Freed %u pages (total %u) from %u blocks (total %u) from %s.\n",
3038 pages_freed, total_pages_freed, end - start,
3039 total_blocks_freed, udev->name);
3040 }
3041 mutex_unlock(&root_udev_mutex);
3042
3043 if (atomic_read(&global_page_count) > tcmu_global_max_pages)
3044 schedule_delayed_work(&tcmu_unmap_work, msecs_to_jiffies(5000));
3045}
3046
3047static void check_timedout_devices(void)
3048{
3049 struct tcmu_dev *udev, *tmp_dev;
3050 struct tcmu_cmd *cmd, *tmp_cmd;
3051 LIST_HEAD(devs);
3052
3053 spin_lock_bh(&timed_out_udevs_lock);
3054 list_splice_init(&timed_out_udevs, &devs);
3055
3056 list_for_each_entry_safe(udev, tmp_dev, &devs, timedout_entry) {
3057 list_del_init(&udev->timedout_entry);
3058 spin_unlock_bh(&timed_out_udevs_lock);
3059
3060 mutex_lock(&udev->cmdr_lock);
3061
3062
3063
3064
3065
3066 if (udev->cmd_time_out) {
3067 list_for_each_entry_safe(cmd, tmp_cmd,
3068 &udev->inflight_queue,
3069 queue_entry) {
3070 tcmu_check_expired_ring_cmd(cmd);
3071 }
3072 tcmu_set_next_deadline(&udev->inflight_queue,
3073 &udev->cmd_timer);
3074 }
3075 list_for_each_entry_safe(cmd, tmp_cmd, &udev->qfull_queue,
3076 queue_entry) {
3077 tcmu_check_expired_queue_cmd(cmd);
3078 }
3079 tcmu_set_next_deadline(&udev->qfull_queue, &udev->qfull_timer);
3080
3081 mutex_unlock(&udev->cmdr_lock);
3082
3083 spin_lock_bh(&timed_out_udevs_lock);
3084 }
3085
3086 spin_unlock_bh(&timed_out_udevs_lock);
3087}
3088
3089static void tcmu_unmap_work_fn(struct work_struct *work)
3090{
3091 check_timedout_devices();
3092 find_free_blocks();
3093}
3094
3095static int __init tcmu_module_init(void)
3096{
3097 int ret, i, k, len = 0;
3098
3099 BUILD_BUG_ON((sizeof(struct tcmu_cmd_entry) % TCMU_OP_ALIGN_SIZE) != 0);
3100
3101 INIT_DELAYED_WORK(&tcmu_unmap_work, tcmu_unmap_work_fn);
3102
3103 tcmu_cmd_cache = kmem_cache_create("tcmu_cmd_cache",
3104 sizeof(struct tcmu_cmd),
3105 __alignof__(struct tcmu_cmd),
3106 0, NULL);
3107 if (!tcmu_cmd_cache)
3108 return -ENOMEM;
3109
3110 tcmu_root_device = root_device_register("tcm_user");
3111 if (IS_ERR(tcmu_root_device)) {
3112 ret = PTR_ERR(tcmu_root_device);
3113 goto out_free_cache;
3114 }
3115
3116 ret = genl_register_family(&tcmu_genl_family);
3117 if (ret < 0) {
3118 goto out_unreg_device;
3119 }
3120
3121 for (i = 0; passthrough_attrib_attrs[i] != NULL; i++)
3122 len += sizeof(struct configfs_attribute *);
3123 for (i = 0; passthrough_pr_attrib_attrs[i] != NULL; i++)
3124 len += sizeof(struct configfs_attribute *);
3125 for (i = 0; tcmu_attrib_attrs[i] != NULL; i++)
3126 len += sizeof(struct configfs_attribute *);
3127 len += sizeof(struct configfs_attribute *);
3128
3129 tcmu_attrs = kzalloc(len, GFP_KERNEL);
3130 if (!tcmu_attrs) {
3131 ret = -ENOMEM;
3132 goto out_unreg_genl;
3133 }
3134
3135 for (i = 0; passthrough_attrib_attrs[i] != NULL; i++)
3136 tcmu_attrs[i] = passthrough_attrib_attrs[i];
3137 for (k = 0; passthrough_pr_attrib_attrs[k] != NULL; k++)
3138 tcmu_attrs[i++] = passthrough_pr_attrib_attrs[k];
3139 for (k = 0; tcmu_attrib_attrs[k] != NULL; k++)
3140 tcmu_attrs[i++] = tcmu_attrib_attrs[k];
3141 tcmu_ops.tb_dev_attrib_attrs = tcmu_attrs;
3142
3143 ret = transport_backend_register(&tcmu_ops);
3144 if (ret)
3145 goto out_attrs;
3146
3147 return 0;
3148
3149out_attrs:
3150 kfree(tcmu_attrs);
3151out_unreg_genl:
3152 genl_unregister_family(&tcmu_genl_family);
3153out_unreg_device:
3154 root_device_unregister(tcmu_root_device);
3155out_free_cache:
3156 kmem_cache_destroy(tcmu_cmd_cache);
3157
3158 return ret;
3159}
3160
3161static void __exit tcmu_module_exit(void)
3162{
3163 cancel_delayed_work_sync(&tcmu_unmap_work);
3164 target_backend_unregister(&tcmu_ops);
3165 kfree(tcmu_attrs);
3166 genl_unregister_family(&tcmu_genl_family);
3167 root_device_unregister(tcmu_root_device);
3168 kmem_cache_destroy(tcmu_cmd_cache);
3169}
3170
3171MODULE_DESCRIPTION("TCM USER subsystem plugin");
3172MODULE_AUTHOR("Shaohua Li <shli@kernel.org>");
3173MODULE_AUTHOR("Andy Grover <agrover@redhat.com>");
3174MODULE_LICENSE("GPL");
3175
3176module_init(tcmu_module_init);
3177module_exit(tcmu_module_exit);
3178