1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20#include <linux/kernel.h>
21#include <linux/types.h>
22#include <linux/pci.h>
23#include <linux/list.h>
24#include <linux/moduleparam.h>
25#include <linux/module.h>
26#include <linux/spinlock.h>
27#include <linux/interrupt.h>
28#include <linux/delay.h>
29#include <linux/uio.h>
30#include <linux/uaccess.h>
31#include <linux/fs.h>
32#include <linux/compat.h>
33#include <linux/blkdev.h>
34#include <linux/mutex.h>
35#include <linux/poll.h>
36#include <linux/vmalloc.h>
37#include <linux/workqueue.h>
38#include <linux/irq_poll.h>
39
40#include <scsi/scsi.h>
41#include <scsi/scsi_cmnd.h>
42#include <scsi/scsi_device.h>
43#include <scsi/scsi_host.h>
44#include <scsi/scsi_dbg.h>
45#include <linux/dmi.h>
46
47#include "megaraid_sas_fusion.h"
48#include "megaraid_sas.h"
49
50
51extern void
52megasas_complete_cmd(struct megasas_instance *instance,
53 struct megasas_cmd *cmd, u8 alt_status);
54int
55wait_and_poll(struct megasas_instance *instance, struct megasas_cmd *cmd,
56 int seconds);
57
58int
59megasas_clear_intr_fusion(struct megasas_instance *instance);
60
61int megasas_transition_to_ready(struct megasas_instance *instance, int ocr);
62
63extern u32 megasas_dbg_lvl;
64int megasas_sriov_start_heartbeat(struct megasas_instance *instance,
65 int initial);
66extern struct megasas_mgmt_info megasas_mgmt_info;
67extern unsigned int resetwaittime;
68extern unsigned int dual_qdepth_disable;
69static void megasas_free_rdpq_fusion(struct megasas_instance *instance);
70static void megasas_free_reply_fusion(struct megasas_instance *instance);
71static inline
72void megasas_configure_queue_sizes(struct megasas_instance *instance);
73static void megasas_fusion_crash_dump(struct megasas_instance *instance);
74
75
76
77
78
79
80
81
82
83
84
85
86
87int
88megasas_adp_reset_wait_for_ready(struct megasas_instance *instance,
89 bool do_adp_reset,
90 int ocr_context)
91{
92 int ret = FAILED;
93
94
95
96
97
98 if (megasas_dbg_lvl & OCR_DEBUG)
99 dev_info(&instance->pdev->dev,
100 "Block access to PCI config space %s %d\n",
101 __func__, __LINE__);
102
103 pci_cfg_access_lock(instance->pdev);
104
105 if (do_adp_reset) {
106 if (instance->instancet->adp_reset
107 (instance, instance->reg_set))
108 goto out;
109 }
110
111
112 if (megasas_transition_to_ready(instance, ocr_context)) {
113 dev_warn(&instance->pdev->dev,
114 "Failed to transition controller to ready for scsi%d.\n",
115 instance->host->host_no);
116 goto out;
117 }
118
119 ret = SUCCESS;
120out:
121 if (megasas_dbg_lvl & OCR_DEBUG)
122 dev_info(&instance->pdev->dev,
123 "Unlock access to PCI config space %s %d\n",
124 __func__, __LINE__);
125
126 pci_cfg_access_unlock(instance->pdev);
127
128 return ret;
129}
130
131
132
133
134
135
136
137
138
139
140
141
142static inline bool megasas_check_same_4gb_region
143 (struct megasas_instance *instance, dma_addr_t start_addr, size_t size)
144{
145 dma_addr_t end_addr;
146
147 end_addr = start_addr + size;
148
149 if (upper_32_bits(start_addr) != upper_32_bits(end_addr)) {
150 dev_err(&instance->pdev->dev,
151 "Failed to get same 4GB boundary: start_addr: 0x%llx end_addr: 0x%llx\n",
152 (unsigned long long)start_addr,
153 (unsigned long long)end_addr);
154 return false;
155 }
156
157 return true;
158}
159
160
161
162
163
164static void
165megasas_enable_intr_fusion(struct megasas_instance *instance)
166{
167 struct megasas_register_set __iomem *regs;
168 regs = instance->reg_set;
169
170 instance->mask_interrupts = 0;
171
172 writel(~0, ®s->outbound_intr_status);
173 readl(®s->outbound_intr_status);
174
175 writel(~MFI_FUSION_ENABLE_INTERRUPT_MASK, &(regs)->outbound_intr_mask);
176
177
178 dev_info(&instance->pdev->dev, "%s is called outbound_intr_mask:0x%08x\n",
179 __func__, readl(®s->outbound_intr_mask));
180}
181
182
183
184
185
186static void
187megasas_disable_intr_fusion(struct megasas_instance *instance)
188{
189 u32 mask = 0xFFFFFFFF;
190 struct megasas_register_set __iomem *regs;
191 regs = instance->reg_set;
192 instance->mask_interrupts = 1;
193
194 writel(mask, ®s->outbound_intr_mask);
195
196 dev_info(&instance->pdev->dev, "%s is called outbound_intr_mask:0x%08x\n",
197 __func__, readl(®s->outbound_intr_mask));
198}
199
200int
201megasas_clear_intr_fusion(struct megasas_instance *instance)
202{
203 u32 status;
204 struct megasas_register_set __iomem *regs;
205 regs = instance->reg_set;
206
207
208
209 status = megasas_readl(instance,
210 ®s->outbound_intr_status);
211
212 if (status & 1) {
213 writel(status, ®s->outbound_intr_status);
214 readl(®s->outbound_intr_status);
215 return 1;
216 }
217 if (!(status & MFI_FUSION_ENABLE_INTERRUPT_MASK))
218 return 0;
219
220 return 1;
221}
222
223static inline void
224megasas_sdev_busy_inc(struct megasas_instance *instance,
225 struct scsi_cmnd *scmd)
226{
227 if (instance->perf_mode == MR_BALANCED_PERF_MODE) {
228 struct MR_PRIV_DEVICE *mr_device_priv_data =
229 scmd->device->hostdata;
230 atomic_inc(&mr_device_priv_data->sdev_priv_busy);
231 }
232}
233
234static inline void
235megasas_sdev_busy_dec(struct megasas_instance *instance,
236 struct scsi_cmnd *scmd)
237{
238 if (instance->perf_mode == MR_BALANCED_PERF_MODE) {
239 struct MR_PRIV_DEVICE *mr_device_priv_data =
240 scmd->device->hostdata;
241 atomic_dec(&mr_device_priv_data->sdev_priv_busy);
242 }
243}
244
245static inline int
246megasas_sdev_busy_read(struct megasas_instance *instance,
247 struct scsi_cmnd *scmd)
248{
249 if (instance->perf_mode == MR_BALANCED_PERF_MODE) {
250 struct MR_PRIV_DEVICE *mr_device_priv_data =
251 scmd->device->hostdata;
252 return atomic_read(&mr_device_priv_data->sdev_priv_busy);
253 }
254 return 0;
255}
256
257
258
259
260
261
262
263
264inline struct megasas_cmd_fusion *megasas_get_cmd_fusion(struct megasas_instance
265 *instance, u32 blk_tag)
266{
267 struct fusion_context *fusion;
268
269 fusion = instance->ctrl_context;
270 return fusion->cmd_list[blk_tag];
271}
272
273
274
275
276
277
278inline void megasas_return_cmd_fusion(struct megasas_instance *instance,
279 struct megasas_cmd_fusion *cmd)
280{
281 cmd->scmd = NULL;
282 memset(cmd->io_request, 0, MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE);
283 cmd->r1_alt_dev_handle = MR_DEVHANDLE_INVALID;
284 cmd->cmd_completed = false;
285}
286
287
288
289
290
291
292static void
293megasas_write_64bit_req_desc(struct megasas_instance *instance,
294 union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc)
295{
296#if defined(writeq) && defined(CONFIG_64BIT)
297 u64 req_data = (((u64)le32_to_cpu(req_desc->u.high) << 32) |
298 le32_to_cpu(req_desc->u.low));
299 writeq(req_data, &instance->reg_set->inbound_low_queue_port);
300#else
301 unsigned long flags;
302 spin_lock_irqsave(&instance->hba_lock, flags);
303 writel(le32_to_cpu(req_desc->u.low),
304 &instance->reg_set->inbound_low_queue_port);
305 writel(le32_to_cpu(req_desc->u.high),
306 &instance->reg_set->inbound_high_queue_port);
307 spin_unlock_irqrestore(&instance->hba_lock, flags);
308#endif
309}
310
311
312
313
314
315
316
317
318
319static void
320megasas_fire_cmd_fusion(struct megasas_instance *instance,
321 union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc)
322{
323 if (instance->atomic_desc_support)
324 writel(le32_to_cpu(req_desc->u.low),
325 &instance->reg_set->inbound_single_queue_port);
326 else
327 megasas_write_64bit_req_desc(instance, req_desc);
328}
329
330
331
332
333
334
335
336
337
338
339
340
341static void
342megasas_fusion_update_can_queue(struct megasas_instance *instance, int fw_boot_context)
343{
344 u16 cur_max_fw_cmds = 0;
345 u16 ldio_threshold = 0;
346
347
348 if (instance->adapter_type < VENTURA_SERIES)
349 cur_max_fw_cmds =
350 megasas_readl(instance,
351 &instance->reg_set->outbound_scratch_pad_2) & 0x00FFFF;
352
353 if (dual_qdepth_disable || !cur_max_fw_cmds)
354 cur_max_fw_cmds = instance->instancet->read_fw_status_reg(instance) & 0x00FFFF;
355 else
356 ldio_threshold =
357 (instance->instancet->read_fw_status_reg(instance) & 0x00FFFF) - MEGASAS_FUSION_IOCTL_CMDS;
358
359 dev_info(&instance->pdev->dev,
360 "Current firmware supports maximum commands: %d\t LDIO threshold: %d\n",
361 cur_max_fw_cmds, ldio_threshold);
362
363 if (fw_boot_context == OCR_CONTEXT) {
364 cur_max_fw_cmds = cur_max_fw_cmds - 1;
365 if (cur_max_fw_cmds < instance->max_fw_cmds) {
366 instance->cur_can_queue =
367 cur_max_fw_cmds - (MEGASAS_FUSION_INTERNAL_CMDS +
368 MEGASAS_FUSION_IOCTL_CMDS);
369 instance->host->can_queue = instance->cur_can_queue;
370 instance->ldio_threshold = ldio_threshold;
371 }
372 } else {
373 instance->max_fw_cmds = cur_max_fw_cmds;
374 instance->ldio_threshold = ldio_threshold;
375
376 if (reset_devices)
377 instance->max_fw_cmds = min(instance->max_fw_cmds,
378 (u16)MEGASAS_KDUMP_QUEUE_DEPTH);
379
380
381
382
383
384 instance->max_fw_cmds = instance->max_fw_cmds-1;
385 }
386}
387
388static inline void
389megasas_get_msix_index(struct megasas_instance *instance,
390 struct scsi_cmnd *scmd,
391 struct megasas_cmd_fusion *cmd,
392 u8 data_arms)
393{
394 if (instance->perf_mode == MR_BALANCED_PERF_MODE &&
395 (megasas_sdev_busy_read(instance, scmd) >
396 (data_arms * MR_DEVICE_HIGH_IOPS_DEPTH))) {
397 cmd->request_desc->SCSIIO.MSIxIndex =
398 mega_mod64((atomic64_add_return(1, &instance->high_iops_outstanding) /
399 MR_HIGH_IOPS_BATCH_COUNT), instance->low_latency_index_start);
400 } else if (instance->msix_load_balance) {
401 cmd->request_desc->SCSIIO.MSIxIndex =
402 (mega_mod64(atomic64_add_return(1, &instance->total_io_count),
403 instance->msix_vectors));
404 } else if (instance->host->nr_hw_queues > 1) {
405 u32 tag = blk_mq_unique_tag(scmd->request);
406
407 cmd->request_desc->SCSIIO.MSIxIndex = blk_mq_unique_tag_to_hwq(tag) +
408 instance->low_latency_index_start;
409 } else {
410 cmd->request_desc->SCSIIO.MSIxIndex =
411 instance->reply_map[raw_smp_processor_id()];
412 }
413}
414
415
416
417
418
419void
420megasas_free_cmds_fusion(struct megasas_instance *instance)
421{
422 int i;
423 struct fusion_context *fusion = instance->ctrl_context;
424 struct megasas_cmd_fusion *cmd;
425
426 if (fusion->sense)
427 dma_pool_free(fusion->sense_dma_pool, fusion->sense,
428 fusion->sense_phys_addr);
429
430
431 if (fusion->cmd_list) {
432 for (i = 0; i < instance->max_mpt_cmds; i++) {
433 cmd = fusion->cmd_list[i];
434 if (cmd) {
435 if (cmd->sg_frame)
436 dma_pool_free(fusion->sg_dma_pool,
437 cmd->sg_frame,
438 cmd->sg_frame_phys_addr);
439 }
440 kfree(cmd);
441 }
442 kfree(fusion->cmd_list);
443 }
444
445 if (fusion->sg_dma_pool) {
446 dma_pool_destroy(fusion->sg_dma_pool);
447 fusion->sg_dma_pool = NULL;
448 }
449 if (fusion->sense_dma_pool) {
450 dma_pool_destroy(fusion->sense_dma_pool);
451 fusion->sense_dma_pool = NULL;
452 }
453
454
455
456 if (instance->is_rdpq)
457 megasas_free_rdpq_fusion(instance);
458 else
459 megasas_free_reply_fusion(instance);
460
461
462 if (fusion->req_frames_desc)
463 dma_free_coherent(&instance->pdev->dev,
464 fusion->request_alloc_sz, fusion->req_frames_desc,
465 fusion->req_frames_desc_phys);
466 if (fusion->io_request_frames)
467 dma_pool_free(fusion->io_request_frames_pool,
468 fusion->io_request_frames,
469 fusion->io_request_frames_phys);
470 if (fusion->io_request_frames_pool) {
471 dma_pool_destroy(fusion->io_request_frames_pool);
472 fusion->io_request_frames_pool = NULL;
473 }
474}
475
476
477
478
479
480
481static int megasas_create_sg_sense_fusion(struct megasas_instance *instance)
482{
483 int i;
484 u16 max_cmd;
485 struct fusion_context *fusion;
486 struct megasas_cmd_fusion *cmd;
487 int sense_sz;
488 u32 offset;
489
490 fusion = instance->ctrl_context;
491 max_cmd = instance->max_fw_cmds;
492 sense_sz = instance->max_mpt_cmds * SCSI_SENSE_BUFFERSIZE;
493
494 fusion->sg_dma_pool =
495 dma_pool_create("mr_sg", &instance->pdev->dev,
496 instance->max_chain_frame_sz,
497 MR_DEFAULT_NVME_PAGE_SIZE, 0);
498
499 fusion->sense_dma_pool =
500 dma_pool_create("mr_sense", &instance->pdev->dev,
501 sense_sz, 64, 0);
502
503 if (!fusion->sense_dma_pool || !fusion->sg_dma_pool) {
504 dev_err(&instance->pdev->dev,
505 "Failed from %s %d\n", __func__, __LINE__);
506 return -ENOMEM;
507 }
508
509 fusion->sense = dma_pool_alloc(fusion->sense_dma_pool,
510 GFP_KERNEL, &fusion->sense_phys_addr);
511 if (!fusion->sense) {
512 dev_err(&instance->pdev->dev,
513 "failed from %s %d\n", __func__, __LINE__);
514 return -ENOMEM;
515 }
516
517
518
519
520
521
522
523
524
525
526
527
528 if (!megasas_check_same_4gb_region(instance, fusion->sense_phys_addr,
529 sense_sz)) {
530 dma_pool_free(fusion->sense_dma_pool, fusion->sense,
531 fusion->sense_phys_addr);
532 fusion->sense = NULL;
533 dma_pool_destroy(fusion->sense_dma_pool);
534
535 fusion->sense_dma_pool =
536 dma_pool_create("mr_sense_align", &instance->pdev->dev,
537 sense_sz, roundup_pow_of_two(sense_sz),
538 0);
539 if (!fusion->sense_dma_pool) {
540 dev_err(&instance->pdev->dev,
541 "Failed from %s %d\n", __func__, __LINE__);
542 return -ENOMEM;
543 }
544 fusion->sense = dma_pool_alloc(fusion->sense_dma_pool,
545 GFP_KERNEL,
546 &fusion->sense_phys_addr);
547 if (!fusion->sense) {
548 dev_err(&instance->pdev->dev,
549 "failed from %s %d\n", __func__, __LINE__);
550 return -ENOMEM;
551 }
552 }
553
554
555
556
557 for (i = 0; i < max_cmd; i++) {
558 cmd = fusion->cmd_list[i];
559 cmd->sg_frame = dma_pool_alloc(fusion->sg_dma_pool,
560 GFP_KERNEL, &cmd->sg_frame_phys_addr);
561
562 offset = SCSI_SENSE_BUFFERSIZE * i;
563 cmd->sense = (u8 *)fusion->sense + offset;
564 cmd->sense_phys_addr = fusion->sense_phys_addr + offset;
565
566 if (!cmd->sg_frame) {
567 dev_err(&instance->pdev->dev,
568 "Failed from %s %d\n", __func__, __LINE__);
569 return -ENOMEM;
570 }
571 }
572
573
574 for (i = max_cmd; i < instance->max_mpt_cmds; i++) {
575 cmd = fusion->cmd_list[i];
576 offset = SCSI_SENSE_BUFFERSIZE * i;
577 cmd->sense = (u8 *)fusion->sense + offset;
578 cmd->sense_phys_addr = fusion->sense_phys_addr + offset;
579
580 }
581
582 return 0;
583}
584
585static int
586megasas_alloc_cmdlist_fusion(struct megasas_instance *instance)
587{
588 u32 max_mpt_cmd, i, j;
589 struct fusion_context *fusion;
590
591 fusion = instance->ctrl_context;
592
593 max_mpt_cmd = instance->max_mpt_cmds;
594
595
596
597
598
599
600 fusion->cmd_list =
601 kcalloc(max_mpt_cmd, sizeof(struct megasas_cmd_fusion *),
602 GFP_KERNEL);
603 if (!fusion->cmd_list) {
604 dev_err(&instance->pdev->dev,
605 "Failed from %s %d\n", __func__, __LINE__);
606 return -ENOMEM;
607 }
608
609 for (i = 0; i < max_mpt_cmd; i++) {
610 fusion->cmd_list[i] = kzalloc(sizeof(struct megasas_cmd_fusion),
611 GFP_KERNEL);
612 if (!fusion->cmd_list[i]) {
613 for (j = 0; j < i; j++)
614 kfree(fusion->cmd_list[j]);
615 kfree(fusion->cmd_list);
616 dev_err(&instance->pdev->dev,
617 "Failed from %s %d\n", __func__, __LINE__);
618 return -ENOMEM;
619 }
620 }
621
622 return 0;
623}
624
625static int
626megasas_alloc_request_fusion(struct megasas_instance *instance)
627{
628 struct fusion_context *fusion;
629
630 fusion = instance->ctrl_context;
631
632retry_alloc:
633 fusion->io_request_frames_pool =
634 dma_pool_create("mr_ioreq", &instance->pdev->dev,
635 fusion->io_frames_alloc_sz, 16, 0);
636
637 if (!fusion->io_request_frames_pool) {
638 dev_err(&instance->pdev->dev,
639 "Failed from %s %d\n", __func__, __LINE__);
640 return -ENOMEM;
641 }
642
643 fusion->io_request_frames =
644 dma_pool_alloc(fusion->io_request_frames_pool,
645 GFP_KERNEL | __GFP_NOWARN,
646 &fusion->io_request_frames_phys);
647 if (!fusion->io_request_frames) {
648 if (instance->max_fw_cmds >= (MEGASAS_REDUCE_QD_COUNT * 2)) {
649 instance->max_fw_cmds -= MEGASAS_REDUCE_QD_COUNT;
650 dma_pool_destroy(fusion->io_request_frames_pool);
651 megasas_configure_queue_sizes(instance);
652 goto retry_alloc;
653 } else {
654 dev_err(&instance->pdev->dev,
655 "Failed from %s %d\n", __func__, __LINE__);
656 return -ENOMEM;
657 }
658 }
659
660 if (!megasas_check_same_4gb_region(instance,
661 fusion->io_request_frames_phys,
662 fusion->io_frames_alloc_sz)) {
663 dma_pool_free(fusion->io_request_frames_pool,
664 fusion->io_request_frames,
665 fusion->io_request_frames_phys);
666 fusion->io_request_frames = NULL;
667 dma_pool_destroy(fusion->io_request_frames_pool);
668
669 fusion->io_request_frames_pool =
670 dma_pool_create("mr_ioreq_align",
671 &instance->pdev->dev,
672 fusion->io_frames_alloc_sz,
673 roundup_pow_of_two(fusion->io_frames_alloc_sz),
674 0);
675
676 if (!fusion->io_request_frames_pool) {
677 dev_err(&instance->pdev->dev,
678 "Failed from %s %d\n", __func__, __LINE__);
679 return -ENOMEM;
680 }
681
682 fusion->io_request_frames =
683 dma_pool_alloc(fusion->io_request_frames_pool,
684 GFP_KERNEL | __GFP_NOWARN,
685 &fusion->io_request_frames_phys);
686
687 if (!fusion->io_request_frames) {
688 dev_err(&instance->pdev->dev,
689 "Failed from %s %d\n", __func__, __LINE__);
690 return -ENOMEM;
691 }
692 }
693
694 fusion->req_frames_desc =
695 dma_alloc_coherent(&instance->pdev->dev,
696 fusion->request_alloc_sz,
697 &fusion->req_frames_desc_phys, GFP_KERNEL);
698 if (!fusion->req_frames_desc) {
699 dev_err(&instance->pdev->dev,
700 "Failed from %s %d\n", __func__, __LINE__);
701 return -ENOMEM;
702 }
703
704 return 0;
705}
706
707static int
708megasas_alloc_reply_fusion(struct megasas_instance *instance)
709{
710 int i, count;
711 struct fusion_context *fusion;
712 union MPI2_REPLY_DESCRIPTORS_UNION *reply_desc;
713 fusion = instance->ctrl_context;
714
715 count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
716 count += instance->iopoll_q_count;
717
718 fusion->reply_frames_desc_pool =
719 dma_pool_create("mr_reply", &instance->pdev->dev,
720 fusion->reply_alloc_sz * count, 16, 0);
721
722 if (!fusion->reply_frames_desc_pool) {
723 dev_err(&instance->pdev->dev,
724 "Failed from %s %d\n", __func__, __LINE__);
725 return -ENOMEM;
726 }
727
728 fusion->reply_frames_desc[0] =
729 dma_pool_alloc(fusion->reply_frames_desc_pool,
730 GFP_KERNEL, &fusion->reply_frames_desc_phys[0]);
731 if (!fusion->reply_frames_desc[0]) {
732 dev_err(&instance->pdev->dev,
733 "Failed from %s %d\n", __func__, __LINE__);
734 return -ENOMEM;
735 }
736
737 if (!megasas_check_same_4gb_region(instance,
738 fusion->reply_frames_desc_phys[0],
739 (fusion->reply_alloc_sz * count))) {
740 dma_pool_free(fusion->reply_frames_desc_pool,
741 fusion->reply_frames_desc[0],
742 fusion->reply_frames_desc_phys[0]);
743 fusion->reply_frames_desc[0] = NULL;
744 dma_pool_destroy(fusion->reply_frames_desc_pool);
745
746 fusion->reply_frames_desc_pool =
747 dma_pool_create("mr_reply_align",
748 &instance->pdev->dev,
749 fusion->reply_alloc_sz * count,
750 roundup_pow_of_two(fusion->reply_alloc_sz * count),
751 0);
752
753 if (!fusion->reply_frames_desc_pool) {
754 dev_err(&instance->pdev->dev,
755 "Failed from %s %d\n", __func__, __LINE__);
756 return -ENOMEM;
757 }
758
759 fusion->reply_frames_desc[0] =
760 dma_pool_alloc(fusion->reply_frames_desc_pool,
761 GFP_KERNEL,
762 &fusion->reply_frames_desc_phys[0]);
763
764 if (!fusion->reply_frames_desc[0]) {
765 dev_err(&instance->pdev->dev,
766 "Failed from %s %d\n", __func__, __LINE__);
767 return -ENOMEM;
768 }
769 }
770
771 reply_desc = fusion->reply_frames_desc[0];
772 for (i = 0; i < fusion->reply_q_depth * count; i++, reply_desc++)
773 reply_desc->Words = cpu_to_le64(ULLONG_MAX);
774
775
776
777
778 for (i = 0; i < (count - 1); i++)
779 fusion->reply_frames_desc[i + 1] =
780 fusion->reply_frames_desc[i] +
781 (fusion->reply_alloc_sz)/sizeof(union MPI2_REPLY_DESCRIPTORS_UNION);
782
783 return 0;
784}
785
786static int
787megasas_alloc_rdpq_fusion(struct megasas_instance *instance)
788{
789 int i, j, k, msix_count;
790 struct fusion_context *fusion;
791 union MPI2_REPLY_DESCRIPTORS_UNION *reply_desc;
792 union MPI2_REPLY_DESCRIPTORS_UNION *rdpq_chunk_virt[RDPQ_MAX_CHUNK_COUNT];
793 dma_addr_t rdpq_chunk_phys[RDPQ_MAX_CHUNK_COUNT];
794 u8 dma_alloc_count, abs_index;
795 u32 chunk_size, array_size, offset;
796
797 fusion = instance->ctrl_context;
798 chunk_size = fusion->reply_alloc_sz * RDPQ_MAX_INDEX_IN_ONE_CHUNK;
799 array_size = sizeof(struct MPI2_IOC_INIT_RDPQ_ARRAY_ENTRY) *
800 MAX_MSIX_QUEUES_FUSION;
801
802 fusion->rdpq_virt = dma_alloc_coherent(&instance->pdev->dev,
803 array_size, &fusion->rdpq_phys,
804 GFP_KERNEL);
805 if (!fusion->rdpq_virt) {
806 dev_err(&instance->pdev->dev,
807 "Failed from %s %d\n", __func__, __LINE__);
808 return -ENOMEM;
809 }
810
811 msix_count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
812 msix_count += instance->iopoll_q_count;
813
814 fusion->reply_frames_desc_pool = dma_pool_create("mr_rdpq",
815 &instance->pdev->dev,
816 chunk_size, 16, 0);
817 fusion->reply_frames_desc_pool_align =
818 dma_pool_create("mr_rdpq_align",
819 &instance->pdev->dev,
820 chunk_size,
821 roundup_pow_of_two(chunk_size),
822 0);
823
824 if (!fusion->reply_frames_desc_pool ||
825 !fusion->reply_frames_desc_pool_align) {
826 dev_err(&instance->pdev->dev,
827 "Failed from %s %d\n", __func__, __LINE__);
828 return -ENOMEM;
829 }
830
831
832
833
834
835
836
837
838
839 dma_alloc_count = DIV_ROUND_UP(msix_count, RDPQ_MAX_INDEX_IN_ONE_CHUNK);
840
841 for (i = 0; i < dma_alloc_count; i++) {
842 rdpq_chunk_virt[i] =
843 dma_pool_alloc(fusion->reply_frames_desc_pool,
844 GFP_KERNEL, &rdpq_chunk_phys[i]);
845 if (!rdpq_chunk_virt[i]) {
846 dev_err(&instance->pdev->dev,
847 "Failed from %s %d\n", __func__, __LINE__);
848 return -ENOMEM;
849 }
850
851
852
853
854
855
856
857
858
859
860
861
862 if (!megasas_check_same_4gb_region(instance, rdpq_chunk_phys[i],
863 chunk_size)) {
864 dma_pool_free(fusion->reply_frames_desc_pool,
865 rdpq_chunk_virt[i],
866 rdpq_chunk_phys[i]);
867
868 rdpq_chunk_virt[i] =
869 dma_pool_alloc(fusion->reply_frames_desc_pool_align,
870 GFP_KERNEL, &rdpq_chunk_phys[i]);
871 if (!rdpq_chunk_virt[i]) {
872 dev_err(&instance->pdev->dev,
873 "Failed from %s %d\n",
874 __func__, __LINE__);
875 return -ENOMEM;
876 }
877 fusion->rdpq_tracker[i].dma_pool_ptr =
878 fusion->reply_frames_desc_pool_align;
879 } else {
880 fusion->rdpq_tracker[i].dma_pool_ptr =
881 fusion->reply_frames_desc_pool;
882 }
883
884 fusion->rdpq_tracker[i].pool_entry_phys = rdpq_chunk_phys[i];
885 fusion->rdpq_tracker[i].pool_entry_virt = rdpq_chunk_virt[i];
886 }
887
888 for (k = 0; k < dma_alloc_count; k++) {
889 for (i = 0; i < RDPQ_MAX_INDEX_IN_ONE_CHUNK; i++) {
890 abs_index = (k * RDPQ_MAX_INDEX_IN_ONE_CHUNK) + i;
891
892 if (abs_index == msix_count)
893 break;
894 offset = fusion->reply_alloc_sz * i;
895 fusion->rdpq_virt[abs_index].RDPQBaseAddress =
896 cpu_to_le64(rdpq_chunk_phys[k] + offset);
897 fusion->reply_frames_desc_phys[abs_index] =
898 rdpq_chunk_phys[k] + offset;
899 fusion->reply_frames_desc[abs_index] =
900 (union MPI2_REPLY_DESCRIPTORS_UNION *)((u8 *)rdpq_chunk_virt[k] + offset);
901
902 reply_desc = fusion->reply_frames_desc[abs_index];
903 for (j = 0; j < fusion->reply_q_depth; j++, reply_desc++)
904 reply_desc->Words = ULLONG_MAX;
905 }
906 }
907
908 return 0;
909}
910
911static void
912megasas_free_rdpq_fusion(struct megasas_instance *instance) {
913
914 int i;
915 struct fusion_context *fusion;
916
917 fusion = instance->ctrl_context;
918
919 for (i = 0; i < RDPQ_MAX_CHUNK_COUNT; i++) {
920 if (fusion->rdpq_tracker[i].pool_entry_virt)
921 dma_pool_free(fusion->rdpq_tracker[i].dma_pool_ptr,
922 fusion->rdpq_tracker[i].pool_entry_virt,
923 fusion->rdpq_tracker[i].pool_entry_phys);
924
925 }
926
927 dma_pool_destroy(fusion->reply_frames_desc_pool);
928 dma_pool_destroy(fusion->reply_frames_desc_pool_align);
929
930 if (fusion->rdpq_virt)
931 dma_free_coherent(&instance->pdev->dev,
932 sizeof(struct MPI2_IOC_INIT_RDPQ_ARRAY_ENTRY) * MAX_MSIX_QUEUES_FUSION,
933 fusion->rdpq_virt, fusion->rdpq_phys);
934}
935
936static void
937megasas_free_reply_fusion(struct megasas_instance *instance) {
938
939 struct fusion_context *fusion;
940
941 fusion = instance->ctrl_context;
942
943 if (fusion->reply_frames_desc[0])
944 dma_pool_free(fusion->reply_frames_desc_pool,
945 fusion->reply_frames_desc[0],
946 fusion->reply_frames_desc_phys[0]);
947
948 dma_pool_destroy(fusion->reply_frames_desc_pool);
949
950}
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969static int
970megasas_alloc_cmds_fusion(struct megasas_instance *instance)
971{
972 int i;
973 struct fusion_context *fusion;
974 struct megasas_cmd_fusion *cmd;
975 u32 offset;
976 dma_addr_t io_req_base_phys;
977 u8 *io_req_base;
978
979
980 fusion = instance->ctrl_context;
981
982 if (megasas_alloc_request_fusion(instance))
983 goto fail_exit;
984
985 if (instance->is_rdpq) {
986 if (megasas_alloc_rdpq_fusion(instance))
987 goto fail_exit;
988 } else
989 if (megasas_alloc_reply_fusion(instance))
990 goto fail_exit;
991
992 if (megasas_alloc_cmdlist_fusion(instance))
993 goto fail_exit;
994
995
996 io_req_base = fusion->io_request_frames + MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE;
997 io_req_base_phys = fusion->io_request_frames_phys + MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE;
998
999
1000
1001
1002
1003
1004 for (i = 0; i < instance->max_mpt_cmds; i++) {
1005 cmd = fusion->cmd_list[i];
1006 offset = MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE * i;
1007 memset(cmd, 0, sizeof(struct megasas_cmd_fusion));
1008 cmd->index = i + 1;
1009 cmd->scmd = NULL;
1010 cmd->sync_cmd_idx =
1011 (i >= instance->max_scsi_cmds && i < instance->max_fw_cmds) ?
1012 (i - instance->max_scsi_cmds) :
1013 (u32)ULONG_MAX;
1014 cmd->instance = instance;
1015 cmd->io_request =
1016 (struct MPI2_RAID_SCSI_IO_REQUEST *)
1017 (io_req_base + offset);
1018 memset(cmd->io_request, 0,
1019 sizeof(struct MPI2_RAID_SCSI_IO_REQUEST));
1020 cmd->io_request_phys_addr = io_req_base_phys + offset;
1021 cmd->r1_alt_dev_handle = MR_DEVHANDLE_INVALID;
1022 }
1023
1024 if (megasas_create_sg_sense_fusion(instance))
1025 goto fail_exit;
1026
1027 return 0;
1028
1029fail_exit:
1030 megasas_free_cmds_fusion(instance);
1031 return -ENOMEM;
1032}
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042int
1043wait_and_poll(struct megasas_instance *instance, struct megasas_cmd *cmd,
1044 int seconds)
1045{
1046 int i;
1047 struct megasas_header *frame_hdr = &cmd->frame->hdr;
1048 u32 status_reg;
1049
1050 u32 msecs = seconds * 1000;
1051
1052
1053
1054
1055 for (i = 0; (i < msecs) && (frame_hdr->cmd_status == 0xff); i += 20) {
1056 rmb();
1057 msleep(20);
1058 if (!(i % 5000)) {
1059 status_reg = instance->instancet->read_fw_status_reg(instance)
1060 & MFI_STATE_MASK;
1061 if (status_reg == MFI_STATE_FAULT)
1062 break;
1063 }
1064 }
1065
1066 if (frame_hdr->cmd_status == MFI_STAT_INVALID_STATUS)
1067 return DCMD_TIMEOUT;
1068 else if (frame_hdr->cmd_status == MFI_STAT_OK)
1069 return DCMD_SUCCESS;
1070 else
1071 return DCMD_FAILED;
1072}
1073
1074
1075
1076
1077
1078
1079
1080int
1081megasas_ioc_init_fusion(struct megasas_instance *instance)
1082{
1083 struct megasas_init_frame *init_frame;
1084 struct MPI2_IOC_INIT_REQUEST *IOCInitMessage = NULL;
1085 dma_addr_t ioc_init_handle;
1086 struct megasas_cmd *cmd;
1087 u8 ret, cur_rdpq_mode;
1088 struct fusion_context *fusion;
1089 union MEGASAS_REQUEST_DESCRIPTOR_UNION req_desc;
1090 int i;
1091 struct megasas_header *frame_hdr;
1092 const char *sys_info;
1093 MFI_CAPABILITIES *drv_ops;
1094 u32 scratch_pad_1;
1095 ktime_t time;
1096 bool cur_fw_64bit_dma_capable;
1097 bool cur_intr_coalescing;
1098
1099 fusion = instance->ctrl_context;
1100
1101 ioc_init_handle = fusion->ioc_init_request_phys;
1102 IOCInitMessage = fusion->ioc_init_request;
1103
1104 cmd = fusion->ioc_init_cmd;
1105
1106 scratch_pad_1 = megasas_readl
1107 (instance, &instance->reg_set->outbound_scratch_pad_1);
1108
1109 cur_rdpq_mode = (scratch_pad_1 & MR_RDPQ_MODE_OFFSET) ? 1 : 0;
1110
1111 if (instance->adapter_type == INVADER_SERIES) {
1112 cur_fw_64bit_dma_capable =
1113 (scratch_pad_1 & MR_CAN_HANDLE_64_BIT_DMA_OFFSET) ? true : false;
1114
1115 if (instance->consistent_mask_64bit && !cur_fw_64bit_dma_capable) {
1116 dev_err(&instance->pdev->dev, "Driver was operating on 64bit "
1117 "DMA mask, but upcoming FW does not support 64bit DMA mask\n");
1118 megaraid_sas_kill_hba(instance);
1119 ret = 1;
1120 goto fail_fw_init;
1121 }
1122 }
1123
1124 if (instance->is_rdpq && !cur_rdpq_mode) {
1125 dev_err(&instance->pdev->dev, "Firmware downgrade *NOT SUPPORTED*"
1126 " from RDPQ mode to non RDPQ mode\n");
1127 ret = 1;
1128 goto fail_fw_init;
1129 }
1130
1131 cur_intr_coalescing = (scratch_pad_1 & MR_INTR_COALESCING_SUPPORT_OFFSET) ?
1132 true : false;
1133
1134 if ((instance->low_latency_index_start ==
1135 MR_HIGH_IOPS_QUEUE_COUNT) && cur_intr_coalescing)
1136 instance->perf_mode = MR_BALANCED_PERF_MODE;
1137
1138 dev_info(&instance->pdev->dev, "Performance mode :%s (latency index = %d)\n",
1139 MEGASAS_PERF_MODE_2STR(instance->perf_mode),
1140 instance->low_latency_index_start);
1141
1142 instance->fw_sync_cache_support = (scratch_pad_1 &
1143 MR_CAN_HANDLE_SYNC_CACHE_OFFSET) ? 1 : 0;
1144 dev_info(&instance->pdev->dev, "FW supports sync cache\t: %s\n",
1145 instance->fw_sync_cache_support ? "Yes" : "No");
1146
1147 memset(IOCInitMessage, 0, sizeof(struct MPI2_IOC_INIT_REQUEST));
1148
1149 IOCInitMessage->Function = MPI2_FUNCTION_IOC_INIT;
1150 IOCInitMessage->WhoInit = MPI2_WHOINIT_HOST_DRIVER;
1151 IOCInitMessage->MsgVersion = cpu_to_le16(MPI2_VERSION);
1152 IOCInitMessage->HeaderVersion = cpu_to_le16(MPI2_HEADER_VERSION);
1153 IOCInitMessage->SystemRequestFrameSize = cpu_to_le16(MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE / 4);
1154
1155 IOCInitMessage->ReplyDescriptorPostQueueDepth = cpu_to_le16(fusion->reply_q_depth);
1156 IOCInitMessage->ReplyDescriptorPostQueueAddress = instance->is_rdpq ?
1157 cpu_to_le64(fusion->rdpq_phys) :
1158 cpu_to_le64(fusion->reply_frames_desc_phys[0]);
1159 IOCInitMessage->MsgFlags = instance->is_rdpq ?
1160 MPI2_IOCINIT_MSGFLAG_RDPQ_ARRAY_MODE : 0;
1161 IOCInitMessage->SystemRequestFrameBaseAddress = cpu_to_le64(fusion->io_request_frames_phys);
1162 IOCInitMessage->SenseBufferAddressHigh = cpu_to_le32(upper_32_bits(fusion->sense_phys_addr));
1163 IOCInitMessage->HostMSIxVectors = instance->msix_vectors + instance->iopoll_q_count;
1164 IOCInitMessage->HostPageSize = MR_DEFAULT_NVME_PAGE_SHIFT;
1165
1166 time = ktime_get_real();
1167
1168 IOCInitMessage->TimeStamp = cpu_to_le64(ktime_to_ms(time));
1169
1170 init_frame = (struct megasas_init_frame *)cmd->frame;
1171 memset(init_frame, 0, IOC_INIT_FRAME_SIZE);
1172
1173 frame_hdr = &cmd->frame->hdr;
1174 frame_hdr->cmd_status = 0xFF;
1175 frame_hdr->flags |= cpu_to_le16(MFI_FRAME_DONT_POST_IN_REPLY_QUEUE);
1176
1177 init_frame->cmd = MFI_CMD_INIT;
1178 init_frame->cmd_status = 0xFF;
1179
1180 drv_ops = (MFI_CAPABILITIES *) &(init_frame->driver_operations);
1181
1182
1183 if (instance->adapter_type >= INVADER_SERIES)
1184 drv_ops->mfi_capabilities.support_additional_msix = 1;
1185
1186 drv_ops->mfi_capabilities.support_fp_remote_lun = 1;
1187
1188 drv_ops->mfi_capabilities.support_max_255lds = 1;
1189 drv_ops->mfi_capabilities.support_ndrive_r1_lb = 1;
1190 drv_ops->mfi_capabilities.security_protocol_cmds_fw = 1;
1191
1192 if (instance->max_chain_frame_sz > MEGASAS_CHAIN_FRAME_SZ_MIN)
1193 drv_ops->mfi_capabilities.support_ext_io_size = 1;
1194
1195 drv_ops->mfi_capabilities.support_fp_rlbypass = 1;
1196 if (!dual_qdepth_disable)
1197 drv_ops->mfi_capabilities.support_ext_queue_depth = 1;
1198
1199 drv_ops->mfi_capabilities.support_qd_throttling = 1;
1200 drv_ops->mfi_capabilities.support_pd_map_target_id = 1;
1201 drv_ops->mfi_capabilities.support_nvme_passthru = 1;
1202 drv_ops->mfi_capabilities.support_fw_exposed_dev_list = 1;
1203
1204 if (instance->consistent_mask_64bit)
1205 drv_ops->mfi_capabilities.support_64bit_mode = 1;
1206
1207
1208 cpu_to_le32s((u32 *)&init_frame->driver_operations.mfi_capabilities);
1209
1210 sys_info = dmi_get_system_info(DMI_PRODUCT_UUID);
1211 if (instance->system_info_buf && sys_info) {
1212 memcpy(instance->system_info_buf->systemId, sys_info,
1213 strlen(sys_info) > 64 ? 64 : strlen(sys_info));
1214 instance->system_info_buf->systemIdLength =
1215 strlen(sys_info) > 64 ? 64 : strlen(sys_info);
1216 init_frame->system_info_lo = cpu_to_le32(lower_32_bits(instance->system_info_h));
1217 init_frame->system_info_hi = cpu_to_le32(upper_32_bits(instance->system_info_h));
1218 }
1219
1220 init_frame->queue_info_new_phys_addr_hi =
1221 cpu_to_le32(upper_32_bits(ioc_init_handle));
1222 init_frame->queue_info_new_phys_addr_lo =
1223 cpu_to_le32(lower_32_bits(ioc_init_handle));
1224 init_frame->data_xfer_len = cpu_to_le32(sizeof(struct MPI2_IOC_INIT_REQUEST));
1225
1226
1227
1228
1229
1230 switch (instance->perf_mode) {
1231 case MR_BALANCED_PERF_MODE:
1232 init_frame->replyqueue_mask =
1233 cpu_to_le16(~(~0 << instance->low_latency_index_start/8));
1234 break;
1235 case MR_IOPS_PERF_MODE:
1236 init_frame->replyqueue_mask =
1237 cpu_to_le16(~(~0 << instance->msix_vectors/8));
1238 break;
1239 }
1240
1241
1242 req_desc.u.low = cpu_to_le32(lower_32_bits(cmd->frame_phys_addr));
1243 req_desc.u.high = cpu_to_le32(upper_32_bits(cmd->frame_phys_addr));
1244 req_desc.MFAIo.RequestFlags =
1245 (MEGASAS_REQ_DESCRIPT_FLAGS_MFA <<
1246 MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
1247
1248
1249
1250
1251 instance->instancet->disable_intr(instance);
1252
1253 for (i = 0; i < (10 * 1000); i += 20) {
1254 if (megasas_readl(instance, &instance->reg_set->doorbell) & 1)
1255 msleep(20);
1256 else
1257 break;
1258 }
1259
1260
1261 megasas_write_64bit_req_desc(instance, &req_desc);
1262
1263 wait_and_poll(instance, cmd, MFI_IO_TIMEOUT_SECS);
1264
1265 frame_hdr = &cmd->frame->hdr;
1266 if (frame_hdr->cmd_status != 0) {
1267 ret = 1;
1268 goto fail_fw_init;
1269 }
1270
1271 if (instance->adapter_type >= AERO_SERIES) {
1272 scratch_pad_1 = megasas_readl
1273 (instance, &instance->reg_set->outbound_scratch_pad_1);
1274
1275 instance->atomic_desc_support =
1276 (scratch_pad_1 & MR_ATOMIC_DESCRIPTOR_SUPPORT_OFFSET) ? 1 : 0;
1277
1278 dev_info(&instance->pdev->dev, "FW supports atomic descriptor\t: %s\n",
1279 instance->atomic_desc_support ? "Yes" : "No");
1280 }
1281
1282 return 0;
1283
1284fail_fw_init:
1285 dev_err(&instance->pdev->dev,
1286 "Init cmd return status FAILED for SCSI host %d\n",
1287 instance->host->host_no);
1288
1289 return ret;
1290}
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301int
1302megasas_sync_pd_seq_num(struct megasas_instance *instance, bool pend) {
1303 int ret = 0;
1304 size_t pd_seq_map_sz;
1305 struct megasas_cmd *cmd;
1306 struct megasas_dcmd_frame *dcmd;
1307 struct fusion_context *fusion = instance->ctrl_context;
1308 struct MR_PD_CFG_SEQ_NUM_SYNC *pd_sync;
1309 dma_addr_t pd_seq_h;
1310
1311 pd_sync = (void *)fusion->pd_seq_sync[(instance->pd_seq_map_id & 1)];
1312 pd_seq_h = fusion->pd_seq_phys[(instance->pd_seq_map_id & 1)];
1313 pd_seq_map_sz = struct_size(pd_sync, seq, MAX_PHYSICAL_DEVICES - 1);
1314
1315 cmd = megasas_get_cmd(instance);
1316 if (!cmd) {
1317 dev_err(&instance->pdev->dev,
1318 "Could not get mfi cmd. Fail from %s %d\n",
1319 __func__, __LINE__);
1320 return -ENOMEM;
1321 }
1322
1323 dcmd = &cmd->frame->dcmd;
1324
1325 memset(pd_sync, 0, pd_seq_map_sz);
1326 memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE);
1327
1328 if (pend) {
1329 dcmd->mbox.b[0] = MEGASAS_DCMD_MBOX_PEND_FLAG;
1330 dcmd->flags = MFI_FRAME_DIR_WRITE;
1331 instance->jbod_seq_cmd = cmd;
1332 } else {
1333 dcmd->flags = MFI_FRAME_DIR_READ;
1334 }
1335
1336 dcmd->cmd = MFI_CMD_DCMD;
1337 dcmd->cmd_status = 0xFF;
1338 dcmd->sge_count = 1;
1339 dcmd->timeout = 0;
1340 dcmd->pad_0 = 0;
1341 dcmd->data_xfer_len = cpu_to_le32(pd_seq_map_sz);
1342 dcmd->opcode = cpu_to_le32(MR_DCMD_SYSTEM_PD_MAP_GET_INFO);
1343
1344 megasas_set_dma_settings(instance, dcmd, pd_seq_h, pd_seq_map_sz);
1345
1346 if (pend) {
1347 instance->instancet->issue_dcmd(instance, cmd);
1348 return 0;
1349 }
1350
1351
1352 if (!instance->mask_interrupts)
1353 ret = megasas_issue_blocked_cmd(instance, cmd,
1354 MFI_IO_TIMEOUT_SECS);
1355 else
1356 ret = megasas_issue_polled(instance, cmd);
1357
1358 if (le32_to_cpu(pd_sync->count) > MAX_PHYSICAL_DEVICES) {
1359 dev_warn(&instance->pdev->dev,
1360 "driver supports max %d JBOD, but FW reports %d\n",
1361 MAX_PHYSICAL_DEVICES, le32_to_cpu(pd_sync->count));
1362 ret = -EINVAL;
1363 }
1364
1365 if (ret == DCMD_TIMEOUT)
1366 dev_warn(&instance->pdev->dev,
1367 "%s DCMD timed out, continue without JBOD sequence map\n",
1368 __func__);
1369
1370 if (ret == DCMD_SUCCESS)
1371 instance->pd_seq_map_id++;
1372
1373 megasas_return_cmd(instance, cmd);
1374 return ret;
1375}
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392static int
1393megasas_get_ld_map_info(struct megasas_instance *instance)
1394{
1395 int ret = 0;
1396 struct megasas_cmd *cmd;
1397 struct megasas_dcmd_frame *dcmd;
1398 void *ci;
1399 dma_addr_t ci_h = 0;
1400 u32 size_map_info;
1401 struct fusion_context *fusion;
1402
1403 cmd = megasas_get_cmd(instance);
1404
1405 if (!cmd) {
1406 dev_printk(KERN_DEBUG, &instance->pdev->dev, "Failed to get cmd for map info\n");
1407 return -ENOMEM;
1408 }
1409
1410 fusion = instance->ctrl_context;
1411
1412 if (!fusion) {
1413 megasas_return_cmd(instance, cmd);
1414 return -ENXIO;
1415 }
1416
1417 dcmd = &cmd->frame->dcmd;
1418
1419 size_map_info = fusion->current_map_sz;
1420
1421 ci = (void *) fusion->ld_map[(instance->map_id & 1)];
1422 ci_h = fusion->ld_map_phys[(instance->map_id & 1)];
1423
1424 if (!ci) {
1425 dev_printk(KERN_DEBUG, &instance->pdev->dev, "Failed to alloc mem for ld_map_info\n");
1426 megasas_return_cmd(instance, cmd);
1427 return -ENOMEM;
1428 }
1429
1430 memset(ci, 0, fusion->max_map_sz);
1431 memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE);
1432 dcmd->cmd = MFI_CMD_DCMD;
1433 dcmd->cmd_status = 0xFF;
1434 dcmd->sge_count = 1;
1435 dcmd->flags = MFI_FRAME_DIR_READ;
1436 dcmd->timeout = 0;
1437 dcmd->pad_0 = 0;
1438 dcmd->data_xfer_len = cpu_to_le32(size_map_info);
1439 dcmd->opcode = cpu_to_le32(MR_DCMD_LD_MAP_GET_INFO);
1440
1441 megasas_set_dma_settings(instance, dcmd, ci_h, size_map_info);
1442
1443 if (!instance->mask_interrupts)
1444 ret = megasas_issue_blocked_cmd(instance, cmd,
1445 MFI_IO_TIMEOUT_SECS);
1446 else
1447 ret = megasas_issue_polled(instance, cmd);
1448
1449 if (ret == DCMD_TIMEOUT)
1450 dev_warn(&instance->pdev->dev,
1451 "%s DCMD timed out, RAID map is disabled\n",
1452 __func__);
1453
1454 megasas_return_cmd(instance, cmd);
1455
1456 return ret;
1457}
1458
1459u8
1460megasas_get_map_info(struct megasas_instance *instance)
1461{
1462 struct fusion_context *fusion = instance->ctrl_context;
1463
1464 fusion->fast_path_io = 0;
1465 if (!megasas_get_ld_map_info(instance)) {
1466 if (MR_ValidateMapInfo(instance, instance->map_id)) {
1467 fusion->fast_path_io = 1;
1468 return 0;
1469 }
1470 }
1471 return 1;
1472}
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482int
1483megasas_sync_map_info(struct megasas_instance *instance)
1484{
1485 int i;
1486 struct megasas_cmd *cmd;
1487 struct megasas_dcmd_frame *dcmd;
1488 u16 num_lds;
1489 struct fusion_context *fusion;
1490 struct MR_LD_TARGET_SYNC *ci = NULL;
1491 struct MR_DRV_RAID_MAP_ALL *map;
1492 struct MR_LD_RAID *raid;
1493 struct MR_LD_TARGET_SYNC *ld_sync;
1494 dma_addr_t ci_h = 0;
1495 u32 size_map_info;
1496
1497 cmd = megasas_get_cmd(instance);
1498
1499 if (!cmd) {
1500 dev_printk(KERN_DEBUG, &instance->pdev->dev, "Failed to get cmd for sync info\n");
1501 return -ENOMEM;
1502 }
1503
1504 fusion = instance->ctrl_context;
1505
1506 if (!fusion) {
1507 megasas_return_cmd(instance, cmd);
1508 return 1;
1509 }
1510
1511 map = fusion->ld_drv_map[instance->map_id & 1];
1512
1513 num_lds = le16_to_cpu(map->raidMap.ldCount);
1514
1515 dcmd = &cmd->frame->dcmd;
1516
1517 memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE);
1518
1519 ci = (struct MR_LD_TARGET_SYNC *)
1520 fusion->ld_map[(instance->map_id - 1) & 1];
1521 memset(ci, 0, fusion->max_map_sz);
1522
1523 ci_h = fusion->ld_map_phys[(instance->map_id - 1) & 1];
1524
1525 ld_sync = (struct MR_LD_TARGET_SYNC *)ci;
1526
1527 for (i = 0; i < num_lds; i++, ld_sync++) {
1528 raid = MR_LdRaidGet(i, map);
1529 ld_sync->targetId = MR_GetLDTgtId(i, map);
1530 ld_sync->seqNum = raid->seqNum;
1531 }
1532
1533 size_map_info = fusion->current_map_sz;
1534
1535 dcmd->cmd = MFI_CMD_DCMD;
1536 dcmd->cmd_status = 0xFF;
1537 dcmd->sge_count = 1;
1538 dcmd->flags = MFI_FRAME_DIR_WRITE;
1539 dcmd->timeout = 0;
1540 dcmd->pad_0 = 0;
1541 dcmd->data_xfer_len = cpu_to_le32(size_map_info);
1542 dcmd->mbox.b[0] = num_lds;
1543 dcmd->mbox.b[1] = MEGASAS_DCMD_MBOX_PEND_FLAG;
1544 dcmd->opcode = cpu_to_le32(MR_DCMD_LD_MAP_GET_INFO);
1545
1546 megasas_set_dma_settings(instance, dcmd, ci_h, size_map_info);
1547
1548 instance->map_update_cmd = cmd;
1549
1550 instance->instancet->issue_dcmd(instance, cmd);
1551
1552 return 0;
1553}
1554
1555
1556
1557
1558
1559
1560
1561static void
1562megasas_display_intel_branding(struct megasas_instance *instance)
1563{
1564 if (instance->pdev->subsystem_vendor != PCI_VENDOR_ID_INTEL)
1565 return;
1566
1567 switch (instance->pdev->device) {
1568 case PCI_DEVICE_ID_LSI_INVADER:
1569 switch (instance->pdev->subsystem_device) {
1570 case MEGARAID_INTEL_RS3DC080_SSDID:
1571 dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1572 instance->host->host_no,
1573 MEGARAID_INTEL_RS3DC080_BRANDING);
1574 break;
1575 case MEGARAID_INTEL_RS3DC040_SSDID:
1576 dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1577 instance->host->host_no,
1578 MEGARAID_INTEL_RS3DC040_BRANDING);
1579 break;
1580 case MEGARAID_INTEL_RS3SC008_SSDID:
1581 dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1582 instance->host->host_no,
1583 MEGARAID_INTEL_RS3SC008_BRANDING);
1584 break;
1585 case MEGARAID_INTEL_RS3MC044_SSDID:
1586 dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1587 instance->host->host_no,
1588 MEGARAID_INTEL_RS3MC044_BRANDING);
1589 break;
1590 default:
1591 break;
1592 }
1593 break;
1594 case PCI_DEVICE_ID_LSI_FURY:
1595 switch (instance->pdev->subsystem_device) {
1596 case MEGARAID_INTEL_RS3WC080_SSDID:
1597 dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1598 instance->host->host_no,
1599 MEGARAID_INTEL_RS3WC080_BRANDING);
1600 break;
1601 case MEGARAID_INTEL_RS3WC040_SSDID:
1602 dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1603 instance->host->host_no,
1604 MEGARAID_INTEL_RS3WC040_BRANDING);
1605 break;
1606 default:
1607 break;
1608 }
1609 break;
1610 case PCI_DEVICE_ID_LSI_CUTLASS_52:
1611 case PCI_DEVICE_ID_LSI_CUTLASS_53:
1612 switch (instance->pdev->subsystem_device) {
1613 case MEGARAID_INTEL_RMS3BC160_SSDID:
1614 dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1615 instance->host->host_no,
1616 MEGARAID_INTEL_RMS3BC160_BRANDING);
1617 break;
1618 default:
1619 break;
1620 }
1621 break;
1622 default:
1623 break;
1624 }
1625}
1626
1627
1628
1629
1630
1631
1632
1633
1634static inline int megasas_allocate_raid_maps(struct megasas_instance *instance)
1635{
1636 struct fusion_context *fusion;
1637 int i = 0;
1638
1639 fusion = instance->ctrl_context;
1640
1641 fusion->drv_map_pages = get_order(fusion->drv_map_sz);
1642
1643 for (i = 0; i < 2; i++) {
1644 fusion->ld_map[i] = NULL;
1645
1646 fusion->ld_drv_map[i] = (void *)
1647 __get_free_pages(__GFP_ZERO | GFP_KERNEL,
1648 fusion->drv_map_pages);
1649
1650 if (!fusion->ld_drv_map[i]) {
1651 fusion->ld_drv_map[i] = vzalloc(fusion->drv_map_sz);
1652
1653 if (!fusion->ld_drv_map[i]) {
1654 dev_err(&instance->pdev->dev,
1655 "Could not allocate memory for local map"
1656 " size requested: %d\n",
1657 fusion->drv_map_sz);
1658 goto ld_drv_map_alloc_fail;
1659 }
1660 }
1661 }
1662
1663 for (i = 0; i < 2; i++) {
1664 fusion->ld_map[i] = dma_alloc_coherent(&instance->pdev->dev,
1665 fusion->max_map_sz,
1666 &fusion->ld_map_phys[i],
1667 GFP_KERNEL);
1668 if (!fusion->ld_map[i]) {
1669 dev_err(&instance->pdev->dev,
1670 "Could not allocate memory for map info %s:%d\n",
1671 __func__, __LINE__);
1672 goto ld_map_alloc_fail;
1673 }
1674 }
1675
1676 return 0;
1677
1678ld_map_alloc_fail:
1679 for (i = 0; i < 2; i++) {
1680 if (fusion->ld_map[i])
1681 dma_free_coherent(&instance->pdev->dev,
1682 fusion->max_map_sz,
1683 fusion->ld_map[i],
1684 fusion->ld_map_phys[i]);
1685 }
1686
1687ld_drv_map_alloc_fail:
1688 for (i = 0; i < 2; i++) {
1689 if (fusion->ld_drv_map[i]) {
1690 if (is_vmalloc_addr(fusion->ld_drv_map[i]))
1691 vfree(fusion->ld_drv_map[i]);
1692 else
1693 free_pages((ulong)fusion->ld_drv_map[i],
1694 fusion->drv_map_pages);
1695 }
1696 }
1697
1698 return -ENOMEM;
1699}
1700
1701
1702
1703
1704
1705
1706
1707
1708static inline
1709void megasas_configure_queue_sizes(struct megasas_instance *instance)
1710{
1711 struct fusion_context *fusion;
1712 u16 max_cmd;
1713
1714 fusion = instance->ctrl_context;
1715 max_cmd = instance->max_fw_cmds;
1716
1717 if (instance->adapter_type >= VENTURA_SERIES)
1718 instance->max_mpt_cmds = instance->max_fw_cmds * RAID_1_PEER_CMDS;
1719 else
1720 instance->max_mpt_cmds = instance->max_fw_cmds;
1721
1722 instance->max_scsi_cmds = instance->max_fw_cmds - instance->max_mfi_cmds;
1723 instance->cur_can_queue = instance->max_scsi_cmds;
1724 instance->host->can_queue = instance->cur_can_queue;
1725
1726 fusion->reply_q_depth = 2 * ((max_cmd + 1 + 15) / 16) * 16;
1727
1728 fusion->request_alloc_sz = sizeof(union MEGASAS_REQUEST_DESCRIPTOR_UNION) *
1729 instance->max_mpt_cmds;
1730 fusion->reply_alloc_sz = sizeof(union MPI2_REPLY_DESCRIPTORS_UNION) *
1731 (fusion->reply_q_depth);
1732 fusion->io_frames_alloc_sz = MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE +
1733 (MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE
1734 * (instance->max_mpt_cmds + 1));
1735}
1736
1737static int megasas_alloc_ioc_init_frame(struct megasas_instance *instance)
1738{
1739 struct fusion_context *fusion;
1740 struct megasas_cmd *cmd;
1741
1742 fusion = instance->ctrl_context;
1743
1744 cmd = kzalloc(sizeof(struct megasas_cmd), GFP_KERNEL);
1745
1746 if (!cmd) {
1747 dev_err(&instance->pdev->dev, "Failed from func: %s line: %d\n",
1748 __func__, __LINE__);
1749 return -ENOMEM;
1750 }
1751
1752 cmd->frame = dma_alloc_coherent(&instance->pdev->dev,
1753 IOC_INIT_FRAME_SIZE,
1754 &cmd->frame_phys_addr, GFP_KERNEL);
1755
1756 if (!cmd->frame) {
1757 dev_err(&instance->pdev->dev, "Failed from func: %s line: %d\n",
1758 __func__, __LINE__);
1759 kfree(cmd);
1760 return -ENOMEM;
1761 }
1762
1763 fusion->ioc_init_cmd = cmd;
1764 return 0;
1765}
1766
1767
1768
1769
1770
1771static inline void megasas_free_ioc_init_cmd(struct megasas_instance *instance)
1772{
1773 struct fusion_context *fusion;
1774
1775 fusion = instance->ctrl_context;
1776
1777 if (fusion->ioc_init_cmd && fusion->ioc_init_cmd->frame)
1778 dma_free_coherent(&instance->pdev->dev,
1779 IOC_INIT_FRAME_SIZE,
1780 fusion->ioc_init_cmd->frame,
1781 fusion->ioc_init_cmd->frame_phys_addr);
1782
1783 kfree(fusion->ioc_init_cmd);
1784}
1785
1786
1787
1788
1789
1790
1791
1792static u32
1793megasas_init_adapter_fusion(struct megasas_instance *instance)
1794{
1795 struct fusion_context *fusion;
1796 u32 scratch_pad_1;
1797 int i = 0, count;
1798 u32 status_reg;
1799
1800 fusion = instance->ctrl_context;
1801
1802 megasas_fusion_update_can_queue(instance, PROBE_CONTEXT);
1803
1804
1805
1806
1807 instance->max_mfi_cmds =
1808 MEGASAS_FUSION_INTERNAL_CMDS + MEGASAS_FUSION_IOCTL_CMDS;
1809
1810 megasas_configure_queue_sizes(instance);
1811
1812 scratch_pad_1 = megasas_readl(instance,
1813 &instance->reg_set->outbound_scratch_pad_1);
1814
1815
1816
1817
1818
1819
1820 if (scratch_pad_1 & MEGASAS_MAX_CHAIN_SIZE_UNITS_MASK)
1821 instance->max_chain_frame_sz =
1822 ((scratch_pad_1 & MEGASAS_MAX_CHAIN_SIZE_MASK) >>
1823 MEGASAS_MAX_CHAIN_SHIFT) * MEGASAS_1MB_IO;
1824 else
1825 instance->max_chain_frame_sz =
1826 ((scratch_pad_1 & MEGASAS_MAX_CHAIN_SIZE_MASK) >>
1827 MEGASAS_MAX_CHAIN_SHIFT) * MEGASAS_256K_IO;
1828
1829 if (instance->max_chain_frame_sz < MEGASAS_CHAIN_FRAME_SZ_MIN) {
1830 dev_warn(&instance->pdev->dev, "frame size %d invalid, fall back to legacy max frame size %d\n",
1831 instance->max_chain_frame_sz,
1832 MEGASAS_CHAIN_FRAME_SZ_MIN);
1833 instance->max_chain_frame_sz = MEGASAS_CHAIN_FRAME_SZ_MIN;
1834 }
1835
1836 fusion->max_sge_in_main_msg =
1837 (MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE
1838 - offsetof(struct MPI2_RAID_SCSI_IO_REQUEST, SGL))/16;
1839
1840 fusion->max_sge_in_chain =
1841 instance->max_chain_frame_sz
1842 / sizeof(union MPI2_SGE_IO_UNION);
1843
1844 instance->max_num_sge =
1845 rounddown_pow_of_two(fusion->max_sge_in_main_msg
1846 + fusion->max_sge_in_chain - 2);
1847
1848
1849 fusion->chain_offset_mfi_pthru =
1850 offsetof(struct MPI2_RAID_SCSI_IO_REQUEST, SGL)/16;
1851
1852 fusion->chain_offset_io_request =
1853 (MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE -
1854 sizeof(union MPI2_SGE_IO_UNION))/16;
1855
1856 count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
1857 count += instance->iopoll_q_count;
1858
1859 for (i = 0 ; i < count; i++)
1860 fusion->last_reply_idx[i] = 0;
1861
1862
1863
1864
1865
1866 instance->max_scsi_cmds = instance->max_fw_cmds -
1867 (MEGASAS_FUSION_INTERNAL_CMDS +
1868 MEGASAS_FUSION_IOCTL_CMDS);
1869 sema_init(&instance->ioctl_sem, MEGASAS_FUSION_IOCTL_CMDS);
1870
1871 for (i = 0; i < MAX_MSIX_QUEUES_FUSION; i++)
1872 atomic_set(&fusion->busy_mq_poll[i], 0);
1873
1874 if (megasas_alloc_ioc_init_frame(instance))
1875 return 1;
1876
1877
1878
1879
1880
1881 if (megasas_alloc_cmds(instance))
1882 goto fail_alloc_mfi_cmds;
1883 if (megasas_alloc_cmds_fusion(instance))
1884 goto fail_alloc_cmds;
1885
1886 if (megasas_ioc_init_fusion(instance)) {
1887 status_reg = instance->instancet->read_fw_status_reg(instance);
1888 if (((status_reg & MFI_STATE_MASK) == MFI_STATE_FAULT) &&
1889 (status_reg & MFI_RESET_ADAPTER)) {
1890
1891 if (megasas_adp_reset_wait_for_ready
1892 (instance, true, 0) == FAILED)
1893 goto fail_ioc_init;
1894
1895 if (megasas_ioc_init_fusion(instance))
1896 goto fail_ioc_init;
1897 } else {
1898 goto fail_ioc_init;
1899 }
1900 }
1901
1902 megasas_display_intel_branding(instance);
1903 if (megasas_get_ctrl_info(instance)) {
1904 dev_err(&instance->pdev->dev,
1905 "Could not get controller info. Fail from %s %d\n",
1906 __func__, __LINE__);
1907 goto fail_ioc_init;
1908 }
1909
1910 instance->flag_ieee = 1;
1911 instance->r1_ldio_hint_default = MR_R1_LDIO_PIGGYBACK_DEFAULT;
1912 instance->threshold_reply_count = instance->max_fw_cmds / 4;
1913 fusion->fast_path_io = 0;
1914
1915 if (megasas_allocate_raid_maps(instance))
1916 goto fail_ioc_init;
1917
1918 if (!megasas_get_map_info(instance))
1919 megasas_sync_map_info(instance);
1920
1921 return 0;
1922
1923fail_ioc_init:
1924 megasas_free_cmds_fusion(instance);
1925fail_alloc_cmds:
1926 megasas_free_cmds(instance);
1927fail_alloc_mfi_cmds:
1928 megasas_free_ioc_init_cmd(instance);
1929 return 1;
1930}
1931
1932
1933
1934
1935
1936
1937static void
1938megasas_fault_detect_work(struct work_struct *work)
1939{
1940 struct megasas_instance *instance =
1941 container_of(work, struct megasas_instance,
1942 fw_fault_work.work);
1943 u32 fw_state, dma_state, status;
1944
1945
1946 fw_state = instance->instancet->read_fw_status_reg(instance) &
1947 MFI_STATE_MASK;
1948
1949 if (fw_state == MFI_STATE_FAULT) {
1950 dma_state = instance->instancet->read_fw_status_reg(instance) &
1951 MFI_STATE_DMADONE;
1952
1953 if (instance->crash_dump_drv_support &&
1954 instance->crash_dump_app_support && dma_state) {
1955 megasas_fusion_crash_dump(instance);
1956 } else {
1957 if (instance->unload == 0) {
1958 status = megasas_reset_fusion(instance->host, 0);
1959 if (status != SUCCESS) {
1960 dev_err(&instance->pdev->dev,
1961 "Failed from %s %d, do not re-arm timer\n",
1962 __func__, __LINE__);
1963 return;
1964 }
1965 }
1966 }
1967 }
1968
1969 if (instance->fw_fault_work_q)
1970 queue_delayed_work(instance->fw_fault_work_q,
1971 &instance->fw_fault_work,
1972 msecs_to_jiffies(MEGASAS_WATCHDOG_THREAD_INTERVAL));
1973}
1974
1975int
1976megasas_fusion_start_watchdog(struct megasas_instance *instance)
1977{
1978
1979 if (instance->fw_fault_work_q)
1980 return SUCCESS;
1981
1982 INIT_DELAYED_WORK(&instance->fw_fault_work, megasas_fault_detect_work);
1983
1984 snprintf(instance->fault_handler_work_q_name,
1985 sizeof(instance->fault_handler_work_q_name),
1986 "poll_megasas%d_status", instance->host->host_no);
1987
1988 instance->fw_fault_work_q =
1989 create_singlethread_workqueue(instance->fault_handler_work_q_name);
1990 if (!instance->fw_fault_work_q) {
1991 dev_err(&instance->pdev->dev, "Failed from %s %d\n",
1992 __func__, __LINE__);
1993 return FAILED;
1994 }
1995
1996 queue_delayed_work(instance->fw_fault_work_q,
1997 &instance->fw_fault_work,
1998 msecs_to_jiffies(MEGASAS_WATCHDOG_THREAD_INTERVAL));
1999
2000 return SUCCESS;
2001}
2002
2003void
2004megasas_fusion_stop_watchdog(struct megasas_instance *instance)
2005{
2006 struct workqueue_struct *wq;
2007
2008 if (instance->fw_fault_work_q) {
2009 wq = instance->fw_fault_work_q;
2010 instance->fw_fault_work_q = NULL;
2011 if (!cancel_delayed_work_sync(&instance->fw_fault_work))
2012 flush_workqueue(wq);
2013 destroy_workqueue(wq);
2014 }
2015}
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026static void
2027map_cmd_status(struct fusion_context *fusion,
2028 struct scsi_cmnd *scmd, u8 status, u8 ext_status,
2029 u32 data_length, u8 *sense)
2030{
2031 u8 cmd_type;
2032 int resid;
2033
2034 cmd_type = megasas_cmd_type(scmd);
2035 switch (status) {
2036
2037 case MFI_STAT_OK:
2038 scmd->result = DID_OK << 16;
2039 break;
2040
2041 case MFI_STAT_SCSI_IO_FAILED:
2042 case MFI_STAT_LD_INIT_IN_PROGRESS:
2043 scmd->result = (DID_ERROR << 16) | ext_status;
2044 break;
2045
2046 case MFI_STAT_SCSI_DONE_WITH_ERROR:
2047
2048 scmd->result = (DID_OK << 16) | ext_status;
2049 if (ext_status == SAM_STAT_CHECK_CONDITION) {
2050 memset(scmd->sense_buffer, 0,
2051 SCSI_SENSE_BUFFERSIZE);
2052 memcpy(scmd->sense_buffer, sense,
2053 SCSI_SENSE_BUFFERSIZE);
2054 }
2055
2056
2057
2058
2059
2060
2061
2062 resid = (scsi_bufflen(scmd) - data_length);
2063 scsi_set_resid(scmd, resid);
2064
2065 if (resid &&
2066 ((cmd_type == READ_WRITE_LDIO) ||
2067 (cmd_type == READ_WRITE_SYSPDIO)))
2068 scmd_printk(KERN_INFO, scmd, "BRCM Debug mfi stat 0x%x, data len"
2069 " requested/completed 0x%x/0x%x\n",
2070 status, scsi_bufflen(scmd), data_length);
2071 break;
2072
2073 case MFI_STAT_LD_OFFLINE:
2074 case MFI_STAT_DEVICE_NOT_FOUND:
2075 scmd->result = DID_BAD_TARGET << 16;
2076 break;
2077 case MFI_STAT_CONFIG_SEQ_MISMATCH:
2078 scmd->result = DID_IMM_RETRY << 16;
2079 break;
2080 default:
2081 scmd->result = DID_ERROR << 16;
2082 break;
2083 }
2084}
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097static bool
2098megasas_is_prp_possible(struct megasas_instance *instance,
2099 struct scsi_cmnd *scmd, int sge_count)
2100{
2101 u32 data_length = 0;
2102 struct scatterlist *sg_scmd;
2103 bool build_prp = false;
2104 u32 mr_nvme_pg_size;
2105
2106 mr_nvme_pg_size = max_t(u32, instance->nvme_page_size,
2107 MR_DEFAULT_NVME_PAGE_SIZE);
2108 data_length = scsi_bufflen(scmd);
2109 sg_scmd = scsi_sglist(scmd);
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120 if (data_length > (mr_nvme_pg_size * 5)) {
2121 build_prp = true;
2122 } else if ((data_length > (mr_nvme_pg_size * 4)) &&
2123 (data_length <= (mr_nvme_pg_size * 5))) {
2124
2125 if (sg_dma_len(sg_scmd) < (data_length - (mr_nvme_pg_size * 4)))
2126 build_prp = true;
2127 }
2128
2129 return build_prp;
2130}
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145static bool
2146megasas_make_prp_nvme(struct megasas_instance *instance, struct scsi_cmnd *scmd,
2147 struct MPI25_IEEE_SGE_CHAIN64 *sgl_ptr,
2148 struct megasas_cmd_fusion *cmd, int sge_count)
2149{
2150 int sge_len, offset, num_prp_in_chain = 0;
2151 struct MPI25_IEEE_SGE_CHAIN64 *main_chain_element, *ptr_first_sgl;
2152 u64 *ptr_sgl;
2153 dma_addr_t ptr_sgl_phys;
2154 u64 sge_addr;
2155 u32 page_mask, page_mask_result;
2156 struct scatterlist *sg_scmd;
2157 u32 first_prp_len;
2158 bool build_prp = false;
2159 int data_len = scsi_bufflen(scmd);
2160 u32 mr_nvme_pg_size = max_t(u32, instance->nvme_page_size,
2161 MR_DEFAULT_NVME_PAGE_SIZE);
2162
2163 build_prp = megasas_is_prp_possible(instance, scmd, sge_count);
2164
2165 if (!build_prp)
2166 return false;
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180 page_mask = mr_nvme_pg_size - 1;
2181 ptr_sgl = (u64 *)cmd->sg_frame;
2182 ptr_sgl_phys = cmd->sg_frame_phys_addr;
2183 memset(ptr_sgl, 0, instance->max_chain_frame_sz);
2184
2185
2186 main_chain_element = (struct MPI25_IEEE_SGE_CHAIN64 *)
2187 ((u8 *)sgl_ptr + sizeof(struct MPI25_IEEE_SGE_CHAIN64));
2188
2189 main_chain_element->Address = cpu_to_le64(ptr_sgl_phys);
2190 main_chain_element->NextChainOffset = 0;
2191 main_chain_element->Flags = IEEE_SGE_FLAGS_CHAIN_ELEMENT |
2192 IEEE_SGE_FLAGS_SYSTEM_ADDR |
2193 MPI26_IEEE_SGE_FLAGS_NSF_NVME_PRP;
2194
2195
2196 ptr_first_sgl = sgl_ptr;
2197 sg_scmd = scsi_sglist(scmd);
2198 sge_addr = sg_dma_address(sg_scmd);
2199 sge_len = sg_dma_len(sg_scmd);
2200
2201 offset = (u32)(sge_addr & page_mask);
2202 first_prp_len = mr_nvme_pg_size - offset;
2203
2204 ptr_first_sgl->Address = cpu_to_le64(sge_addr);
2205 ptr_first_sgl->Length = cpu_to_le32(first_prp_len);
2206
2207 data_len -= first_prp_len;
2208
2209 if (sge_len > first_prp_len) {
2210 sge_addr += first_prp_len;
2211 sge_len -= first_prp_len;
2212 } else if (sge_len == first_prp_len) {
2213 sg_scmd = sg_next(sg_scmd);
2214 sge_addr = sg_dma_address(sg_scmd);
2215 sge_len = sg_dma_len(sg_scmd);
2216 }
2217
2218 for (;;) {
2219 offset = (u32)(sge_addr & page_mask);
2220
2221
2222 page_mask_result = (uintptr_t)(ptr_sgl + 1) & page_mask;
2223 if (unlikely(!page_mask_result)) {
2224 scmd_printk(KERN_NOTICE,
2225 scmd, "page boundary ptr_sgl: 0x%p\n",
2226 ptr_sgl);
2227 ptr_sgl_phys += 8;
2228 *ptr_sgl = cpu_to_le64(ptr_sgl_phys);
2229 ptr_sgl++;
2230 num_prp_in_chain++;
2231 }
2232
2233 *ptr_sgl = cpu_to_le64(sge_addr);
2234 ptr_sgl++;
2235 ptr_sgl_phys += 8;
2236 num_prp_in_chain++;
2237
2238 sge_addr += mr_nvme_pg_size;
2239 sge_len -= mr_nvme_pg_size;
2240 data_len -= mr_nvme_pg_size;
2241
2242 if (data_len <= 0)
2243 break;
2244
2245 if (sge_len > 0)
2246 continue;
2247
2248 sg_scmd = sg_next(sg_scmd);
2249 sge_addr = sg_dma_address(sg_scmd);
2250 sge_len = sg_dma_len(sg_scmd);
2251 }
2252
2253 main_chain_element->Length =
2254 cpu_to_le32(num_prp_in_chain * sizeof(u64));
2255
2256 return build_prp;
2257}
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268static void
2269megasas_make_sgl_fusion(struct megasas_instance *instance,
2270 struct scsi_cmnd *scp,
2271 struct MPI25_IEEE_SGE_CHAIN64 *sgl_ptr,
2272 struct megasas_cmd_fusion *cmd, int sge_count)
2273{
2274 int i, sg_processed;
2275 struct scatterlist *os_sgl;
2276 struct fusion_context *fusion;
2277
2278 fusion = instance->ctrl_context;
2279
2280 if (instance->adapter_type >= INVADER_SERIES) {
2281 struct MPI25_IEEE_SGE_CHAIN64 *sgl_ptr_end = sgl_ptr;
2282 sgl_ptr_end += fusion->max_sge_in_main_msg - 1;
2283 sgl_ptr_end->Flags = 0;
2284 }
2285
2286 scsi_for_each_sg(scp, os_sgl, sge_count, i) {
2287 sgl_ptr->Length = cpu_to_le32(sg_dma_len(os_sgl));
2288 sgl_ptr->Address = cpu_to_le64(sg_dma_address(os_sgl));
2289 sgl_ptr->Flags = 0;
2290 if (instance->adapter_type >= INVADER_SERIES)
2291 if (i == sge_count - 1)
2292 sgl_ptr->Flags = IEEE_SGE_FLAGS_END_OF_LIST;
2293 sgl_ptr++;
2294 sg_processed = i + 1;
2295
2296 if ((sg_processed == (fusion->max_sge_in_main_msg - 1)) &&
2297 (sge_count > fusion->max_sge_in_main_msg)) {
2298
2299 struct MPI25_IEEE_SGE_CHAIN64 *sg_chain;
2300 if (instance->adapter_type >= INVADER_SERIES) {
2301 if ((le16_to_cpu(cmd->io_request->IoFlags) &
2302 MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH) !=
2303 MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH)
2304 cmd->io_request->ChainOffset =
2305 fusion->
2306 chain_offset_io_request;
2307 else
2308 cmd->io_request->ChainOffset = 0;
2309 } else
2310 cmd->io_request->ChainOffset =
2311 fusion->chain_offset_io_request;
2312
2313 sg_chain = sgl_ptr;
2314
2315 sg_chain->NextChainOffset = 0;
2316 if (instance->adapter_type >= INVADER_SERIES)
2317 sg_chain->Flags = IEEE_SGE_FLAGS_CHAIN_ELEMENT;
2318 else
2319 sg_chain->Flags =
2320 (IEEE_SGE_FLAGS_CHAIN_ELEMENT |
2321 MPI2_IEEE_SGE_FLAGS_IOCPLBNTA_ADDR);
2322 sg_chain->Length = cpu_to_le32((sizeof(union MPI2_SGE_IO_UNION) * (sge_count - sg_processed)));
2323 sg_chain->Address = cpu_to_le64(cmd->sg_frame_phys_addr);
2324
2325 sgl_ptr =
2326 (struct MPI25_IEEE_SGE_CHAIN64 *)cmd->sg_frame;
2327 memset(sgl_ptr, 0, instance->max_chain_frame_sz);
2328 }
2329 }
2330}
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345static
2346int megasas_make_sgl(struct megasas_instance *instance, struct scsi_cmnd *scp,
2347 struct megasas_cmd_fusion *cmd)
2348{
2349 int sge_count;
2350 bool build_prp = false;
2351 struct MPI25_IEEE_SGE_CHAIN64 *sgl_chain64;
2352
2353 sge_count = scsi_dma_map(scp);
2354
2355 if ((sge_count > instance->max_num_sge) || (sge_count <= 0))
2356 return sge_count;
2357
2358 sgl_chain64 = (struct MPI25_IEEE_SGE_CHAIN64 *)&cmd->io_request->SGL;
2359 if ((le16_to_cpu(cmd->io_request->IoFlags) &
2360 MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH) &&
2361 (cmd->pd_interface == NVME_PD))
2362 build_prp = megasas_make_prp_nvme(instance, scp, sgl_chain64,
2363 cmd, sge_count);
2364
2365 if (!build_prp)
2366 megasas_make_sgl_fusion(instance, scp, sgl_chain64,
2367 cmd, sge_count);
2368
2369 return sge_count;
2370}
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383static void
2384megasas_set_pd_lba(struct MPI2_RAID_SCSI_IO_REQUEST *io_request, u8 cdb_len,
2385 struct IO_REQUEST_INFO *io_info, struct scsi_cmnd *scp,
2386 struct MR_DRV_RAID_MAP_ALL *local_map_ptr, u32 ref_tag)
2387{
2388 struct MR_LD_RAID *raid;
2389 u16 ld;
2390 u64 start_blk = io_info->pdBlock;
2391 u8 *cdb = io_request->CDB.CDB32;
2392 u32 num_blocks = io_info->numBlocks;
2393 u8 opcode = 0, flagvals = 0, groupnum = 0, control = 0;
2394
2395
2396 ld = MR_TargetIdToLdGet(io_info->ldTgtId, local_map_ptr);
2397 raid = MR_LdRaidGet(ld, local_map_ptr);
2398 if (raid->capability.ldPiMode == MR_PROT_INFO_TYPE_CONTROLLER) {
2399 memset(cdb, 0, sizeof(io_request->CDB.CDB32));
2400 cdb[0] = MEGASAS_SCSI_VARIABLE_LENGTH_CMD;
2401 cdb[7] = MEGASAS_SCSI_ADDL_CDB_LEN;
2402
2403 if (scp->sc_data_direction == DMA_FROM_DEVICE)
2404 cdb[9] = MEGASAS_SCSI_SERVICE_ACTION_READ32;
2405 else
2406 cdb[9] = MEGASAS_SCSI_SERVICE_ACTION_WRITE32;
2407 cdb[10] = MEGASAS_RD_WR_PROTECT_CHECK_ALL;
2408
2409
2410 cdb[12] = (u8)((start_blk >> 56) & 0xff);
2411 cdb[13] = (u8)((start_blk >> 48) & 0xff);
2412 cdb[14] = (u8)((start_blk >> 40) & 0xff);
2413 cdb[15] = (u8)((start_blk >> 32) & 0xff);
2414 cdb[16] = (u8)((start_blk >> 24) & 0xff);
2415 cdb[17] = (u8)((start_blk >> 16) & 0xff);
2416 cdb[18] = (u8)((start_blk >> 8) & 0xff);
2417 cdb[19] = (u8)(start_blk & 0xff);
2418
2419
2420 io_request->CDB.EEDP32.PrimaryReferenceTag =
2421 cpu_to_be32(ref_tag);
2422 io_request->CDB.EEDP32.PrimaryApplicationTagMask = cpu_to_be16(0xffff);
2423 io_request->IoFlags = cpu_to_le16(32);
2424
2425
2426 cdb[28] = (u8)((num_blocks >> 24) & 0xff);
2427 cdb[29] = (u8)((num_blocks >> 16) & 0xff);
2428 cdb[30] = (u8)((num_blocks >> 8) & 0xff);
2429 cdb[31] = (u8)(num_blocks & 0xff);
2430
2431
2432 if (scp->sc_data_direction == DMA_FROM_DEVICE) {
2433 io_request->EEDPFlags = cpu_to_le16(
2434 MPI2_SCSIIO_EEDPFLAGS_INC_PRI_REFTAG |
2435 MPI2_SCSIIO_EEDPFLAGS_CHECK_REFTAG |
2436 MPI2_SCSIIO_EEDPFLAGS_CHECK_REMOVE_OP |
2437 MPI2_SCSIIO_EEDPFLAGS_CHECK_APPTAG |
2438 MPI25_SCSIIO_EEDPFLAGS_DO_NOT_DISABLE_MODE |
2439 MPI2_SCSIIO_EEDPFLAGS_CHECK_GUARD);
2440 } else {
2441 io_request->EEDPFlags = cpu_to_le16(
2442 MPI2_SCSIIO_EEDPFLAGS_INC_PRI_REFTAG |
2443 MPI2_SCSIIO_EEDPFLAGS_INSERT_OP);
2444 }
2445 io_request->Control |= cpu_to_le32((0x4 << 26));
2446 io_request->EEDPBlockSize = cpu_to_le32(scp->device->sector_size);
2447 } else {
2448
2449 if (((cdb_len == 12) || (cdb_len == 16)) &&
2450 (start_blk <= 0xffffffff)) {
2451 if (cdb_len == 16) {
2452 opcode = cdb[0] == READ_16 ? READ_10 : WRITE_10;
2453 flagvals = cdb[1];
2454 groupnum = cdb[14];
2455 control = cdb[15];
2456 } else {
2457 opcode = cdb[0] == READ_12 ? READ_10 : WRITE_10;
2458 flagvals = cdb[1];
2459 groupnum = cdb[10];
2460 control = cdb[11];
2461 }
2462
2463 memset(cdb, 0, sizeof(io_request->CDB.CDB32));
2464
2465 cdb[0] = opcode;
2466 cdb[1] = flagvals;
2467 cdb[6] = groupnum;
2468 cdb[9] = control;
2469
2470
2471 cdb[8] = (u8)(num_blocks & 0xff);
2472 cdb[7] = (u8)((num_blocks >> 8) & 0xff);
2473
2474 io_request->IoFlags = cpu_to_le16(10);
2475 cdb_len = 10;
2476 } else if ((cdb_len < 16) && (start_blk > 0xffffffff)) {
2477
2478 switch (cdb_len) {
2479 case 6:
2480 opcode = cdb[0] == READ_6 ? READ_16 : WRITE_16;
2481 control = cdb[5];
2482 break;
2483 case 10:
2484 opcode =
2485 cdb[0] == READ_10 ? READ_16 : WRITE_16;
2486 flagvals = cdb[1];
2487 groupnum = cdb[6];
2488 control = cdb[9];
2489 break;
2490 case 12:
2491 opcode =
2492 cdb[0] == READ_12 ? READ_16 : WRITE_16;
2493 flagvals = cdb[1];
2494 groupnum = cdb[10];
2495 control = cdb[11];
2496 break;
2497 }
2498
2499 memset(cdb, 0, sizeof(io_request->CDB.CDB32));
2500
2501 cdb[0] = opcode;
2502 cdb[1] = flagvals;
2503 cdb[14] = groupnum;
2504 cdb[15] = control;
2505
2506
2507 cdb[13] = (u8)(num_blocks & 0xff);
2508 cdb[12] = (u8)((num_blocks >> 8) & 0xff);
2509 cdb[11] = (u8)((num_blocks >> 16) & 0xff);
2510 cdb[10] = (u8)((num_blocks >> 24) & 0xff);
2511
2512 io_request->IoFlags = cpu_to_le16(16);
2513 cdb_len = 16;
2514 }
2515
2516
2517 switch (cdb_len) {
2518 case 6:
2519 {
2520 u8 val = cdb[1] & 0xE0;
2521 cdb[3] = (u8)(start_blk & 0xff);
2522 cdb[2] = (u8)((start_blk >> 8) & 0xff);
2523 cdb[1] = val | ((u8)(start_blk >> 16) & 0x1f);
2524 break;
2525 }
2526 case 10:
2527 cdb[5] = (u8)(start_blk & 0xff);
2528 cdb[4] = (u8)((start_blk >> 8) & 0xff);
2529 cdb[3] = (u8)((start_blk >> 16) & 0xff);
2530 cdb[2] = (u8)((start_blk >> 24) & 0xff);
2531 break;
2532 case 12:
2533 cdb[5] = (u8)(start_blk & 0xff);
2534 cdb[4] = (u8)((start_blk >> 8) & 0xff);
2535 cdb[3] = (u8)((start_blk >> 16) & 0xff);
2536 cdb[2] = (u8)((start_blk >> 24) & 0xff);
2537 break;
2538 case 16:
2539 cdb[9] = (u8)(start_blk & 0xff);
2540 cdb[8] = (u8)((start_blk >> 8) & 0xff);
2541 cdb[7] = (u8)((start_blk >> 16) & 0xff);
2542 cdb[6] = (u8)((start_blk >> 24) & 0xff);
2543 cdb[5] = (u8)((start_blk >> 32) & 0xff);
2544 cdb[4] = (u8)((start_blk >> 40) & 0xff);
2545 cdb[3] = (u8)((start_blk >> 48) & 0xff);
2546 cdb[2] = (u8)((start_blk >> 56) & 0xff);
2547 break;
2548 }
2549 }
2550}
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561static void megasas_stream_detect(struct megasas_instance *instance,
2562 struct megasas_cmd_fusion *cmd,
2563 struct IO_REQUEST_INFO *io_info)
2564{
2565 struct fusion_context *fusion = instance->ctrl_context;
2566 u32 device_id = io_info->ldTgtId;
2567 struct LD_STREAM_DETECT *current_ld_sd
2568 = fusion->stream_detect_by_ld[device_id];
2569 u32 *track_stream = ¤t_ld_sd->mru_bit_map, stream_num;
2570 u32 shifted_values, unshifted_values;
2571 u32 index_value_mask, shifted_values_mask;
2572 int i;
2573 bool is_read_ahead = false;
2574 struct STREAM_DETECT *current_sd;
2575
2576 for (i = 0; i < MAX_STREAMS_TRACKED; ++i) {
2577 stream_num = (*track_stream >>
2578 (i * BITS_PER_INDEX_STREAM)) &
2579 STREAM_MASK;
2580 current_sd = ¤t_ld_sd->stream_track[stream_num];
2581
2582
2583
2584
2585 if ((current_sd->next_seq_lba) &&
2586 (io_info->ldStartBlock >= current_sd->next_seq_lba) &&
2587 (io_info->ldStartBlock <= (current_sd->next_seq_lba + 32)) &&
2588 (current_sd->is_read == io_info->isRead)) {
2589
2590 if ((io_info->ldStartBlock != current_sd->next_seq_lba) &&
2591 ((!io_info->isRead) || (!is_read_ahead)))
2592
2593
2594
2595
2596 continue;
2597
2598 SET_STREAM_DETECTED(cmd->io_request->RaidContext.raid_context_g35);
2599 current_sd->next_seq_lba =
2600 io_info->ldStartBlock + io_info->numBlocks;
2601
2602
2603
2604 shifted_values_mask =
2605 (1 << i * BITS_PER_INDEX_STREAM) - 1;
2606 shifted_values = ((*track_stream & shifted_values_mask)
2607 << BITS_PER_INDEX_STREAM);
2608 index_value_mask =
2609 STREAM_MASK << i * BITS_PER_INDEX_STREAM;
2610 unshifted_values =
2611 *track_stream & ~(shifted_values_mask |
2612 index_value_mask);
2613 *track_stream =
2614 unshifted_values | shifted_values | stream_num;
2615 return;
2616 }
2617 }
2618
2619
2620
2621
2622 stream_num = (*track_stream >>
2623 ((MAX_STREAMS_TRACKED - 1) * BITS_PER_INDEX_STREAM)) &
2624 STREAM_MASK;
2625 current_sd = ¤t_ld_sd->stream_track[stream_num];
2626 current_sd->is_read = io_info->isRead;
2627 current_sd->next_seq_lba = io_info->ldStartBlock + io_info->numBlocks;
2628 *track_stream = (((*track_stream & ZERO_LAST_STREAM) << 4) | stream_num);
2629 return;
2630}
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645static void
2646megasas_set_raidflag_cpu_affinity(struct fusion_context *fusion,
2647 union RAID_CONTEXT_UNION *praid_context,
2648 struct MR_LD_RAID *raid, bool fp_possible,
2649 u8 is_read, u32 scsi_buff_len)
2650{
2651 u8 cpu_sel = MR_RAID_CTX_CPUSEL_0;
2652 struct RAID_CONTEXT_G35 *rctx_g35;
2653
2654 rctx_g35 = &praid_context->raid_context_g35;
2655 if (fp_possible) {
2656 if (is_read) {
2657 if ((raid->cpuAffinity.pdRead.cpu0) &&
2658 (raid->cpuAffinity.pdRead.cpu1))
2659 cpu_sel = MR_RAID_CTX_CPUSEL_FCFS;
2660 else if (raid->cpuAffinity.pdRead.cpu1)
2661 cpu_sel = MR_RAID_CTX_CPUSEL_1;
2662 } else {
2663 if ((raid->cpuAffinity.pdWrite.cpu0) &&
2664 (raid->cpuAffinity.pdWrite.cpu1))
2665 cpu_sel = MR_RAID_CTX_CPUSEL_FCFS;
2666 else if (raid->cpuAffinity.pdWrite.cpu1)
2667 cpu_sel = MR_RAID_CTX_CPUSEL_1;
2668
2669 if ((raid->level <= 1) &&
2670 (raid->capability.fp_cache_bypass_capable)) {
2671 rctx_g35->routing_flags |=
2672 (1 << MR_RAID_CTX_ROUTINGFLAGS_SLD_SHIFT);
2673 rctx_g35->raid_flags =
2674 (MR_RAID_FLAGS_IO_SUB_TYPE_CACHE_BYPASS
2675 << MR_RAID_CTX_RAID_FLAGS_IO_SUB_TYPE_SHIFT);
2676 }
2677 }
2678 } else {
2679 if (is_read) {
2680 if ((raid->cpuAffinity.ldRead.cpu0) &&
2681 (raid->cpuAffinity.ldRead.cpu1))
2682 cpu_sel = MR_RAID_CTX_CPUSEL_FCFS;
2683 else if (raid->cpuAffinity.ldRead.cpu1)
2684 cpu_sel = MR_RAID_CTX_CPUSEL_1;
2685 } else {
2686 if ((raid->cpuAffinity.ldWrite.cpu0) &&
2687 (raid->cpuAffinity.ldWrite.cpu1))
2688 cpu_sel = MR_RAID_CTX_CPUSEL_FCFS;
2689 else if (raid->cpuAffinity.ldWrite.cpu1)
2690 cpu_sel = MR_RAID_CTX_CPUSEL_1;
2691
2692 if (is_stream_detected(rctx_g35) &&
2693 ((raid->level == 5) || (raid->level == 6)) &&
2694 (raid->writeMode == MR_RL_WRITE_THROUGH_MODE) &&
2695 (cpu_sel == MR_RAID_CTX_CPUSEL_FCFS))
2696 cpu_sel = MR_RAID_CTX_CPUSEL_0;
2697 }
2698 }
2699
2700 rctx_g35->routing_flags |=
2701 (cpu_sel << MR_RAID_CTX_ROUTINGFLAGS_CPUSEL_SHIFT);
2702
2703
2704
2705
2706
2707 if ((fusion->pcie_bw_limitation) && (raid->level == 1) && (!is_read) &&
2708 (scsi_buff_len > MR_LARGE_IO_MIN_SIZE)) {
2709 praid_context->raid_context_g35.raid_flags =
2710 (MR_RAID_FLAGS_IO_SUB_TYPE_LDIO_BW_LIMIT
2711 << MR_RAID_CTX_RAID_FLAGS_IO_SUB_TYPE_SHIFT);
2712 }
2713}
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724static void
2725megasas_build_ldio_fusion(struct megasas_instance *instance,
2726 struct scsi_cmnd *scp,
2727 struct megasas_cmd_fusion *cmd)
2728{
2729 bool fp_possible;
2730 u16 ld;
2731 u32 start_lba_lo, start_lba_hi, device_id, datalength = 0;
2732 u32 scsi_buff_len;
2733 struct MPI2_RAID_SCSI_IO_REQUEST *io_request;
2734 struct IO_REQUEST_INFO io_info;
2735 struct fusion_context *fusion;
2736 struct MR_DRV_RAID_MAP_ALL *local_map_ptr;
2737 u8 *raidLUN;
2738 unsigned long spinlock_flags;
2739 struct MR_LD_RAID *raid = NULL;
2740 struct MR_PRIV_DEVICE *mrdev_priv;
2741 struct RAID_CONTEXT *rctx;
2742 struct RAID_CONTEXT_G35 *rctx_g35;
2743
2744 device_id = MEGASAS_DEV_INDEX(scp);
2745
2746 fusion = instance->ctrl_context;
2747
2748 io_request = cmd->io_request;
2749 rctx = &io_request->RaidContext.raid_context;
2750 rctx_g35 = &io_request->RaidContext.raid_context_g35;
2751
2752 rctx->virtual_disk_tgt_id = cpu_to_le16(device_id);
2753 rctx->status = 0;
2754 rctx->ex_status = 0;
2755
2756 start_lba_lo = 0;
2757 start_lba_hi = 0;
2758 fp_possible = false;
2759
2760
2761
2762
2763 if (scp->cmd_len == 6) {
2764 datalength = (u32) scp->cmnd[4];
2765 start_lba_lo = ((u32) scp->cmnd[1] << 16) |
2766 ((u32) scp->cmnd[2] << 8) | (u32) scp->cmnd[3];
2767
2768 start_lba_lo &= 0x1FFFFF;
2769 }
2770
2771
2772
2773
2774 else if (scp->cmd_len == 10) {
2775 datalength = (u32) scp->cmnd[8] |
2776 ((u32) scp->cmnd[7] << 8);
2777 start_lba_lo = ((u32) scp->cmnd[2] << 24) |
2778 ((u32) scp->cmnd[3] << 16) |
2779 ((u32) scp->cmnd[4] << 8) | (u32) scp->cmnd[5];
2780 }
2781
2782
2783
2784
2785 else if (scp->cmd_len == 12) {
2786 datalength = ((u32) scp->cmnd[6] << 24) |
2787 ((u32) scp->cmnd[7] << 16) |
2788 ((u32) scp->cmnd[8] << 8) | (u32) scp->cmnd[9];
2789 start_lba_lo = ((u32) scp->cmnd[2] << 24) |
2790 ((u32) scp->cmnd[3] << 16) |
2791 ((u32) scp->cmnd[4] << 8) | (u32) scp->cmnd[5];
2792 }
2793
2794
2795
2796
2797 else if (scp->cmd_len == 16) {
2798 datalength = ((u32) scp->cmnd[10] << 24) |
2799 ((u32) scp->cmnd[11] << 16) |
2800 ((u32) scp->cmnd[12] << 8) | (u32) scp->cmnd[13];
2801 start_lba_lo = ((u32) scp->cmnd[6] << 24) |
2802 ((u32) scp->cmnd[7] << 16) |
2803 ((u32) scp->cmnd[8] << 8) | (u32) scp->cmnd[9];
2804
2805 start_lba_hi = ((u32) scp->cmnd[2] << 24) |
2806 ((u32) scp->cmnd[3] << 16) |
2807 ((u32) scp->cmnd[4] << 8) | (u32) scp->cmnd[5];
2808 }
2809
2810 memset(&io_info, 0, sizeof(struct IO_REQUEST_INFO));
2811 io_info.ldStartBlock = ((u64)start_lba_hi << 32) | start_lba_lo;
2812 io_info.numBlocks = datalength;
2813 io_info.ldTgtId = device_id;
2814 io_info.r1_alt_dev_handle = MR_DEVHANDLE_INVALID;
2815 scsi_buff_len = scsi_bufflen(scp);
2816 io_request->DataLength = cpu_to_le32(scsi_buff_len);
2817 io_info.data_arms = 1;
2818
2819 if (scp->sc_data_direction == DMA_FROM_DEVICE)
2820 io_info.isRead = 1;
2821
2822 local_map_ptr = fusion->ld_drv_map[(instance->map_id & 1)];
2823 ld = MR_TargetIdToLdGet(device_id, local_map_ptr);
2824
2825 if (ld < instance->fw_supported_vd_count)
2826 raid = MR_LdRaidGet(ld, local_map_ptr);
2827
2828 if (!raid || (!fusion->fast_path_io)) {
2829 rctx->reg_lock_flags = 0;
2830 fp_possible = false;
2831 } else {
2832 if (MR_BuildRaidContext(instance, &io_info, rctx,
2833 local_map_ptr, &raidLUN))
2834 fp_possible = (io_info.fpOkForIo > 0) ? true : false;
2835 }
2836
2837 megasas_get_msix_index(instance, scp, cmd, io_info.data_arms);
2838
2839 if (instance->adapter_type >= VENTURA_SERIES) {
2840
2841
2842
2843
2844
2845
2846
2847
2848 if (io_info.r1_alt_dev_handle != MR_DEVHANDLE_INVALID) {
2849 mrdev_priv = scp->device->hostdata;
2850
2851 if (atomic_inc_return(&instance->fw_outstanding) >
2852 (instance->host->can_queue)) {
2853 fp_possible = false;
2854 atomic_dec(&instance->fw_outstanding);
2855 } else if (fusion->pcie_bw_limitation &&
2856 ((scsi_buff_len > MR_LARGE_IO_MIN_SIZE) ||
2857 (atomic_dec_if_positive(&mrdev_priv->r1_ldio_hint) > 0))) {
2858 fp_possible = false;
2859 atomic_dec(&instance->fw_outstanding);
2860 if (scsi_buff_len > MR_LARGE_IO_MIN_SIZE)
2861 atomic_set(&mrdev_priv->r1_ldio_hint,
2862 instance->r1_ldio_hint_default);
2863 }
2864 }
2865
2866 if (!fp_possible ||
2867 (io_info.isRead && io_info.ra_capable)) {
2868 spin_lock_irqsave(&instance->stream_lock,
2869 spinlock_flags);
2870 megasas_stream_detect(instance, cmd, &io_info);
2871 spin_unlock_irqrestore(&instance->stream_lock,
2872 spinlock_flags);
2873
2874
2875
2876 if (is_stream_detected(rctx_g35))
2877 fp_possible = false;
2878 }
2879
2880
2881 if (raid)
2882 megasas_set_raidflag_cpu_affinity(fusion, &io_request->RaidContext,
2883 raid, fp_possible, io_info.isRead,
2884 scsi_buff_len);
2885 else
2886 rctx_g35->routing_flags |=
2887 (MR_RAID_CTX_CPUSEL_0 << MR_RAID_CTX_ROUTINGFLAGS_CPUSEL_SHIFT);
2888 }
2889
2890 if (fp_possible) {
2891 megasas_set_pd_lba(io_request, scp->cmd_len, &io_info, scp,
2892 local_map_ptr, start_lba_lo);
2893 io_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
2894 cmd->request_desc->SCSIIO.RequestFlags =
2895 (MPI2_REQ_DESCRIPT_FLAGS_FP_IO
2896 << MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
2897 if (instance->adapter_type == INVADER_SERIES) {
2898 rctx->type = MPI2_TYPE_CUDA;
2899 rctx->nseg = 0x1;
2900 io_request->IoFlags |= cpu_to_le16(MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH);
2901 rctx->reg_lock_flags |=
2902 (MR_RL_FLAGS_GRANT_DESTINATION_CUDA |
2903 MR_RL_FLAGS_SEQ_NUM_ENABLE);
2904 } else if (instance->adapter_type >= VENTURA_SERIES) {
2905 rctx_g35->nseg_type |= (1 << RAID_CONTEXT_NSEG_SHIFT);
2906 rctx_g35->nseg_type |= (MPI2_TYPE_CUDA << RAID_CONTEXT_TYPE_SHIFT);
2907 rctx_g35->routing_flags |= (1 << MR_RAID_CTX_ROUTINGFLAGS_SQN_SHIFT);
2908 io_request->IoFlags |=
2909 cpu_to_le16(MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH);
2910 }
2911 if (fusion->load_balance_info &&
2912 (fusion->load_balance_info[device_id].loadBalanceFlag) &&
2913 (io_info.isRead)) {
2914 io_info.devHandle =
2915 get_updated_dev_handle(instance,
2916 &fusion->load_balance_info[device_id],
2917 &io_info, local_map_ptr);
2918 scp->SCp.Status |= MEGASAS_LOAD_BALANCE_FLAG;
2919 cmd->pd_r1_lb = io_info.pd_after_lb;
2920 if (instance->adapter_type >= VENTURA_SERIES)
2921 rctx_g35->span_arm = io_info.span_arm;
2922 else
2923 rctx->span_arm = io_info.span_arm;
2924
2925 } else
2926 scp->SCp.Status &= ~MEGASAS_LOAD_BALANCE_FLAG;
2927
2928 if (instance->adapter_type >= VENTURA_SERIES)
2929 cmd->r1_alt_dev_handle = io_info.r1_alt_dev_handle;
2930 else
2931 cmd->r1_alt_dev_handle = MR_DEVHANDLE_INVALID;
2932
2933 if ((raidLUN[0] == 1) &&
2934 (local_map_ptr->raidMap.devHndlInfo[io_info.pd_after_lb].validHandles > 1)) {
2935 instance->dev_handle = !(instance->dev_handle);
2936 io_info.devHandle =
2937 local_map_ptr->raidMap.devHndlInfo[io_info.pd_after_lb].devHandle[instance->dev_handle];
2938 }
2939
2940 cmd->request_desc->SCSIIO.DevHandle = io_info.devHandle;
2941 io_request->DevHandle = io_info.devHandle;
2942 cmd->pd_interface = io_info.pd_interface;
2943
2944 memcpy(io_request->LUN, raidLUN, 8);
2945 } else {
2946 rctx->timeout_value =
2947 cpu_to_le16(local_map_ptr->raidMap.fpPdIoTimeoutSec);
2948 cmd->request_desc->SCSIIO.RequestFlags =
2949 (MEGASAS_REQ_DESCRIPT_FLAGS_LD_IO
2950 << MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
2951 if (instance->adapter_type == INVADER_SERIES) {
2952 if (io_info.do_fp_rlbypass ||
2953 (rctx->reg_lock_flags == REGION_TYPE_UNUSED))
2954 cmd->request_desc->SCSIIO.RequestFlags =
2955 (MEGASAS_REQ_DESCRIPT_FLAGS_NO_LOCK <<
2956 MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
2957 rctx->type = MPI2_TYPE_CUDA;
2958 rctx->reg_lock_flags |=
2959 (MR_RL_FLAGS_GRANT_DESTINATION_CPU0 |
2960 MR_RL_FLAGS_SEQ_NUM_ENABLE);
2961 rctx->nseg = 0x1;
2962 } else if (instance->adapter_type >= VENTURA_SERIES) {
2963 rctx_g35->routing_flags |= (1 << MR_RAID_CTX_ROUTINGFLAGS_SQN_SHIFT);
2964 rctx_g35->nseg_type |= (1 << RAID_CONTEXT_NSEG_SHIFT);
2965 rctx_g35->nseg_type |= (MPI2_TYPE_CUDA << RAID_CONTEXT_TYPE_SHIFT);
2966 }
2967 io_request->Function = MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST;
2968 io_request->DevHandle = cpu_to_le16(device_id);
2969
2970 }
2971}
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981static void megasas_build_ld_nonrw_fusion(struct megasas_instance *instance,
2982 struct scsi_cmnd *scmd, struct megasas_cmd_fusion *cmd)
2983{
2984 u32 device_id;
2985 struct MPI2_RAID_SCSI_IO_REQUEST *io_request;
2986 u16 ld;
2987 struct MR_DRV_RAID_MAP_ALL *local_map_ptr;
2988 struct fusion_context *fusion = instance->ctrl_context;
2989 u8 span, physArm;
2990 __le16 devHandle;
2991 u32 arRef, pd;
2992 struct MR_LD_RAID *raid;
2993 struct RAID_CONTEXT *pRAID_Context;
2994 u8 fp_possible = 1;
2995
2996 io_request = cmd->io_request;
2997 device_id = MEGASAS_DEV_INDEX(scmd);
2998 local_map_ptr = fusion->ld_drv_map[(instance->map_id & 1)];
2999 io_request->DataLength = cpu_to_le32(scsi_bufflen(scmd));
3000
3001 pRAID_Context = &io_request->RaidContext.raid_context;
3002
3003 pRAID_Context->virtual_disk_tgt_id = cpu_to_le16(device_id);
3004 pRAID_Context->reg_lock_row_lba = 0;
3005 pRAID_Context->reg_lock_length = 0;
3006
3007 if (fusion->fast_path_io && (
3008 device_id < instance->fw_supported_vd_count)) {
3009
3010 ld = MR_TargetIdToLdGet(device_id, local_map_ptr);
3011 if (ld >= instance->fw_supported_vd_count - 1)
3012 fp_possible = 0;
3013 else {
3014 raid = MR_LdRaidGet(ld, local_map_ptr);
3015 if (!(raid->capability.fpNonRWCapable))
3016 fp_possible = 0;
3017 }
3018 } else
3019 fp_possible = 0;
3020
3021 if (!fp_possible) {
3022 io_request->Function = MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST;
3023 io_request->DevHandle = cpu_to_le16(device_id);
3024 io_request->LUN[1] = scmd->device->lun;
3025 pRAID_Context->timeout_value =
3026 cpu_to_le16 (scmd->request->timeout / HZ);
3027 cmd->request_desc->SCSIIO.RequestFlags =
3028 (MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO <<
3029 MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
3030 } else {
3031
3032
3033 pRAID_Context->config_seq_num = raid->seqNum;
3034 if (instance->adapter_type < VENTURA_SERIES)
3035 pRAID_Context->reg_lock_flags = REGION_TYPE_SHARED_READ;
3036 pRAID_Context->timeout_value =
3037 cpu_to_le16(raid->fpIoTimeoutForLd);
3038
3039
3040
3041 span = physArm = 0;
3042 arRef = MR_LdSpanArrayGet(ld, span, local_map_ptr);
3043 pd = MR_ArPdGet(arRef, physArm, local_map_ptr);
3044 devHandle = MR_PdDevHandleGet(pd, local_map_ptr);
3045
3046
3047 cmd->request_desc->SCSIIO.RequestFlags =
3048 (MPI2_REQ_DESCRIPT_FLAGS_FP_IO <<
3049 MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
3050 cmd->request_desc->SCSIIO.DevHandle = devHandle;
3051
3052
3053 memcpy(io_request->LUN, raid->LUN, 8);
3054
3055
3056 io_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
3057 io_request->DevHandle = devHandle;
3058 }
3059}
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070static void
3071megasas_build_syspd_fusion(struct megasas_instance *instance,
3072 struct scsi_cmnd *scmd, struct megasas_cmd_fusion *cmd,
3073 bool fp_possible)
3074{
3075 u32 device_id;
3076 struct MPI2_RAID_SCSI_IO_REQUEST *io_request;
3077 u16 pd_index = 0;
3078 u16 os_timeout_value;
3079 u16 timeout_limit;
3080 struct MR_DRV_RAID_MAP_ALL *local_map_ptr;
3081 struct RAID_CONTEXT *pRAID_Context;
3082 struct MR_PD_CFG_SEQ_NUM_SYNC *pd_sync;
3083 struct MR_PRIV_DEVICE *mr_device_priv_data;
3084 struct fusion_context *fusion = instance->ctrl_context;
3085 pd_sync = (void *)fusion->pd_seq_sync[(instance->pd_seq_map_id - 1) & 1];
3086
3087 device_id = MEGASAS_DEV_INDEX(scmd);
3088 pd_index = MEGASAS_PD_INDEX(scmd);
3089 os_timeout_value = scmd->request->timeout / HZ;
3090 mr_device_priv_data = scmd->device->hostdata;
3091 cmd->pd_interface = mr_device_priv_data->interface_type;
3092
3093 io_request = cmd->io_request;
3094
3095 pRAID_Context = &io_request->RaidContext.raid_context;
3096 pRAID_Context->reg_lock_flags = 0;
3097 pRAID_Context->reg_lock_row_lba = 0;
3098 pRAID_Context->reg_lock_length = 0;
3099 io_request->DataLength = cpu_to_le32(scsi_bufflen(scmd));
3100 io_request->LUN[1] = scmd->device->lun;
3101 pRAID_Context->raid_flags = MR_RAID_FLAGS_IO_SUB_TYPE_SYSTEM_PD
3102 << MR_RAID_CTX_RAID_FLAGS_IO_SUB_TYPE_SHIFT;
3103
3104
3105 if (instance->support_seqnum_jbod_fp) {
3106 if (instance->use_seqnum_jbod_fp &&
3107 instance->pd_list[pd_index].driveType == TYPE_DISK) {
3108
3109
3110 if (instance->support_morethan256jbod)
3111 pRAID_Context->virtual_disk_tgt_id =
3112 pd_sync->seq[pd_index].pd_target_id;
3113 else
3114 pRAID_Context->virtual_disk_tgt_id =
3115 cpu_to_le16(device_id +
3116 (MAX_PHYSICAL_DEVICES - 1));
3117 pRAID_Context->config_seq_num =
3118 pd_sync->seq[pd_index].seqNum;
3119 io_request->DevHandle =
3120 pd_sync->seq[pd_index].devHandle;
3121 if (instance->adapter_type >= VENTURA_SERIES) {
3122 io_request->RaidContext.raid_context_g35.routing_flags |=
3123 (1 << MR_RAID_CTX_ROUTINGFLAGS_SQN_SHIFT);
3124 io_request->RaidContext.raid_context_g35.nseg_type |=
3125 (1 << RAID_CONTEXT_NSEG_SHIFT);
3126 io_request->RaidContext.raid_context_g35.nseg_type |=
3127 (MPI2_TYPE_CUDA << RAID_CONTEXT_TYPE_SHIFT);
3128 } else {
3129 pRAID_Context->type = MPI2_TYPE_CUDA;
3130 pRAID_Context->nseg = 0x1;
3131 pRAID_Context->reg_lock_flags |=
3132 (MR_RL_FLAGS_SEQ_NUM_ENABLE |
3133 MR_RL_FLAGS_GRANT_DESTINATION_CUDA);
3134 }
3135 } else {
3136 pRAID_Context->virtual_disk_tgt_id =
3137 cpu_to_le16(device_id +
3138 (MAX_PHYSICAL_DEVICES - 1));
3139 pRAID_Context->config_seq_num = 0;
3140 io_request->DevHandle = cpu_to_le16(0xFFFF);
3141 }
3142 } else {
3143 pRAID_Context->virtual_disk_tgt_id = cpu_to_le16(device_id);
3144 pRAID_Context->config_seq_num = 0;
3145
3146 if (fusion->fast_path_io) {
3147 local_map_ptr =
3148 fusion->ld_drv_map[(instance->map_id & 1)];
3149 io_request->DevHandle =
3150 local_map_ptr->raidMap.devHndlInfo[device_id].curDevHdl;
3151 } else {
3152 io_request->DevHandle = cpu_to_le16(0xFFFF);
3153 }
3154 }
3155
3156 cmd->request_desc->SCSIIO.DevHandle = io_request->DevHandle;
3157
3158 megasas_get_msix_index(instance, scmd, cmd, 1);
3159
3160 if (!fp_possible) {
3161
3162 io_request->Function = MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST;
3163 cmd->request_desc->SCSIIO.RequestFlags =
3164 (MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO <<
3165 MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
3166 pRAID_Context->timeout_value = cpu_to_le16(os_timeout_value);
3167 pRAID_Context->virtual_disk_tgt_id = cpu_to_le16(device_id);
3168 } else {
3169 if (os_timeout_value)
3170 os_timeout_value++;
3171
3172
3173 io_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
3174 timeout_limit = (scmd->device->type == TYPE_DISK) ?
3175 255 : 0xFFFF;
3176 pRAID_Context->timeout_value =
3177 cpu_to_le16((os_timeout_value > timeout_limit) ?
3178 timeout_limit : os_timeout_value);
3179 if (instance->adapter_type >= INVADER_SERIES)
3180 io_request->IoFlags |=
3181 cpu_to_le16(MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH);
3182
3183 cmd->request_desc->SCSIIO.RequestFlags =
3184 (MPI2_REQ_DESCRIPT_FLAGS_FP_IO <<
3185 MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
3186 }
3187}
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198static int
3199megasas_build_io_fusion(struct megasas_instance *instance,
3200 struct scsi_cmnd *scp,
3201 struct megasas_cmd_fusion *cmd)
3202{
3203 int sge_count;
3204 u8 cmd_type;
3205 u16 pd_index = 0;
3206 u8 drive_type = 0;
3207 struct MPI2_RAID_SCSI_IO_REQUEST *io_request = cmd->io_request;
3208 struct MR_PRIV_DEVICE *mr_device_priv_data;
3209 mr_device_priv_data = scp->device->hostdata;
3210
3211
3212 memset(io_request->LUN, 0x0, 8);
3213 io_request->CDB.EEDP32.PrimaryReferenceTag = 0;
3214 io_request->CDB.EEDP32.PrimaryApplicationTagMask = 0;
3215 io_request->EEDPFlags = 0;
3216 io_request->Control = 0;
3217 io_request->EEDPBlockSize = 0;
3218 io_request->ChainOffset = 0;
3219 io_request->RaidContext.raid_context.raid_flags = 0;
3220 io_request->RaidContext.raid_context.type = 0;
3221 io_request->RaidContext.raid_context.nseg = 0;
3222
3223 memcpy(io_request->CDB.CDB32, scp->cmnd, scp->cmd_len);
3224
3225
3226
3227
3228 io_request->IoFlags = cpu_to_le16(scp->cmd_len);
3229
3230 switch (cmd_type = megasas_cmd_type(scp)) {
3231 case READ_WRITE_LDIO:
3232 megasas_build_ldio_fusion(instance, scp, cmd);
3233 break;
3234 case NON_READ_WRITE_LDIO:
3235 megasas_build_ld_nonrw_fusion(instance, scp, cmd);
3236 break;
3237 case READ_WRITE_SYSPDIO:
3238 megasas_build_syspd_fusion(instance, scp, cmd, true);
3239 break;
3240 case NON_READ_WRITE_SYSPDIO:
3241 pd_index = MEGASAS_PD_INDEX(scp);
3242 drive_type = instance->pd_list[pd_index].driveType;
3243 if ((instance->secure_jbod_support ||
3244 mr_device_priv_data->is_tm_capable) ||
3245 (instance->adapter_type >= VENTURA_SERIES &&
3246 drive_type == TYPE_ENCLOSURE))
3247 megasas_build_syspd_fusion(instance, scp, cmd, false);
3248 else
3249 megasas_build_syspd_fusion(instance, scp, cmd, true);
3250 break;
3251 default:
3252 break;
3253 }
3254
3255
3256
3257
3258
3259 sge_count = megasas_make_sgl(instance, scp, cmd);
3260
3261 if (sge_count > instance->max_num_sge || (sge_count < 0)) {
3262 dev_err(&instance->pdev->dev,
3263 "%s %d sge_count (%d) is out of range. Range is: 0-%d\n",
3264 __func__, __LINE__, sge_count, instance->max_num_sge);
3265 return 1;
3266 }
3267
3268 if (instance->adapter_type >= VENTURA_SERIES) {
3269 set_num_sge(&io_request->RaidContext.raid_context_g35, sge_count);
3270 cpu_to_le16s(&io_request->RaidContext.raid_context_g35.routing_flags);
3271 cpu_to_le16s(&io_request->RaidContext.raid_context_g35.nseg_type);
3272 } else {
3273
3274
3275
3276 io_request->RaidContext.raid_context.num_sge = sge_count;
3277 io_request->RaidContext.raid_context.num_sge_ext =
3278 (u8)(sge_count >> 8);
3279 }
3280
3281 io_request->SGLFlags = cpu_to_le16(MPI2_SGE_FLAGS_64_BIT_ADDRESSING);
3282
3283 if (scp->sc_data_direction == DMA_TO_DEVICE)
3284 io_request->Control |= cpu_to_le32(MPI2_SCSIIO_CONTROL_WRITE);
3285 else if (scp->sc_data_direction == DMA_FROM_DEVICE)
3286 io_request->Control |= cpu_to_le32(MPI2_SCSIIO_CONTROL_READ);
3287
3288 io_request->SGLOffset0 =
3289 offsetof(struct MPI2_RAID_SCSI_IO_REQUEST, SGL) / 4;
3290
3291 io_request->SenseBufferLowAddress =
3292 cpu_to_le32(lower_32_bits(cmd->sense_phys_addr));
3293 io_request->SenseBufferLength = SCSI_SENSE_BUFFERSIZE;
3294
3295 cmd->scmd = scp;
3296 scp->SCp.ptr = (char *)cmd;
3297
3298 return 0;
3299}
3300
3301static union MEGASAS_REQUEST_DESCRIPTOR_UNION *
3302megasas_get_request_descriptor(struct megasas_instance *instance, u16 index)
3303{
3304 u8 *p;
3305 struct fusion_context *fusion;
3306
3307 fusion = instance->ctrl_context;
3308 p = fusion->req_frames_desc +
3309 sizeof(union MEGASAS_REQUEST_DESCRIPTOR_UNION) * index;
3310
3311 return (union MEGASAS_REQUEST_DESCRIPTOR_UNION *)p;
3312}
3313
3314
3315
3316
3317
3318static void megasas_prepare_secondRaid1_IO(struct megasas_instance *instance,
3319 struct megasas_cmd_fusion *cmd,
3320 struct megasas_cmd_fusion *r1_cmd)
3321{
3322 union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc, *req_desc2 = NULL;
3323 struct fusion_context *fusion;
3324 fusion = instance->ctrl_context;
3325 req_desc = cmd->request_desc;
3326
3327 memcpy(r1_cmd->io_request, cmd->io_request,
3328 (sizeof(struct MPI2_RAID_SCSI_IO_REQUEST)));
3329 memcpy(&r1_cmd->io_request->SGL, &cmd->io_request->SGL,
3330 (fusion->max_sge_in_main_msg * sizeof(union MPI2_SGE_IO_UNION)));
3331
3332 r1_cmd->io_request->SenseBufferLowAddress =
3333 cpu_to_le32(lower_32_bits(r1_cmd->sense_phys_addr));
3334 r1_cmd->scmd = cmd->scmd;
3335 req_desc2 = megasas_get_request_descriptor(instance,
3336 (r1_cmd->index - 1));
3337 req_desc2->Words = 0;
3338 r1_cmd->request_desc = req_desc2;
3339 req_desc2->SCSIIO.SMID = cpu_to_le16(r1_cmd->index);
3340 req_desc2->SCSIIO.RequestFlags = req_desc->SCSIIO.RequestFlags;
3341 r1_cmd->request_desc->SCSIIO.DevHandle = cmd->r1_alt_dev_handle;
3342 r1_cmd->io_request->DevHandle = cmd->r1_alt_dev_handle;
3343 r1_cmd->r1_alt_dev_handle = cmd->io_request->DevHandle;
3344 cmd->io_request->RaidContext.raid_context_g35.flow_specific.peer_smid =
3345 cpu_to_le16(r1_cmd->index);
3346 r1_cmd->io_request->RaidContext.raid_context_g35.flow_specific.peer_smid =
3347 cpu_to_le16(cmd->index);
3348
3349 r1_cmd->request_desc->SCSIIO.MSIxIndex =
3350 cmd->request_desc->SCSIIO.MSIxIndex;
3351
3352 r1_cmd->io_request->RaidContext.raid_context_g35.span_arm =
3353 cmd->io_request->RaidContext.raid_context_g35.span_arm + 1;
3354}
3355
3356
3357
3358
3359
3360
3361
3362static u32
3363megasas_build_and_issue_cmd_fusion(struct megasas_instance *instance,
3364 struct scsi_cmnd *scmd)
3365{
3366 struct megasas_cmd_fusion *cmd, *r1_cmd = NULL;
3367 union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc;
3368 u32 index;
3369
3370 if ((megasas_cmd_type(scmd) == READ_WRITE_LDIO) &&
3371 instance->ldio_threshold &&
3372 (atomic_inc_return(&instance->ldio_outstanding) >
3373 instance->ldio_threshold)) {
3374 atomic_dec(&instance->ldio_outstanding);
3375 return SCSI_MLQUEUE_DEVICE_BUSY;
3376 }
3377
3378 if (atomic_inc_return(&instance->fw_outstanding) >
3379 instance->host->can_queue) {
3380 atomic_dec(&instance->fw_outstanding);
3381 return SCSI_MLQUEUE_HOST_BUSY;
3382 }
3383
3384 cmd = megasas_get_cmd_fusion(instance, scmd->request->tag);
3385
3386 if (!cmd) {
3387 atomic_dec(&instance->fw_outstanding);
3388 return SCSI_MLQUEUE_HOST_BUSY;
3389 }
3390
3391 index = cmd->index;
3392
3393 req_desc = megasas_get_request_descriptor(instance, index-1);
3394
3395 req_desc->Words = 0;
3396 cmd->request_desc = req_desc;
3397
3398 if (megasas_build_io_fusion(instance, scmd, cmd)) {
3399 megasas_return_cmd_fusion(instance, cmd);
3400 dev_err(&instance->pdev->dev, "Error building command\n");
3401 cmd->request_desc = NULL;
3402 atomic_dec(&instance->fw_outstanding);
3403 return SCSI_MLQUEUE_HOST_BUSY;
3404 }
3405
3406 req_desc = cmd->request_desc;
3407 req_desc->SCSIIO.SMID = cpu_to_le16(index);
3408
3409 if (cmd->io_request->ChainOffset != 0 &&
3410 cmd->io_request->ChainOffset != 0xF)
3411 dev_err(&instance->pdev->dev, "The chain offset value is not "
3412 "correct : %x\n", cmd->io_request->ChainOffset);
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423 if (cmd->r1_alt_dev_handle != MR_DEVHANDLE_INVALID) {
3424 r1_cmd = megasas_get_cmd_fusion(instance,
3425 (scmd->request->tag + instance->max_fw_cmds));
3426 megasas_prepare_secondRaid1_IO(instance, cmd, r1_cmd);
3427 }
3428
3429
3430
3431
3432
3433
3434 megasas_sdev_busy_inc(instance, scmd);
3435 megasas_fire_cmd_fusion(instance, req_desc);
3436
3437 if (r1_cmd)
3438 megasas_fire_cmd_fusion(instance, r1_cmd->request_desc);
3439
3440
3441 return 0;
3442}
3443
3444
3445
3446
3447
3448
3449
3450
3451static inline void
3452megasas_complete_r1_command(struct megasas_instance *instance,
3453 struct megasas_cmd_fusion *cmd)
3454{
3455 u8 *sense, status, ex_status;
3456 u32 data_length;
3457 u16 peer_smid;
3458 struct fusion_context *fusion;
3459 struct megasas_cmd_fusion *r1_cmd = NULL;
3460 struct scsi_cmnd *scmd_local = NULL;
3461 struct RAID_CONTEXT_G35 *rctx_g35;
3462
3463 rctx_g35 = &cmd->io_request->RaidContext.raid_context_g35;
3464 fusion = instance->ctrl_context;
3465 peer_smid = le16_to_cpu(rctx_g35->flow_specific.peer_smid);
3466
3467 r1_cmd = fusion->cmd_list[peer_smid - 1];
3468 scmd_local = cmd->scmd;
3469 status = rctx_g35->status;
3470 ex_status = rctx_g35->ex_status;
3471 data_length = cmd->io_request->DataLength;
3472 sense = cmd->sense;
3473
3474 cmd->cmd_completed = true;
3475
3476
3477 if (r1_cmd->cmd_completed) {
3478 rctx_g35 = &r1_cmd->io_request->RaidContext.raid_context_g35;
3479 if (rctx_g35->status != MFI_STAT_OK) {
3480 status = rctx_g35->status;
3481 ex_status = rctx_g35->ex_status;
3482 data_length = r1_cmd->io_request->DataLength;
3483 sense = r1_cmd->sense;
3484 }
3485
3486 megasas_return_cmd_fusion(instance, r1_cmd);
3487 map_cmd_status(fusion, scmd_local, status, ex_status,
3488 le32_to_cpu(data_length), sense);
3489 if (instance->ldio_threshold &&
3490 megasas_cmd_type(scmd_local) == READ_WRITE_LDIO)
3491 atomic_dec(&instance->ldio_outstanding);
3492 scmd_local->SCp.ptr = NULL;
3493 megasas_return_cmd_fusion(instance, cmd);
3494 scsi_dma_unmap(scmd_local);
3495 megasas_sdev_busy_dec(instance, scmd_local);
3496 scmd_local->scsi_done(scmd_local);
3497 }
3498}
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508static int
3509complete_cmd_fusion(struct megasas_instance *instance, u32 MSIxIndex,
3510 struct megasas_irq_context *irq_context)
3511{
3512 union MPI2_REPLY_DESCRIPTORS_UNION *desc;
3513 struct MPI2_SCSI_IO_SUCCESS_REPLY_DESCRIPTOR *reply_desc;
3514 struct MPI2_RAID_SCSI_IO_REQUEST *scsi_io_req;
3515 struct fusion_context *fusion;
3516 struct megasas_cmd *cmd_mfi;
3517 struct megasas_cmd_fusion *cmd_fusion;
3518 u16 smid, num_completed;
3519 u8 reply_descript_type, *sense, status, extStatus;
3520 u32 device_id, data_length;
3521 union desc_value d_val;
3522 struct LD_LOAD_BALANCE_INFO *lbinfo;
3523 int threshold_reply_count = 0;
3524 struct scsi_cmnd *scmd_local = NULL;
3525 struct MR_TASK_MANAGE_REQUEST *mr_tm_req;
3526 struct MPI2_SCSI_TASK_MANAGE_REQUEST *mpi_tm_req;
3527
3528 fusion = instance->ctrl_context;
3529
3530 if (atomic_read(&instance->adprecovery) == MEGASAS_HW_CRITICAL_ERROR)
3531 return IRQ_HANDLED;
3532
3533 desc = fusion->reply_frames_desc[MSIxIndex] +
3534 fusion->last_reply_idx[MSIxIndex];
3535
3536 reply_desc = (struct MPI2_SCSI_IO_SUCCESS_REPLY_DESCRIPTOR *)desc;
3537
3538 d_val.word = desc->Words;
3539
3540 reply_descript_type = reply_desc->ReplyFlags &
3541 MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
3542
3543 if (reply_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED)
3544 return IRQ_NONE;
3545
3546 if (irq_context && !atomic_add_unless(&irq_context->in_used, 1, 1))
3547 return 0;
3548
3549 num_completed = 0;
3550
3551 while (d_val.u.low != cpu_to_le32(UINT_MAX) &&
3552 d_val.u.high != cpu_to_le32(UINT_MAX)) {
3553
3554 smid = le16_to_cpu(reply_desc->SMID);
3555 cmd_fusion = fusion->cmd_list[smid - 1];
3556 scsi_io_req = (struct MPI2_RAID_SCSI_IO_REQUEST *)
3557 cmd_fusion->io_request;
3558
3559 scmd_local = cmd_fusion->scmd;
3560 status = scsi_io_req->RaidContext.raid_context.status;
3561 extStatus = scsi_io_req->RaidContext.raid_context.ex_status;
3562 sense = cmd_fusion->sense;
3563 data_length = scsi_io_req->DataLength;
3564
3565 switch (scsi_io_req->Function) {
3566 case MPI2_FUNCTION_SCSI_TASK_MGMT:
3567 mr_tm_req = (struct MR_TASK_MANAGE_REQUEST *)
3568 cmd_fusion->io_request;
3569 mpi_tm_req = (struct MPI2_SCSI_TASK_MANAGE_REQUEST *)
3570 &mr_tm_req->TmRequest;
3571 dev_dbg(&instance->pdev->dev, "TM completion:"
3572 "type: 0x%x TaskMID: 0x%x\n",
3573 mpi_tm_req->TaskType, mpi_tm_req->TaskMID);
3574 complete(&cmd_fusion->done);
3575 break;
3576 case MPI2_FUNCTION_SCSI_IO_REQUEST:
3577
3578 if (fusion->load_balance_info &&
3579 (cmd_fusion->scmd->SCp.Status &
3580 MEGASAS_LOAD_BALANCE_FLAG)) {
3581 device_id = MEGASAS_DEV_INDEX(scmd_local);
3582 lbinfo = &fusion->load_balance_info[device_id];
3583 atomic_dec(&lbinfo->scsi_pending_cmds[cmd_fusion->pd_r1_lb]);
3584 cmd_fusion->scmd->SCp.Status &= ~MEGASAS_LOAD_BALANCE_FLAG;
3585 }
3586 fallthrough;
3587 case MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST:
3588 atomic_dec(&instance->fw_outstanding);
3589 if (cmd_fusion->r1_alt_dev_handle == MR_DEVHANDLE_INVALID) {
3590 map_cmd_status(fusion, scmd_local, status,
3591 extStatus, le32_to_cpu(data_length),
3592 sense);
3593 if (instance->ldio_threshold &&
3594 (megasas_cmd_type(scmd_local) == READ_WRITE_LDIO))
3595 atomic_dec(&instance->ldio_outstanding);
3596 scmd_local->SCp.ptr = NULL;
3597 megasas_return_cmd_fusion(instance, cmd_fusion);
3598 scsi_dma_unmap(scmd_local);
3599 megasas_sdev_busy_dec(instance, scmd_local);
3600 scmd_local->scsi_done(scmd_local);
3601 } else
3602 megasas_complete_r1_command(instance, cmd_fusion);
3603 break;
3604 case MEGASAS_MPI2_FUNCTION_PASSTHRU_IO_REQUEST:
3605 cmd_mfi = instance->cmd_list[cmd_fusion->sync_cmd_idx];
3606
3607
3608
3609 if (cmd_mfi->flags & DRV_DCMD_POLLED_MODE) {
3610 cmd_mfi->flags &= ~DRV_DCMD_POLLED_MODE;
3611 megasas_return_cmd(instance, cmd_mfi);
3612 } else
3613 megasas_complete_cmd(instance, cmd_mfi, DID_OK);
3614 break;
3615 }
3616
3617 fusion->last_reply_idx[MSIxIndex]++;
3618 if (fusion->last_reply_idx[MSIxIndex] >=
3619 fusion->reply_q_depth)
3620 fusion->last_reply_idx[MSIxIndex] = 0;
3621
3622 desc->Words = cpu_to_le64(ULLONG_MAX);
3623 num_completed++;
3624 threshold_reply_count++;
3625
3626
3627 if (!fusion->last_reply_idx[MSIxIndex])
3628 desc = fusion->reply_frames_desc[MSIxIndex];
3629 else
3630 desc++;
3631
3632 reply_desc =
3633 (struct MPI2_SCSI_IO_SUCCESS_REPLY_DESCRIPTOR *)desc;
3634
3635 d_val.word = desc->Words;
3636
3637 reply_descript_type = reply_desc->ReplyFlags &
3638 MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
3639
3640 if (reply_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED)
3641 break;
3642
3643
3644
3645
3646
3647 if (threshold_reply_count >= instance->threshold_reply_count) {
3648 if (instance->msix_combined)
3649 writel(((MSIxIndex & 0x7) << 24) |
3650 fusion->last_reply_idx[MSIxIndex],
3651 instance->reply_post_host_index_addr[MSIxIndex/8]);
3652 else
3653 writel((MSIxIndex << 24) |
3654 fusion->last_reply_idx[MSIxIndex],
3655 instance->reply_post_host_index_addr[0]);
3656 threshold_reply_count = 0;
3657 if (irq_context) {
3658 if (!irq_context->irq_poll_scheduled) {
3659 irq_context->irq_poll_scheduled = true;
3660