1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39#ifndef MTHCA_DEV_H
40#define MTHCA_DEV_H
41
42#include <linux/spinlock.h>
43#include <linux/kernel.h>
44#include <linux/pci.h>
45#include <linux/dma-mapping.h>
46#include <linux/timer.h>
47#include <linux/mutex.h>
48#include <linux/list.h>
49
50#include <asm/semaphore.h>
51
52#include "mthca_provider.h"
53#include "mthca_doorbell.h"
54
55#define DRV_NAME "ib_mthca"
56#define PFX DRV_NAME ": "
57#define DRV_VERSION "0.08"
58#define DRV_RELDATE "February 14, 2006"
59
60enum {
61 MTHCA_FLAG_DDR_HIDDEN = 1 << 1,
62 MTHCA_FLAG_SRQ = 1 << 2,
63 MTHCA_FLAG_MSI_X = 1 << 3,
64 MTHCA_FLAG_NO_LAM = 1 << 4,
65 MTHCA_FLAG_FMR = 1 << 5,
66 MTHCA_FLAG_MEMFREE = 1 << 6,
67 MTHCA_FLAG_PCIE = 1 << 7,
68 MTHCA_FLAG_SINAI_OPT = 1 << 8
69};
70
71enum {
72 MTHCA_MAX_PORTS = 2
73};
74
75enum {
76 MTHCA_BOARD_ID_LEN = 64
77};
78
79enum {
80 MTHCA_EQ_CONTEXT_SIZE = 0x40,
81 MTHCA_CQ_CONTEXT_SIZE = 0x40,
82 MTHCA_QP_CONTEXT_SIZE = 0x200,
83 MTHCA_RDB_ENTRY_SIZE = 0x20,
84 MTHCA_AV_SIZE = 0x20,
85 MTHCA_MGM_ENTRY_SIZE = 0x100,
86
87
88 MTHCA_MPT_ENTRY_SIZE = 0x40,
89 MTHCA_MTT_SEG_SIZE = 0x40,
90
91 MTHCA_QP_PER_MGM = 4 * (MTHCA_MGM_ENTRY_SIZE / 16 - 2)
92};
93
94enum {
95 MTHCA_EQ_CMD,
96 MTHCA_EQ_ASYNC,
97 MTHCA_EQ_COMP,
98 MTHCA_NUM_EQ
99};
100
101enum {
102 MTHCA_OPCODE_NOP = 0x00,
103 MTHCA_OPCODE_RDMA_WRITE = 0x08,
104 MTHCA_OPCODE_RDMA_WRITE_IMM = 0x09,
105 MTHCA_OPCODE_SEND = 0x0a,
106 MTHCA_OPCODE_SEND_IMM = 0x0b,
107 MTHCA_OPCODE_RDMA_READ = 0x10,
108 MTHCA_OPCODE_ATOMIC_CS = 0x11,
109 MTHCA_OPCODE_ATOMIC_FA = 0x12,
110 MTHCA_OPCODE_BIND_MW = 0x18,
111 MTHCA_OPCODE_INVALID = 0xff
112};
113
114enum {
115 MTHCA_CMD_USE_EVENTS = 1 << 0,
116 MTHCA_CMD_POST_DOORBELLS = 1 << 1
117};
118
119enum {
120 MTHCA_CMD_NUM_DBELL_DWORDS = 8
121};
122
123struct mthca_cmd {
124 struct pci_pool *pool;
125 struct mutex hcr_mutex;
126 struct semaphore poll_sem;
127 struct semaphore event_sem;
128 int max_cmds;
129 spinlock_t context_lock;
130 int free_head;
131 struct mthca_cmd_context *context;
132 u16 token_mask;
133 u32 flags;
134 void __iomem *dbell_map;
135 u16 dbell_offsets[MTHCA_CMD_NUM_DBELL_DWORDS];
136};
137
138struct mthca_limits {
139 int num_ports;
140 int vl_cap;
141 int mtu_cap;
142 int gid_table_len;
143 int pkey_table_len;
144 int local_ca_ack_delay;
145 int num_uars;
146 int max_sg;
147 int num_qps;
148 int max_wqes;
149 int max_desc_sz;
150 int max_qp_init_rdma;
151 int reserved_qps;
152 int num_srqs;
153 int max_srq_wqes;
154 int max_srq_sge;
155 int reserved_srqs;
156 int num_eecs;
157 int reserved_eecs;
158 int num_cqs;
159 int max_cqes;
160 int reserved_cqs;
161 int num_eqs;
162 int reserved_eqs;
163 int num_mpts;
164 int num_mtt_segs;
165 int fmr_reserved_mtts;
166 int reserved_mtts;
167 int reserved_mrws;
168 int reserved_uars;
169 int num_mgms;
170 int num_amgms;
171 int reserved_mcgs;
172 int num_pds;
173 int reserved_pds;
174 u32 page_size_cap;
175 u32 flags;
176 u16 stat_rate_support;
177 u8 port_width_cap;
178};
179
180struct mthca_alloc {
181 u32 last;
182 u32 top;
183 u32 max;
184 u32 mask;
185 spinlock_t lock;
186 unsigned long *table;
187};
188
189struct mthca_array {
190 struct {
191 void **page;
192 int used;
193 } *page_list;
194};
195
196struct mthca_uar_table {
197 struct mthca_alloc alloc;
198 u64 uarc_base;
199 int uarc_size;
200};
201
202struct mthca_pd_table {
203 struct mthca_alloc alloc;
204};
205
206struct mthca_buddy {
207 unsigned long **bits;
208 int max_order;
209 spinlock_t lock;
210};
211
212struct mthca_mr_table {
213 struct mthca_alloc mpt_alloc;
214 struct mthca_buddy mtt_buddy;
215 struct mthca_buddy *fmr_mtt_buddy;
216 u64 mtt_base;
217 u64 mpt_base;
218 struct mthca_icm_table *mtt_table;
219 struct mthca_icm_table *mpt_table;
220 struct {
221 void __iomem *mpt_base;
222 void __iomem *mtt_base;
223 struct mthca_buddy mtt_buddy;
224 } tavor_fmr;
225};
226
227struct mthca_eq_table {
228 struct mthca_alloc alloc;
229 void __iomem *clr_int;
230 u32 clr_mask;
231 u32 arm_mask;
232 struct mthca_eq eq[MTHCA_NUM_EQ];
233 u64 icm_virt;
234 struct page *icm_page;
235 dma_addr_t icm_dma;
236 int have_irq;
237 u8 inta_pin;
238};
239
240struct mthca_cq_table {
241 struct mthca_alloc alloc;
242 spinlock_t lock;
243 struct mthca_array cq;
244 struct mthca_icm_table *table;
245};
246
247struct mthca_srq_table {
248 struct mthca_alloc alloc;
249 spinlock_t lock;
250 struct mthca_array srq;
251 struct mthca_icm_table *table;
252};
253
254struct mthca_qp_table {
255 struct mthca_alloc alloc;
256 u32 rdb_base;
257 int rdb_shift;
258 int sqp_start;
259 spinlock_t lock;
260 struct mthca_array qp;
261 struct mthca_icm_table *qp_table;
262 struct mthca_icm_table *eqp_table;
263 struct mthca_icm_table *rdb_table;
264};
265
266struct mthca_av_table {
267 struct pci_pool *pool;
268 int num_ddr_avs;
269 u64 ddr_av_base;
270 void __iomem *av_map;
271 struct mthca_alloc alloc;
272};
273
274struct mthca_mcg_table {
275 struct mutex mutex;
276 struct mthca_alloc alloc;
277 struct mthca_icm_table *table;
278};
279
280struct mthca_catas_err {
281 u64 addr;
282 u32 __iomem *map;
283 unsigned long stop;
284 u32 size;
285 struct timer_list timer;
286 struct list_head list;
287};
288
289extern struct mutex mthca_device_mutex;
290
291struct mthca_dev {
292 struct ib_device ib_dev;
293 struct pci_dev *pdev;
294
295 int hca_type;
296 unsigned long mthca_flags;
297 unsigned long device_cap_flags;
298
299 u32 rev_id;
300 char board_id[MTHCA_BOARD_ID_LEN];
301
302
303 u64 fw_ver;
304 union {
305 struct {
306 u64 fw_start;
307 u64 fw_end;
308 } tavor;
309 struct {
310 u64 clr_int_base;
311 u64 eq_arm_base;
312 u64 eq_set_ci_base;
313 struct mthca_icm *fw_icm;
314 struct mthca_icm *aux_icm;
315 u16 fw_pages;
316 } arbel;
317 } fw;
318
319 u64 ddr_start;
320 u64 ddr_end;
321
322 MTHCA_DECLARE_DOORBELL_LOCK(doorbell_lock)
323 struct mutex cap_mask_mutex;
324
325 void __iomem *hcr;
326 void __iomem *kar;
327 void __iomem *clr_base;
328 union {
329 struct {
330 void __iomem *ecr_base;
331 } tavor;
332 struct {
333 void __iomem *eq_arm;
334 void __iomem *eq_set_ci_base;
335 } arbel;
336 } eq_regs;
337
338 struct mthca_cmd cmd;
339 struct mthca_limits limits;
340
341 struct mthca_uar_table uar_table;
342 struct mthca_pd_table pd_table;
343 struct mthca_mr_table mr_table;
344 struct mthca_eq_table eq_table;
345 struct mthca_cq_table cq_table;
346 struct mthca_srq_table srq_table;
347 struct mthca_qp_table qp_table;
348 struct mthca_av_table av_table;
349 struct mthca_mcg_table mcg_table;
350
351 struct mthca_catas_err catas_err;
352
353 struct mthca_uar driver_uar;
354 struct mthca_db_table *db_tab;
355 struct mthca_pd driver_pd;
356 struct mthca_mr driver_mr;
357
358 struct ib_mad_agent *send_agent[MTHCA_MAX_PORTS][2];
359 struct ib_ah *sm_ah[MTHCA_MAX_PORTS];
360 spinlock_t sm_lock;
361 u8 rate[MTHCA_MAX_PORTS];
362};
363
364#ifdef CONFIG_INFINIBAND_MTHCA_DEBUG
365extern int mthca_debug_level;
366
367#define mthca_dbg(mdev, format, arg...) \
368 do { \
369 if (mthca_debug_level) \
370 dev_printk(KERN_DEBUG, &mdev->pdev->dev, format, ## arg); \
371 } while (0)
372
373#else
374
375#define mthca_dbg(mdev, format, arg...) do { (void) mdev; } while (0)
376
377#endif
378
379#define mthca_err(mdev, format, arg...) \
380 dev_err(&mdev->pdev->dev, format, ## arg)
381#define mthca_info(mdev, format, arg...) \
382 dev_info(&mdev->pdev->dev, format, ## arg)
383#define mthca_warn(mdev, format, arg...) \
384 dev_warn(&mdev->pdev->dev, format, ## arg)
385
386extern void __buggy_use_of_MTHCA_GET(void);
387extern void __buggy_use_of_MTHCA_PUT(void);
388
389#define MTHCA_GET(dest, source, offset) \
390 do { \
391 void *__p = (char *) (source) + (offset); \
392 switch (sizeof (dest)) { \
393 case 1: (dest) = *(u8 *) __p; break; \
394 case 2: (dest) = be16_to_cpup(__p); break; \
395 case 4: (dest) = be32_to_cpup(__p); break; \
396 case 8: (dest) = be64_to_cpup(__p); break; \
397 default: __buggy_use_of_MTHCA_GET(); \
398 } \
399 } while (0)
400
401#define MTHCA_PUT(dest, source, offset) \
402 do { \
403 void *__d = ((char *) (dest) + (offset)); \
404 switch (sizeof(source)) { \
405 case 1: *(u8 *) __d = (source); break; \
406 case 2: *(__be16 *) __d = cpu_to_be16(source); break; \
407 case 4: *(__be32 *) __d = cpu_to_be32(source); break; \
408 case 8: *(__be64 *) __d = cpu_to_be64(source); break; \
409 default: __buggy_use_of_MTHCA_PUT(); \
410 } \
411 } while (0)
412
413int mthca_reset(struct mthca_dev *mdev);
414
415u32 mthca_alloc(struct mthca_alloc *alloc);
416void mthca_free(struct mthca_alloc *alloc, u32 obj);
417int mthca_alloc_init(struct mthca_alloc *alloc, u32 num, u32 mask,
418 u32 reserved);
419void mthca_alloc_cleanup(struct mthca_alloc *alloc);
420void *mthca_array_get(struct mthca_array *array, int index);
421int mthca_array_set(struct mthca_array *array, int index, void *value);
422void mthca_array_clear(struct mthca_array *array, int index);
423int mthca_array_init(struct mthca_array *array, int nent);
424void mthca_array_cleanup(struct mthca_array *array, int nent);
425int mthca_buf_alloc(struct mthca_dev *dev, int size, int max_direct,
426 union mthca_buf *buf, int *is_direct, struct mthca_pd *pd,
427 int hca_write, struct mthca_mr *mr);
428void mthca_buf_free(struct mthca_dev *dev, int size, union mthca_buf *buf,
429 int is_direct, struct mthca_mr *mr);
430
431int mthca_init_uar_table(struct mthca_dev *dev);
432int mthca_init_pd_table(struct mthca_dev *dev);
433int mthca_init_mr_table(struct mthca_dev *dev);
434int mthca_init_eq_table(struct mthca_dev *dev);
435int mthca_init_cq_table(struct mthca_dev *dev);
436int mthca_init_srq_table(struct mthca_dev *dev);
437int mthca_init_qp_table(struct mthca_dev *dev);
438int mthca_init_av_table(struct mthca_dev *dev);
439int mthca_init_mcg_table(struct mthca_dev *dev);
440
441void mthca_cleanup_uar_table(struct mthca_dev *dev);
442void mthca_cleanup_pd_table(struct mthca_dev *dev);
443void mthca_cleanup_mr_table(struct mthca_dev *dev);
444void mthca_cleanup_eq_table(struct mthca_dev *dev);
445void mthca_cleanup_cq_table(struct mthca_dev *dev);
446void mthca_cleanup_srq_table(struct mthca_dev *dev);
447void mthca_cleanup_qp_table(struct mthca_dev *dev);
448void mthca_cleanup_av_table(struct mthca_dev *dev);
449void mthca_cleanup_mcg_table(struct mthca_dev *dev);
450
451int mthca_register_device(struct mthca_dev *dev);
452void mthca_unregister_device(struct mthca_dev *dev);
453
454void mthca_start_catas_poll(struct mthca_dev *dev);
455void mthca_stop_catas_poll(struct mthca_dev *dev);
456int __mthca_restart_one(struct pci_dev *pdev);
457int mthca_catas_init(void);
458void mthca_catas_cleanup(void);
459
460int mthca_uar_alloc(struct mthca_dev *dev, struct mthca_uar *uar);
461void mthca_uar_free(struct mthca_dev *dev, struct mthca_uar *uar);
462
463int mthca_pd_alloc(struct mthca_dev *dev, int privileged, struct mthca_pd *pd);
464void mthca_pd_free(struct mthca_dev *dev, struct mthca_pd *pd);
465
466int mthca_write_mtt_size(struct mthca_dev *dev);
467
468struct mthca_mtt *mthca_alloc_mtt(struct mthca_dev *dev, int size);
469void mthca_free_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt);
470int mthca_write_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt,
471 int start_index, u64 *buffer_list, int list_len);
472int mthca_mr_alloc(struct mthca_dev *dev, u32 pd, int buffer_size_shift,
473 u64 iova, u64 total_size, u32 access, struct mthca_mr *mr);
474int mthca_mr_alloc_notrans(struct mthca_dev *dev, u32 pd,
475 u32 access, struct mthca_mr *mr);
476int mthca_mr_alloc_phys(struct mthca_dev *dev, u32 pd,
477 u64 *buffer_list, int buffer_size_shift,
478 int list_len, u64 iova, u64 total_size,
479 u32 access, struct mthca_mr *mr);
480void mthca_free_mr(struct mthca_dev *dev, struct mthca_mr *mr);
481
482int mthca_fmr_alloc(struct mthca_dev *dev, u32 pd,
483 u32 access, struct mthca_fmr *fmr);
484int mthca_tavor_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list,
485 int list_len, u64 iova);
486void mthca_tavor_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr);
487int mthca_arbel_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list,
488 int list_len, u64 iova);
489void mthca_arbel_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr);
490int mthca_free_fmr(struct mthca_dev *dev, struct mthca_fmr *fmr);
491
492int mthca_map_eq_icm(struct mthca_dev *dev, u64 icm_virt);
493void mthca_unmap_eq_icm(struct mthca_dev *dev);
494
495int mthca_poll_cq(struct ib_cq *ibcq, int num_entries,
496 struct ib_wc *entry);
497int mthca_tavor_arm_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags);
498int mthca_arbel_arm_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags);
499int mthca_init_cq(struct mthca_dev *dev, int nent,
500 struct mthca_ucontext *ctx, u32 pdn,
501 struct mthca_cq *cq);
502void mthca_free_cq(struct mthca_dev *dev,
503 struct mthca_cq *cq);
504void mthca_cq_completion(struct mthca_dev *dev, u32 cqn);
505void mthca_cq_event(struct mthca_dev *dev, u32 cqn,
506 enum ib_event_type event_type);
507void mthca_cq_clean(struct mthca_dev *dev, struct mthca_cq *cq, u32 qpn,
508 struct mthca_srq *srq);
509void mthca_cq_resize_copy_cqes(struct mthca_cq *cq);
510int mthca_alloc_cq_buf(struct mthca_dev *dev, struct mthca_cq_buf *buf, int nent);
511void mthca_free_cq_buf(struct mthca_dev *dev, struct mthca_cq_buf *buf, int cqe);
512
513int mthca_alloc_srq(struct mthca_dev *dev, struct mthca_pd *pd,
514 struct ib_srq_attr *attr, struct mthca_srq *srq);
515void mthca_free_srq(struct mthca_dev *dev, struct mthca_srq *srq);
516int mthca_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
517 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata);
518int mthca_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr);
519int mthca_max_srq_sge(struct mthca_dev *dev);
520void mthca_srq_event(struct mthca_dev *dev, u32 srqn,
521 enum ib_event_type event_type);
522void mthca_free_srq_wqe(struct mthca_srq *srq, u32 wqe_addr);
523int mthca_tavor_post_srq_recv(struct ib_srq *srq, struct ib_recv_wr *wr,
524 struct ib_recv_wr **bad_wr);
525int mthca_arbel_post_srq_recv(struct ib_srq *srq, struct ib_recv_wr *wr,
526 struct ib_recv_wr **bad_wr);
527
528void mthca_qp_event(struct mthca_dev *dev, u32 qpn,
529 enum ib_event_type event_type);
530int mthca_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask,
531 struct ib_qp_init_attr *qp_init_attr);
532int mthca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
533 struct ib_udata *udata);
534int mthca_tavor_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
535 struct ib_send_wr **bad_wr);
536int mthca_tavor_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
537 struct ib_recv_wr **bad_wr);
538int mthca_arbel_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
539 struct ib_send_wr **bad_wr);
540int mthca_arbel_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
541 struct ib_recv_wr **bad_wr);
542void mthca_free_err_wqe(struct mthca_dev *dev, struct mthca_qp *qp, int is_send,
543 int index, int *dbd, __be32 *new_wqe);
544int mthca_alloc_qp(struct mthca_dev *dev,
545 struct mthca_pd *pd,
546 struct mthca_cq *send_cq,
547 struct mthca_cq *recv_cq,
548 enum ib_qp_type type,
549 enum ib_sig_type send_policy,
550 struct ib_qp_cap *cap,
551 struct mthca_qp *qp);
552int mthca_alloc_sqp(struct mthca_dev *dev,
553 struct mthca_pd *pd,
554 struct mthca_cq *send_cq,
555 struct mthca_cq *recv_cq,
556 enum ib_sig_type send_policy,
557 struct ib_qp_cap *cap,
558 int qpn,
559 int port,
560 struct mthca_sqp *sqp);
561void mthca_free_qp(struct mthca_dev *dev, struct mthca_qp *qp);
562int mthca_create_ah(struct mthca_dev *dev,
563 struct mthca_pd *pd,
564 struct ib_ah_attr *ah_attr,
565 struct mthca_ah *ah);
566int mthca_destroy_ah(struct mthca_dev *dev, struct mthca_ah *ah);
567int mthca_read_ah(struct mthca_dev *dev, struct mthca_ah *ah,
568 struct ib_ud_header *header);
569int mthca_ah_query(struct ib_ah *ibah, struct ib_ah_attr *attr);
570int mthca_ah_grh_present(struct mthca_ah *ah);
571u8 mthca_get_rate(struct mthca_dev *dev, int static_rate, u8 port);
572enum ib_rate mthca_rate_to_ib(struct mthca_dev *dev, u8 mthca_rate, u8 port);
573
574int mthca_multicast_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid);
575int mthca_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid);
576
577int mthca_process_mad(struct ib_device *ibdev,
578 int mad_flags,
579 u8 port_num,
580 struct ib_wc *in_wc,
581 struct ib_grh *in_grh,
582 struct ib_mad *in_mad,
583 struct ib_mad *out_mad);
584int mthca_create_agents(struct mthca_dev *dev);
585void mthca_free_agents(struct mthca_dev *dev);
586
587static inline struct mthca_dev *to_mdev(struct ib_device *ibdev)
588{
589 return container_of(ibdev, struct mthca_dev, ib_dev);
590}
591
592static inline int mthca_is_memfree(struct mthca_dev *dev)
593{
594 return dev->mthca_flags & MTHCA_FLAG_MEMFREE;
595}
596
597#endif
598
