1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33#ifndef _MLX5_FS_CORE_
34#define _MLX5_FS_CORE_
35
36#include <linux/refcount.h>
37#include <linux/mlx5/fs.h>
38#include <linux/rhashtable.h>
39#include <linux/llist.h>
40#include <steering/fs_dr.h>
41
42#define FDB_TC_MAX_CHAIN 3
43#define FDB_FT_CHAIN (FDB_TC_MAX_CHAIN + 1)
44#define FDB_TC_SLOW_PATH_CHAIN (FDB_FT_CHAIN + 1)
45
46
47#define FDB_NUM_CHAINS (FDB_FT_CHAIN + 1)
48
49#define FDB_TC_MAX_PRIO 16
50#define FDB_TC_LEVELS_PER_PRIO 2
51
52struct mlx5_modify_hdr {
53 enum mlx5_flow_namespace_type ns_type;
54 union {
55 struct mlx5_fs_dr_action action;
56 u32 id;
57 };
58};
59
60struct mlx5_pkt_reformat {
61 enum mlx5_flow_namespace_type ns_type;
62 int reformat_type;
63 union {
64 struct mlx5_fs_dr_action action;
65 u32 id;
66 };
67};
68
69
70
71
72
73
74
75
76
77
78
79
80enum fs_node_type {
81 FS_TYPE_NAMESPACE,
82 FS_TYPE_PRIO,
83 FS_TYPE_PRIO_CHAINS,
84 FS_TYPE_FLOW_TABLE,
85 FS_TYPE_FLOW_GROUP,
86 FS_TYPE_FLOW_ENTRY,
87 FS_TYPE_FLOW_DEST
88};
89
90enum fs_flow_table_type {
91 FS_FT_NIC_RX = 0x0,
92 FS_FT_NIC_TX = 0x1,
93 FS_FT_ESW_EGRESS_ACL = 0x2,
94 FS_FT_ESW_INGRESS_ACL = 0x3,
95 FS_FT_FDB = 0X4,
96 FS_FT_SNIFFER_RX = 0X5,
97 FS_FT_SNIFFER_TX = 0X6,
98 FS_FT_RDMA_RX = 0X7,
99 FS_FT_RDMA_TX = 0X8,
100 FS_FT_MAX_TYPE = FS_FT_RDMA_TX,
101};
102
103enum fs_flow_table_op_mod {
104 FS_FT_OP_MOD_NORMAL,
105 FS_FT_OP_MOD_LAG_DEMUX,
106};
107
108enum fs_fte_status {
109 FS_FTE_STATUS_EXISTING = 1UL << 0,
110};
111
112enum mlx5_flow_steering_mode {
113 MLX5_FLOW_STEERING_MODE_DMFS,
114 MLX5_FLOW_STEERING_MODE_SMFS
115};
116
117struct mlx5_flow_steering {
118 struct mlx5_core_dev *dev;
119 enum mlx5_flow_steering_mode mode;
120 struct kmem_cache *fgs_cache;
121 struct kmem_cache *ftes_cache;
122 struct mlx5_flow_root_namespace *root_ns;
123 struct mlx5_flow_root_namespace *fdb_root_ns;
124 struct mlx5_flow_namespace **fdb_sub_ns;
125 struct mlx5_flow_root_namespace **esw_egress_root_ns;
126 struct mlx5_flow_root_namespace **esw_ingress_root_ns;
127 struct mlx5_flow_root_namespace *sniffer_tx_root_ns;
128 struct mlx5_flow_root_namespace *sniffer_rx_root_ns;
129 struct mlx5_flow_root_namespace *rdma_rx_root_ns;
130 struct mlx5_flow_root_namespace *rdma_tx_root_ns;
131 struct mlx5_flow_root_namespace *egress_root_ns;
132 int esw_egress_acl_vports;
133 int esw_ingress_acl_vports;
134};
135
136struct fs_node {
137 struct list_head list;
138 struct list_head children;
139 enum fs_node_type type;
140 struct fs_node *parent;
141 struct fs_node *root;
142
143 struct rw_semaphore lock;
144 refcount_t refcount;
145 bool active;
146 void (*del_hw_func)(struct fs_node *);
147 void (*del_sw_func)(struct fs_node *);
148 atomic_t version;
149};
150
151struct mlx5_flow_rule {
152 struct fs_node node;
153 struct mlx5_flow_table *ft;
154 struct mlx5_flow_destination dest_attr;
155
156
157
158 struct list_head next_ft;
159 u32 sw_action;
160};
161
162struct mlx5_flow_handle {
163 int num_rules;
164 struct mlx5_flow_rule *rule[];
165};
166
167
168struct mlx5_flow_table {
169 struct fs_node node;
170 struct mlx5_fs_dr_table fs_dr_table;
171 u32 id;
172 u16 vport;
173 unsigned int max_fte;
174 unsigned int level;
175 enum fs_flow_table_type type;
176 enum fs_flow_table_op_mod op_mod;
177 struct {
178 bool active;
179 unsigned int required_groups;
180 unsigned int group_size;
181 unsigned int num_groups;
182 unsigned int max_fte;
183 } autogroup;
184
185 struct mutex lock;
186
187 struct list_head fwd_rules;
188 u32 flags;
189 struct rhltable fgs_hash;
190 enum mlx5_flow_table_miss_action def_miss_action;
191 struct mlx5_flow_namespace *ns;
192};
193
194struct mlx5_ft_underlay_qp {
195 struct list_head list;
196 u32 qpn;
197};
198
199#define MLX5_FTE_MATCH_PARAM_RESERVED reserved_at_c00
200
201
202
203#define MLX5_ST_SZ_DW_MATCH_PARAM \
204 ((MLX5_BYTE_OFF(fte_match_param, MLX5_FTE_MATCH_PARAM_RESERVED) / sizeof(u32)) + \
205 BUILD_BUG_ON_ZERO(MLX5_ST_SZ_BYTES(fte_match_param) != \
206 MLX5_FLD_SZ_BYTES(fte_match_param, \
207 MLX5_FTE_MATCH_PARAM_RESERVED) +\
208 MLX5_BYTE_OFF(fte_match_param, \
209 MLX5_FTE_MATCH_PARAM_RESERVED)))
210
211
212struct fs_fte {
213 struct fs_node node;
214 struct mlx5_fs_dr_rule fs_dr_rule;
215 u32 val[MLX5_ST_SZ_DW_MATCH_PARAM];
216 u32 dests_size;
217 u32 index;
218 struct mlx5_flow_context flow_context;
219 struct mlx5_flow_act action;
220 enum fs_fte_status status;
221 struct mlx5_fc *counter;
222 struct rhash_head hash;
223 int modify_mask;
224};
225
226
227struct fs_prio {
228 struct fs_node node;
229 unsigned int num_levels;
230 unsigned int start_level;
231 unsigned int prio;
232 unsigned int num_ft;
233};
234
235
236struct mlx5_flow_namespace {
237
238 struct fs_node node;
239 enum mlx5_flow_table_miss_action def_miss_action;
240};
241
242struct mlx5_flow_group_mask {
243 u8 match_criteria_enable;
244 u32 match_criteria[MLX5_ST_SZ_DW_MATCH_PARAM];
245};
246
247
248struct mlx5_flow_group {
249 struct fs_node node;
250 struct mlx5_fs_dr_matcher fs_dr_matcher;
251 struct mlx5_flow_group_mask mask;
252 u32 start_index;
253 u32 max_ftes;
254 struct ida fte_allocator;
255 u32 id;
256 struct rhashtable ftes_hash;
257 struct rhlist_head hash;
258};
259
260struct mlx5_flow_root_namespace {
261 struct mlx5_flow_namespace ns;
262 enum mlx5_flow_steering_mode mode;
263 struct mlx5_fs_dr_domain fs_dr_domain;
264 enum fs_flow_table_type table_type;
265 struct mlx5_core_dev *dev;
266 struct mlx5_flow_table *root_ft;
267
268 struct mutex chain_lock;
269 struct list_head underlay_qpns;
270 const struct mlx5_flow_cmds *cmds;
271};
272
273int mlx5_init_fc_stats(struct mlx5_core_dev *dev);
274void mlx5_cleanup_fc_stats(struct mlx5_core_dev *dev);
275void mlx5_fc_queue_stats_work(struct mlx5_core_dev *dev,
276 struct delayed_work *dwork,
277 unsigned long delay);
278void mlx5_fc_update_sampling_interval(struct mlx5_core_dev *dev,
279 unsigned long interval);
280
281const struct mlx5_flow_cmds *mlx5_fs_cmd_get_fw_cmds(void);
282
283int mlx5_flow_namespace_set_peer(struct mlx5_flow_root_namespace *ns,
284 struct mlx5_flow_root_namespace *peer_ns);
285
286int mlx5_flow_namespace_set_mode(struct mlx5_flow_namespace *ns,
287 enum mlx5_flow_steering_mode mode);
288
289int mlx5_init_fs(struct mlx5_core_dev *dev);
290void mlx5_cleanup_fs(struct mlx5_core_dev *dev);
291
292int mlx5_fs_egress_acls_init(struct mlx5_core_dev *dev, int total_vports);
293void mlx5_fs_egress_acls_cleanup(struct mlx5_core_dev *dev);
294int mlx5_fs_ingress_acls_init(struct mlx5_core_dev *dev, int total_vports);
295void mlx5_fs_ingress_acls_cleanup(struct mlx5_core_dev *dev);
296
297#define fs_get_obj(v, _node) {v = container_of((_node), typeof(*v), node); }
298
299#define fs_list_for_each_entry(pos, root) \
300 list_for_each_entry(pos, root, node.list)
301
302#define fs_list_for_each_entry_safe(pos, tmp, root) \
303 list_for_each_entry_safe(pos, tmp, root, node.list)
304
305#define fs_for_each_ns_or_ft_reverse(pos, prio) \
306 list_for_each_entry_reverse(pos, &(prio)->node.children, list)
307
308#define fs_for_each_ns_or_ft(pos, prio) \
309 list_for_each_entry(pos, (&(prio)->node.children), list)
310
311#define fs_for_each_prio(pos, ns) \
312 fs_list_for_each_entry(pos, &(ns)->node.children)
313
314#define fs_for_each_ns(pos, prio) \
315 fs_list_for_each_entry(pos, &(prio)->node.children)
316
317#define fs_for_each_ft(pos, prio) \
318 fs_list_for_each_entry(pos, &(prio)->node.children)
319
320#define fs_for_each_ft_safe(pos, tmp, prio) \
321 fs_list_for_each_entry_safe(pos, tmp, &(prio)->node.children)
322
323#define fs_for_each_fg(pos, ft) \
324 fs_list_for_each_entry(pos, &(ft)->node.children)
325
326#define fs_for_each_fte(pos, fg) \
327 fs_list_for_each_entry(pos, &(fg)->node.children)
328
329#define fs_for_each_dst(pos, fte) \
330 fs_list_for_each_entry(pos, &(fte)->node.children)
331
332#define MLX5_CAP_FLOWTABLE_TYPE(mdev, cap, type) ( \
333 (type == FS_FT_NIC_RX) ? MLX5_CAP_FLOWTABLE_NIC_RX(mdev, cap) : \
334 (type == FS_FT_NIC_TX) ? MLX5_CAP_FLOWTABLE_NIC_TX(mdev, cap) : \
335 (type == FS_FT_ESW_EGRESS_ACL) ? MLX5_CAP_ESW_EGRESS_ACL(mdev, cap) : \
336 (type == FS_FT_ESW_INGRESS_ACL) ? MLX5_CAP_ESW_INGRESS_ACL(mdev, cap) : \
337 (type == FS_FT_FDB) ? MLX5_CAP_ESW_FLOWTABLE_FDB(mdev, cap) : \
338 (type == FS_FT_SNIFFER_RX) ? MLX5_CAP_FLOWTABLE_SNIFFER_RX(mdev, cap) : \
339 (type == FS_FT_SNIFFER_TX) ? MLX5_CAP_FLOWTABLE_SNIFFER_TX(mdev, cap) : \
340 (type == FS_FT_RDMA_RX) ? MLX5_CAP_FLOWTABLE_RDMA_RX(mdev, cap) : \
341 (type == FS_FT_RDMA_TX) ? MLX5_CAP_FLOWTABLE_RDMA_TX(mdev, cap) : \
342 (BUILD_BUG_ON_ZERO(FS_FT_RDMA_TX != FS_FT_MAX_TYPE))\
343 )
344
345#endif
346