1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3 * Copyright (c) 2004-2007 Intel Corporation. All rights reserved.
4 * Copyright (c) 2004 Topspin Corporation. All rights reserved.
5 * Copyright (c) 2004, 2005 Voltaire Corporation. All rights reserved.
6 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
7 * Copyright (c) 2019, Mellanox Technologies inc. All rights reserved.
8 */
9
10 #include <linux/completion.h>
11 #include <linux/dma-mapping.h>
12 #include <linux/device.h>
13 #include <linux/module.h>
14 #include <linux/err.h>
15 #include <linux/idr.h>
16 #include <linux/interrupt.h>
17 #include <linux/random.h>
18 #include <linux/rbtree.h>
19 #include <linux/spinlock.h>
20 #include <linux/slab.h>
21 #include <linux/sysfs.h>
22 #include <linux/workqueue.h>
23 #include <linux/kdev_t.h>
24 #include <linux/etherdevice.h>
25
26 #include <rdma/ib_cache.h>
27 #include <rdma/ib_cm.h>
28 #include <rdma/ib_sysfs.h>
29 #include "cm_msgs.h"
30 #include "core_priv.h"
31 #include "cm_trace.h"
32
33 MODULE_AUTHOR("Sean Hefty");
34 MODULE_DESCRIPTION("InfiniBand CM");
35 MODULE_LICENSE("Dual BSD/GPL");
36
37 #define CM_DESTROY_ID_WAIT_TIMEOUT 10000 /* msecs */
38 static const char * const ibcm_rej_reason_strs[] = {
39 [IB_CM_REJ_NO_QP] = "no QP",
40 [IB_CM_REJ_NO_EEC] = "no EEC",
41 [IB_CM_REJ_NO_RESOURCES] = "no resources",
42 [IB_CM_REJ_TIMEOUT] = "timeout",
43 [IB_CM_REJ_UNSUPPORTED] = "unsupported",
44 [IB_CM_REJ_INVALID_COMM_ID] = "invalid comm ID",
45 [IB_CM_REJ_INVALID_COMM_INSTANCE] = "invalid comm instance",
46 [IB_CM_REJ_INVALID_SERVICE_ID] = "invalid service ID",
47 [IB_CM_REJ_INVALID_TRANSPORT_TYPE] = "invalid transport type",
48 [IB_CM_REJ_STALE_CONN] = "stale conn",
49 [IB_CM_REJ_RDC_NOT_EXIST] = "RDC not exist",
50 [IB_CM_REJ_INVALID_GID] = "invalid GID",
51 [IB_CM_REJ_INVALID_LID] = "invalid LID",
52 [IB_CM_REJ_INVALID_SL] = "invalid SL",
53 [IB_CM_REJ_INVALID_TRAFFIC_CLASS] = "invalid traffic class",
54 [IB_CM_REJ_INVALID_HOP_LIMIT] = "invalid hop limit",
55 [IB_CM_REJ_INVALID_PACKET_RATE] = "invalid packet rate",
56 [IB_CM_REJ_INVALID_ALT_GID] = "invalid alt GID",
57 [IB_CM_REJ_INVALID_ALT_LID] = "invalid alt LID",
58 [IB_CM_REJ_INVALID_ALT_SL] = "invalid alt SL",
59 [IB_CM_REJ_INVALID_ALT_TRAFFIC_CLASS] = "invalid alt traffic class",
60 [IB_CM_REJ_INVALID_ALT_HOP_LIMIT] = "invalid alt hop limit",
61 [IB_CM_REJ_INVALID_ALT_PACKET_RATE] = "invalid alt packet rate",
62 [IB_CM_REJ_PORT_CM_REDIRECT] = "port CM redirect",
63 [IB_CM_REJ_PORT_REDIRECT] = "port redirect",
64 [IB_CM_REJ_INVALID_MTU] = "invalid MTU",
65 [IB_CM_REJ_INSUFFICIENT_RESP_RESOURCES] = "insufficient resp resources",
66 [IB_CM_REJ_CONSUMER_DEFINED] = "consumer defined",
67 [IB_CM_REJ_INVALID_RNR_RETRY] = "invalid RNR retry",
68 [IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID] = "duplicate local comm ID",
69 [IB_CM_REJ_INVALID_CLASS_VERSION] = "invalid class version",
70 [IB_CM_REJ_INVALID_FLOW_LABEL] = "invalid flow label",
71 [IB_CM_REJ_INVALID_ALT_FLOW_LABEL] = "invalid alt flow label",
72 [IB_CM_REJ_VENDOR_OPTION_NOT_SUPPORTED] =
73 "vendor option is not supported",
74 };
75
ibcm_reject_msg(int reason)76 const char *__attribute_const__ ibcm_reject_msg(int reason)
77 {
78 size_t index = reason;
79
80 if (index < ARRAY_SIZE(ibcm_rej_reason_strs) &&
81 ibcm_rej_reason_strs[index])
82 return ibcm_rej_reason_strs[index];
83 else
84 return "unrecognized reason";
85 }
86 EXPORT_SYMBOL(ibcm_reject_msg);
87
88 struct cm_id_private;
89 struct cm_work;
90 static int cm_add_one(struct ib_device *device);
91 static void cm_remove_one(struct ib_device *device, void *client_data);
92 static void cm_process_work(struct cm_id_private *cm_id_priv,
93 struct cm_work *work);
94 static int cm_send_sidr_rep_locked(struct cm_id_private *cm_id_priv,
95 struct ib_cm_sidr_rep_param *param);
96 static int cm_send_dreq_locked(struct cm_id_private *cm_id_priv,
97 const void *private_data, u8 private_data_len);
98 static int cm_send_drep_locked(struct cm_id_private *cm_id_priv,
99 void *private_data, u8 private_data_len);
100 static int cm_send_rej_locked(struct cm_id_private *cm_id_priv,
101 enum ib_cm_rej_reason reason, void *ari,
102 u8 ari_length, const void *private_data,
103 u8 private_data_len);
104
105 static struct ib_client cm_client = {
106 .name = "cm",
107 .add = cm_add_one,
108 .remove = cm_remove_one
109 };
110
111 static struct ib_cm {
112 spinlock_t lock;
113 struct list_head device_list;
114 rwlock_t device_lock;
115 struct rb_root listen_service_table;
116 u64 listen_service_id;
117 /* struct rb_root peer_service_table; todo: fix peer to peer */
118 struct rb_root remote_qp_table;
119 struct rb_root remote_id_table;
120 struct rb_root remote_sidr_table;
121 struct xarray local_id_table;
122 u32 local_id_next;
123 __be32 random_id_operand;
124 struct list_head timewait_list;
125 struct workqueue_struct *wq;
126 } cm;
127
128 /* Counter indexes ordered by attribute ID */
129 enum {
130 CM_REQ_COUNTER,
131 CM_MRA_COUNTER,
132 CM_REJ_COUNTER,
133 CM_REP_COUNTER,
134 CM_RTU_COUNTER,
135 CM_DREQ_COUNTER,
136 CM_DREP_COUNTER,
137 CM_SIDR_REQ_COUNTER,
138 CM_SIDR_REP_COUNTER,
139 CM_LAP_COUNTER,
140 CM_APR_COUNTER,
141 CM_ATTR_COUNT,
142 CM_ATTR_ID_OFFSET = 0x0010,
143 };
144
145 enum {
146 CM_XMIT,
147 CM_XMIT_RETRIES,
148 CM_RECV,
149 CM_RECV_DUPLICATES,
150 CM_COUNTER_GROUPS
151 };
152
153 struct cm_counter_attribute {
154 struct ib_port_attribute attr;
155 unsigned short group;
156 unsigned short index;
157 };
158
159 struct cm_port {
160 struct cm_device *cm_dev;
161 struct ib_mad_agent *mad_agent;
162 u32 port_num;
163 atomic_long_t counters[CM_COUNTER_GROUPS][CM_ATTR_COUNT];
164 };
165
166 struct cm_device {
167 struct kref kref;
168 struct list_head list;
169 spinlock_t mad_agent_lock;
170 struct ib_device *ib_device;
171 u8 ack_delay;
172 int going_down;
173 struct cm_port *port[];
174 };
175
176 struct cm_av {
177 struct cm_port *port;
178 struct rdma_ah_attr ah_attr;
179 u16 dlid_datapath;
180 u16 pkey_index;
181 u8 timeout;
182 };
183
184 struct cm_work {
185 struct delayed_work work;
186 struct list_head list;
187 struct cm_port *port;
188 struct ib_mad_recv_wc *mad_recv_wc; /* Received MADs */
189 __be32 local_id; /* Established / timewait */
190 __be32 remote_id;
191 struct ib_cm_event cm_event;
192 struct sa_path_rec path[];
193 };
194
195 struct cm_timewait_info {
196 struct cm_work work;
197 struct list_head list;
198 struct rb_node remote_qp_node;
199 struct rb_node remote_id_node;
200 __be64 remote_ca_guid;
201 __be32 remote_qpn;
202 u8 inserted_remote_qp;
203 u8 inserted_remote_id;
204 };
205
206 struct cm_id_private {
207 struct ib_cm_id id;
208
209 struct rb_node service_node;
210 struct rb_node sidr_id_node;
211 u32 sidr_slid;
212 spinlock_t lock; /* Do not acquire inside cm.lock */
213 struct completion comp;
214 refcount_t refcount;
215 /* Number of clients sharing this ib_cm_id. Only valid for listeners.
216 * Protected by the cm.lock spinlock.
217 */
218 int listen_sharecount;
219 struct rcu_head rcu;
220
221 struct ib_mad_send_buf *msg;
222 struct cm_timewait_info *timewait_info;
223 /* todo: use alternate port on send failure */
224 struct cm_av av;
225 struct cm_av alt_av;
226
227 void *private_data;
228 __be64 tid;
229 __be32 local_qpn;
230 __be32 remote_qpn;
231 enum ib_qp_type qp_type;
232 __be32 sq_psn;
233 __be32 rq_psn;
234 int timeout_ms;
235 enum ib_mtu path_mtu;
236 __be16 pkey;
237 u8 private_data_len;
238 u8 max_cm_retries;
239 u8 responder_resources;
240 u8 initiator_depth;
241 u8 retry_count;
242 u8 rnr_retry_count;
243 u8 service_timeout;
244 u8 target_ack_delay;
245
246 struct list_head work_list;
247 atomic_t work_count;
248
249 struct rdma_ucm_ece ece;
250 };
251
cm_dev_release(struct kref * kref)252 static void cm_dev_release(struct kref *kref)
253 {
254 struct cm_device *cm_dev = container_of(kref, struct cm_device, kref);
255 u32 i;
256
257 rdma_for_each_port(cm_dev->ib_device, i)
258 kfree(cm_dev->port[i - 1]);
259
260 kfree(cm_dev);
261 }
262
cm_device_put(struct cm_device * cm_dev)263 static void cm_device_put(struct cm_device *cm_dev)
264 {
265 kref_put(&cm_dev->kref, cm_dev_release);
266 }
267
268 static void cm_work_handler(struct work_struct *work);
269
cm_deref_id(struct cm_id_private * cm_id_priv)270 static inline void cm_deref_id(struct cm_id_private *cm_id_priv)
271 {
272 if (refcount_dec_and_test(&cm_id_priv->refcount))
273 complete(&cm_id_priv->comp);
274 }
275
cm_alloc_msg(struct cm_id_private * cm_id_priv)276 static struct ib_mad_send_buf *cm_alloc_msg(struct cm_id_private *cm_id_priv)
277 {
278 struct ib_mad_agent *mad_agent;
279 struct ib_mad_send_buf *m;
280 struct ib_ah *ah;
281
282 lockdep_assert_held(&cm_id_priv->lock);
283
284 if (!cm_id_priv->av.port)
285 return ERR_PTR(-EINVAL);
286
287 spin_lock(&cm_id_priv->av.port->cm_dev->mad_agent_lock);
288 mad_agent = cm_id_priv->av.port->mad_agent;
289 if (!mad_agent) {
290 m = ERR_PTR(-EINVAL);
291 goto out;
292 }
293
294 ah = rdma_create_ah(mad_agent->qp->pd, &cm_id_priv->av.ah_attr, 0);
295 if (IS_ERR(ah)) {
296 m = ERR_CAST(ah);
297 goto out;
298 }
299
300 m = ib_create_send_mad(mad_agent, cm_id_priv->id.remote_cm_qpn,
301 cm_id_priv->av.pkey_index,
302 0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
303 GFP_ATOMIC,
304 IB_MGMT_BASE_VERSION);
305 if (IS_ERR(m)) {
306 rdma_destroy_ah(ah, 0);
307 goto out;
308 }
309
310 /* Timeout set by caller if response is expected. */
311 m->ah = ah;
312 m->retries = cm_id_priv->max_cm_retries;
313
314 refcount_inc(&cm_id_priv->refcount);
315 m->context[0] = cm_id_priv;
316
317 out:
318 spin_unlock(&cm_id_priv->av.port->cm_dev->mad_agent_lock);
319 return m;
320 }
321
cm_free_msg(struct ib_mad_send_buf * msg)322 static void cm_free_msg(struct ib_mad_send_buf *msg)
323 {
324 struct cm_id_private *cm_id_priv = msg->context[0];
325
326 if (msg->ah)
327 rdma_destroy_ah(msg->ah, 0);
328 cm_deref_id(cm_id_priv);
329 ib_free_send_mad(msg);
330 }
331
332 static struct ib_mad_send_buf *
cm_alloc_priv_msg(struct cm_id_private * cm_id_priv)333 cm_alloc_priv_msg(struct cm_id_private *cm_id_priv)
334 {
335 struct ib_mad_send_buf *msg;
336
337 lockdep_assert_held(&cm_id_priv->lock);
338
339 msg = cm_alloc_msg(cm_id_priv);
340 if (IS_ERR(msg))
341 return msg;
342 cm_id_priv->msg = msg;
343 return msg;
344 }
345
cm_free_priv_msg(struct ib_mad_send_buf * msg)346 static void cm_free_priv_msg(struct ib_mad_send_buf *msg)
347 {
348 struct cm_id_private *cm_id_priv = msg->context[0];
349
350 lockdep_assert_held(&cm_id_priv->lock);
351
352 if (!WARN_ON(cm_id_priv->msg != msg))
353 cm_id_priv->msg = NULL;
354
355 if (msg->ah)
356 rdma_destroy_ah(msg->ah, 0);
357 cm_deref_id(cm_id_priv);
358 ib_free_send_mad(msg);
359 }
360
cm_alloc_response_msg_no_ah(struct cm_port * port,struct ib_mad_recv_wc * mad_recv_wc)361 static struct ib_mad_send_buf *cm_alloc_response_msg_no_ah(struct cm_port *port,
362 struct ib_mad_recv_wc *mad_recv_wc)
363 {
364 return ib_create_send_mad(port->mad_agent, 1, mad_recv_wc->wc->pkey_index,
365 0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
366 GFP_ATOMIC,
367 IB_MGMT_BASE_VERSION);
368 }
369
cm_create_response_msg_ah(struct cm_port * port,struct ib_mad_recv_wc * mad_recv_wc,struct ib_mad_send_buf * msg)370 static int cm_create_response_msg_ah(struct cm_port *port,
371 struct ib_mad_recv_wc *mad_recv_wc,
372 struct ib_mad_send_buf *msg)
373 {
374 struct ib_ah *ah;
375
376 ah = ib_create_ah_from_wc(port->mad_agent->qp->pd, mad_recv_wc->wc,
377 mad_recv_wc->recv_buf.grh, port->port_num);
378 if (IS_ERR(ah))
379 return PTR_ERR(ah);
380
381 msg->ah = ah;
382 return 0;
383 }
384
cm_alloc_response_msg(struct cm_port * port,struct ib_mad_recv_wc * mad_recv_wc,struct ib_mad_send_buf ** msg)385 static int cm_alloc_response_msg(struct cm_port *port,
386 struct ib_mad_recv_wc *mad_recv_wc,
387 struct ib_mad_send_buf **msg)
388 {
389 struct ib_mad_send_buf *m;
390 int ret;
391
392 m = cm_alloc_response_msg_no_ah(port, mad_recv_wc);
393 if (IS_ERR(m))
394 return PTR_ERR(m);
395
396 ret = cm_create_response_msg_ah(port, mad_recv_wc, m);
397 if (ret) {
398 ib_free_send_mad(m);
399 return ret;
400 }
401
402 *msg = m;
403 return 0;
404 }
405
cm_free_response_msg(struct ib_mad_send_buf * msg)406 static void cm_free_response_msg(struct ib_mad_send_buf *msg)
407 {
408 if (msg->ah)
409 rdma_destroy_ah(msg->ah, 0);
410 ib_free_send_mad(msg);
411 }
412
cm_copy_private_data(const void * private_data,u8 private_data_len)413 static void *cm_copy_private_data(const void *private_data, u8 private_data_len)
414 {
415 void *data;
416
417 if (!private_data || !private_data_len)
418 return NULL;
419
420 data = kmemdup(private_data, private_data_len, GFP_KERNEL);
421 if (!data)
422 return ERR_PTR(-ENOMEM);
423
424 return data;
425 }
426
cm_set_private_data(struct cm_id_private * cm_id_priv,void * private_data,u8 private_data_len)427 static void cm_set_private_data(struct cm_id_private *cm_id_priv,
428 void *private_data, u8 private_data_len)
429 {
430 if (cm_id_priv->private_data && cm_id_priv->private_data_len)
431 kfree(cm_id_priv->private_data);
432
433 cm_id_priv->private_data = private_data;
434 cm_id_priv->private_data_len = private_data_len;
435 }
436
cm_set_av_port(struct cm_av * av,struct cm_port * port)437 static void cm_set_av_port(struct cm_av *av, struct cm_port *port)
438 {
439 struct cm_port *old_port = av->port;
440
441 if (old_port == port)
442 return;
443
444 av->port = port;
445 if (old_port)
446 cm_device_put(old_port->cm_dev);
447 if (port)
448 kref_get(&port->cm_dev->kref);
449 }
450
cm_init_av_for_lap(struct cm_port * port,struct ib_wc * wc,struct rdma_ah_attr * ah_attr,struct cm_av * av)451 static void cm_init_av_for_lap(struct cm_port *port, struct ib_wc *wc,
452 struct rdma_ah_attr *ah_attr, struct cm_av *av)
453 {
454 cm_set_av_port(av, port);
455 av->pkey_index = wc->pkey_index;
456 rdma_move_ah_attr(&av->ah_attr, ah_attr);
457 }
458
cm_init_av_for_response(struct cm_port * port,struct ib_wc * wc,struct ib_grh * grh,struct cm_av * av)459 static int cm_init_av_for_response(struct cm_port *port, struct ib_wc *wc,
460 struct ib_grh *grh, struct cm_av *av)
461 {
462 cm_set_av_port(av, port);
463 av->pkey_index = wc->pkey_index;
464 return ib_init_ah_attr_from_wc(port->cm_dev->ib_device,
465 port->port_num, wc,
466 grh, &av->ah_attr);
467 }
468
469 static struct cm_port *
get_cm_port_from_path(struct sa_path_rec * path,const struct ib_gid_attr * attr)470 get_cm_port_from_path(struct sa_path_rec *path, const struct ib_gid_attr *attr)
471 {
472 struct cm_device *cm_dev;
473 struct cm_port *port = NULL;
474 unsigned long flags;
475
476 if (attr) {
477 read_lock_irqsave(&cm.device_lock, flags);
478 list_for_each_entry(cm_dev, &cm.device_list, list) {
479 if (cm_dev->ib_device == attr->device) {
480 port = cm_dev->port[attr->port_num - 1];
481 break;
482 }
483 }
484 read_unlock_irqrestore(&cm.device_lock, flags);
485 } else {
486 /* SGID attribute can be NULL in following
487 * conditions.
488 * (a) Alternative path
489 * (b) IB link layer without GRH
490 * (c) LAP send messages
491 */
492 read_lock_irqsave(&cm.device_lock, flags);
493 list_for_each_entry(cm_dev, &cm.device_list, list) {
494 attr = rdma_find_gid(cm_dev->ib_device,
495 &path->sgid,
496 sa_conv_pathrec_to_gid_type(path),
497 NULL);
498 if (!IS_ERR(attr)) {
499 port = cm_dev->port[attr->port_num - 1];
500 break;
501 }
502 }
503 read_unlock_irqrestore(&cm.device_lock, flags);
504 if (port)
505 rdma_put_gid_attr(attr);
506 }
507 return port;
508 }
509
cm_init_av_by_path(struct sa_path_rec * path,const struct ib_gid_attr * sgid_attr,struct cm_av * av)510 static int cm_init_av_by_path(struct sa_path_rec *path,
511 const struct ib_gid_attr *sgid_attr,
512 struct cm_av *av)
513 {
514 struct rdma_ah_attr new_ah_attr;
515 struct cm_device *cm_dev;
516 struct cm_port *port;
517 int ret;
518
519 port = get_cm_port_from_path(path, sgid_attr);
520 if (!port)
521 return -EINVAL;
522 cm_dev = port->cm_dev;
523
524 ret = ib_find_cached_pkey(cm_dev->ib_device, port->port_num,
525 be16_to_cpu(path->pkey), &av->pkey_index);
526 if (ret)
527 return ret;
528
529 cm_set_av_port(av, port);
530
531 /*
532 * av->ah_attr might be initialized based on wc or during
533 * request processing time which might have reference to sgid_attr.
534 * So initialize a new ah_attr on stack.
535 * If initialization fails, old ah_attr is used for sending any
536 * responses. If initialization is successful, than new ah_attr
537 * is used by overwriting the old one. So that right ah_attr
538 * can be used to return an error response.
539 */
540 ret = ib_init_ah_attr_from_path(cm_dev->ib_device, port->port_num, path,
541 &new_ah_attr, sgid_attr);
542 if (ret)
543 return ret;
544
545 av->timeout = path->packet_life_time + 1;
546 rdma_move_ah_attr(&av->ah_attr, &new_ah_attr);
547 return 0;
548 }
549
550 /* Move av created by cm_init_av_by_path(), so av.dgid is not moved */
cm_move_av_from_path(struct cm_av * dest,struct cm_av * src)551 static void cm_move_av_from_path(struct cm_av *dest, struct cm_av *src)
552 {
553 cm_set_av_port(dest, src->port);
554 cm_set_av_port(src, NULL);
555 dest->pkey_index = src->pkey_index;
556 rdma_move_ah_attr(&dest->ah_attr, &src->ah_attr);
557 dest->timeout = src->timeout;
558 }
559
cm_destroy_av(struct cm_av * av)560 static void cm_destroy_av(struct cm_av *av)
561 {
562 rdma_destroy_ah_attr(&av->ah_attr);
563 cm_set_av_port(av, NULL);
564 }
565
cm_local_id(__be32 local_id)566 static u32 cm_local_id(__be32 local_id)
567 {
568 return (__force u32) (local_id ^ cm.random_id_operand);
569 }
570
cm_acquire_id(__be32 local_id,__be32 remote_id)571 static struct cm_id_private *cm_acquire_id(__be32 local_id, __be32 remote_id)
572 {
573 struct cm_id_private *cm_id_priv;
574
575 rcu_read_lock();
576 cm_id_priv = xa_load(&cm.local_id_table, cm_local_id(local_id));
577 if (!cm_id_priv || cm_id_priv->id.remote_id != remote_id ||
578 !refcount_inc_not_zero(&cm_id_priv->refcount))
579 cm_id_priv = NULL;
580 rcu_read_unlock();
581
582 return cm_id_priv;
583 }
584
585 /*
586 * Trivial helpers to strip endian annotation and compare; the
587 * endianness doesn't actually matter since we just need a stable
588 * order for the RB tree.
589 */
be32_lt(__be32 a,__be32 b)590 static int be32_lt(__be32 a, __be32 b)
591 {
592 return (__force u32) a < (__force u32) b;
593 }
594
be32_gt(__be32 a,__be32 b)595 static int be32_gt(__be32 a, __be32 b)
596 {
597 return (__force u32) a > (__force u32) b;
598 }
599
be64_lt(__be64 a,__be64 b)600 static int be64_lt(__be64 a, __be64 b)
601 {
602 return (__force u64) a < (__force u64) b;
603 }
604
be64_gt(__be64 a,__be64 b)605 static int be64_gt(__be64 a, __be64 b)
606 {
607 return (__force u64) a > (__force u64) b;
608 }
609
610 /*
611 * Inserts a new cm_id_priv into the listen_service_table. Returns cm_id_priv
612 * if the new ID was inserted, NULL if it could not be inserted due to a
613 * collision, or the existing cm_id_priv ready for shared usage.
614 */
cm_insert_listen(struct cm_id_private * cm_id_priv,ib_cm_handler shared_handler)615 static struct cm_id_private *cm_insert_listen(struct cm_id_private *cm_id_priv,
616 ib_cm_handler shared_handler)
617 {
618 struct rb_node **link = &cm.listen_service_table.rb_node;
619 struct rb_node *parent = NULL;
620 struct cm_id_private *cur_cm_id_priv;
621 __be64 service_id = cm_id_priv->id.service_id;
622 unsigned long flags;
623
624 spin_lock_irqsave(&cm.lock, flags);
625 while (*link) {
626 parent = *link;
627 cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
628 service_node);
629
630 if (cm_id_priv->id.device < cur_cm_id_priv->id.device)
631 link = &(*link)->rb_left;
632 else if (cm_id_priv->id.device > cur_cm_id_priv->id.device)
633 link = &(*link)->rb_right;
634 else if (be64_lt(service_id, cur_cm_id_priv->id.service_id))
635 link = &(*link)->rb_left;
636 else if (be64_gt(service_id, cur_cm_id_priv->id.service_id))
637 link = &(*link)->rb_right;
638 else {
639 /*
640 * Sharing an ib_cm_id with different handlers is not
641 * supported
642 */
643 if (cur_cm_id_priv->id.cm_handler != shared_handler ||
644 cur_cm_id_priv->id.context ||
645 WARN_ON(!cur_cm_id_priv->id.cm_handler)) {
646 spin_unlock_irqrestore(&cm.lock, flags);
647 return NULL;
648 }
649 refcount_inc(&cur_cm_id_priv->refcount);
650 cur_cm_id_priv->listen_sharecount++;
651 spin_unlock_irqrestore(&cm.lock, flags);
652 return cur_cm_id_priv;
653 }
654 }
655 cm_id_priv->listen_sharecount++;
656 rb_link_node(&cm_id_priv->service_node, parent, link);
657 rb_insert_color(&cm_id_priv->service_node, &cm.listen_service_table);
658 spin_unlock_irqrestore(&cm.lock, flags);
659 return cm_id_priv;
660 }
661
cm_find_listen(struct ib_device * device,__be64 service_id)662 static struct cm_id_private *cm_find_listen(struct ib_device *device,
663 __be64 service_id)
664 {
665 struct rb_node *node = cm.listen_service_table.rb_node;
666 struct cm_id_private *cm_id_priv;
667
668 while (node) {
669 cm_id_priv = rb_entry(node, struct cm_id_private, service_node);
670
671 if (device < cm_id_priv->id.device)
672 node = node->rb_left;
673 else if (device > cm_id_priv->id.device)
674 node = node->rb_right;
675 else if (be64_lt(service_id, cm_id_priv->id.service_id))
676 node = node->rb_left;
677 else if (be64_gt(service_id, cm_id_priv->id.service_id))
678 node = node->rb_right;
679 else {
680 refcount_inc(&cm_id_priv->refcount);
681 return cm_id_priv;
682 }
683 }
684 return NULL;
685 }
686
687 static struct cm_timewait_info *
cm_insert_remote_id(struct cm_timewait_info * timewait_info)688 cm_insert_remote_id(struct cm_timewait_info *timewait_info)
689 {
690 struct rb_node **link = &cm.remote_id_table.rb_node;
691 struct rb_node *parent = NULL;
692 struct cm_timewait_info *cur_timewait_info;
693 __be64 remote_ca_guid = timewait_info->remote_ca_guid;
694 __be32 remote_id = timewait_info->work.remote_id;
695
696 while (*link) {
697 parent = *link;
698 cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
699 remote_id_node);
700 if (be32_lt(remote_id, cur_timewait_info->work.remote_id))
701 link = &(*link)->rb_left;
702 else if (be32_gt(remote_id, cur_timewait_info->work.remote_id))
703 link = &(*link)->rb_right;
704 else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
705 link = &(*link)->rb_left;
706 else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
707 link = &(*link)->rb_right;
708 else
709 return cur_timewait_info;
710 }
711 timewait_info->inserted_remote_id = 1;
712 rb_link_node(&timewait_info->remote_id_node, parent, link);
713 rb_insert_color(&timewait_info->remote_id_node, &cm.remote_id_table);
714 return NULL;
715 }
716
cm_find_remote_id(__be64 remote_ca_guid,__be32 remote_id)717 static struct cm_id_private *cm_find_remote_id(__be64 remote_ca_guid,
718 __be32 remote_id)
719 {
720 struct rb_node *node = cm.remote_id_table.rb_node;
721 struct cm_timewait_info *timewait_info;
722 struct cm_id_private *res = NULL;
723
724 spin_lock_irq(&cm.lock);
725 while (node) {
726 timewait_info = rb_entry(node, struct cm_timewait_info,
727 remote_id_node);
728 if (be32_lt(remote_id, timewait_info->work.remote_id))
729 node = node->rb_left;
730 else if (be32_gt(remote_id, timewait_info->work.remote_id))
731 node = node->rb_right;
732 else if (be64_lt(remote_ca_guid, timewait_info->remote_ca_guid))
733 node = node->rb_left;
734 else if (be64_gt(remote_ca_guid, timewait_info->remote_ca_guid))
735 node = node->rb_right;
736 else {
737 res = cm_acquire_id(timewait_info->work.local_id,
738 timewait_info->work.remote_id);
739 break;
740 }
741 }
742 spin_unlock_irq(&cm.lock);
743 return res;
744 }
745
746 static struct cm_timewait_info *
cm_insert_remote_qpn(struct cm_timewait_info * timewait_info)747 cm_insert_remote_qpn(struct cm_timewait_info *timewait_info)
748 {
749 struct rb_node **link = &cm.remote_qp_table.rb_node;
750 struct rb_node *parent = NULL;
751 struct cm_timewait_info *cur_timewait_info;
752 __be64 remote_ca_guid = timewait_info->remote_ca_guid;
753 __be32 remote_qpn = timewait_info->remote_qpn;
754
755 while (*link) {
756 parent = *link;
757 cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
758 remote_qp_node);
759 if (be32_lt(remote_qpn, cur_timewait_info->remote_qpn))
760 link = &(*link)->rb_left;
761 else if (be32_gt(remote_qpn, cur_timewait_info->remote_qpn))
762 link = &(*link)->rb_right;
763 else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
764 link = &(*link)->rb_left;
765 else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
766 link = &(*link)->rb_right;
767 else
768 return cur_timewait_info;
769 }
770 timewait_info->inserted_remote_qp = 1;
771 rb_link_node(&timewait_info->remote_qp_node, parent, link);
772 rb_insert_color(&timewait_info->remote_qp_node, &cm.remote_qp_table);
773 return NULL;
774 }
775
776 static struct cm_id_private *
cm_insert_remote_sidr(struct cm_id_private * cm_id_priv)777 cm_insert_remote_sidr(struct cm_id_private *cm_id_priv)
778 {
779 struct rb_node **link = &cm.remote_sidr_table.rb_node;
780 struct rb_node *parent = NULL;
781 struct cm_id_private *cur_cm_id_priv;
782 __be32 remote_id = cm_id_priv->id.remote_id;
783
784 while (*link) {
785 parent = *link;
786 cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
787 sidr_id_node);
788 if (be32_lt(remote_id, cur_cm_id_priv->id.remote_id))
789 link = &(*link)->rb_left;
790 else if (be32_gt(remote_id, cur_cm_id_priv->id.remote_id))
791 link = &(*link)->rb_right;
792 else {
793 if (cur_cm_id_priv->sidr_slid < cm_id_priv->sidr_slid)
794 link = &(*link)->rb_left;
795 else if (cur_cm_id_priv->sidr_slid > cm_id_priv->sidr_slid)
796 link = &(*link)->rb_right;
797 else
798 return cur_cm_id_priv;
799 }
800 }
801 rb_link_node(&cm_id_priv->sidr_id_node, parent, link);
802 rb_insert_color(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
803 return NULL;
804 }
805
cm_alloc_id_priv(struct ib_device * device,ib_cm_handler cm_handler,void * context)806 static struct cm_id_private *cm_alloc_id_priv(struct ib_device *device,
807 ib_cm_handler cm_handler,
808 void *context)
809 {
810 struct cm_id_private *cm_id_priv;
811 u32 id;
812 int ret;
813
814 cm_id_priv = kzalloc(sizeof *cm_id_priv, GFP_KERNEL);
815 if (!cm_id_priv)
816 return ERR_PTR(-ENOMEM);
817
818 cm_id_priv->id.state = IB_CM_IDLE;
819 cm_id_priv->id.device = device;
820 cm_id_priv->id.cm_handler = cm_handler;
821 cm_id_priv->id.context = context;
822 cm_id_priv->id.remote_cm_qpn = 1;
823
824 RB_CLEAR_NODE(&cm_id_priv->service_node);
825 RB_CLEAR_NODE(&cm_id_priv->sidr_id_node);
826 spin_lock_init(&cm_id_priv->lock);
827 init_completion(&cm_id_priv->comp);
828 INIT_LIST_HEAD(&cm_id_priv->work_list);
829 atomic_set(&cm_id_priv->work_count, -1);
830 refcount_set(&cm_id_priv->refcount, 1);
831
832 ret = xa_alloc_cyclic(&cm.local_id_table, &id, NULL, xa_limit_32b,
833 &cm.local_id_next, GFP_KERNEL);
834 if (ret < 0)
835 goto error;
836 cm_id_priv->id.local_id = (__force __be32)id ^ cm.random_id_operand;
837
838 return cm_id_priv;
839
840 error:
841 kfree(cm_id_priv);
842 return ERR_PTR(ret);
843 }
844
845 /*
846 * Make the ID visible to the MAD handlers and other threads that use the
847 * xarray.
848 */
cm_finalize_id(struct cm_id_private * cm_id_priv)849 static void cm_finalize_id(struct cm_id_private *cm_id_priv)
850 {
851 xa_store(&cm.local_id_table, cm_local_id(cm_id_priv->id.local_id),
852 cm_id_priv, GFP_ATOMIC);
853 }
854
ib_create_cm_id(struct ib_device * device,ib_cm_handler cm_handler,void * context)855 struct ib_cm_id *ib_create_cm_id(struct ib_device *device,
856 ib_cm_handler cm_handler,
857 void *context)
858 {
859 struct cm_id_private *cm_id_priv;
860
861 cm_id_priv = cm_alloc_id_priv(device, cm_handler, context);
862 if (IS_ERR(cm_id_priv))
863 return ERR_CAST(cm_id_priv);
864
865 cm_finalize_id(cm_id_priv);
866 return &cm_id_priv->id;
867 }
868 EXPORT_SYMBOL(ib_create_cm_id);
869
cm_dequeue_work(struct cm_id_private * cm_id_priv)870 static struct cm_work *cm_dequeue_work(struct cm_id_private *cm_id_priv)
871 {
872 struct cm_work *work;
873
874 if (list_empty(&cm_id_priv->work_list))
875 return NULL;
876
877 work = list_entry(cm_id_priv->work_list.next, struct cm_work, list);
878 list_del(&work->list);
879 return work;
880 }
881
cm_free_work(struct cm_work * work)882 static void cm_free_work(struct cm_work *work)
883 {
884 if (work->mad_recv_wc)
885 ib_free_recv_mad(work->mad_recv_wc);
886 kfree(work);
887 }
888
cm_queue_work_unlock(struct cm_id_private * cm_id_priv,struct cm_work * work)889 static void cm_queue_work_unlock(struct cm_id_private *cm_id_priv,
890 struct cm_work *work)
891 __releases(&cm_id_priv->lock)
892 {
893 bool immediate;
894
895 /*
896 * To deliver the event to the user callback we have the drop the
897 * spinlock, however, we need to ensure that the user callback is single
898 * threaded and receives events in the temporal order. If there are
899 * already events being processed then thread new events onto a list,
900 * the thread currently processing will pick them up.
901 */
902 immediate = atomic_inc_and_test(&cm_id_priv->work_count);
903 if (!immediate) {
904 list_add_tail(&work->list, &cm_id_priv->work_list);
905 /*
906 * This routine always consumes incoming reference. Once queued
907 * to the work_list then a reference is held by the thread
908 * currently running cm_process_work() and this reference is not
909 * needed.
910 */
911 cm_deref_id(cm_id_priv);
912 }
913 spin_unlock_irq(&cm_id_priv->lock);
914
915 if (immediate)
916 cm_process_work(cm_id_priv, work);
917 }
918
cm_convert_to_ms(int iba_time)919 static inline int cm_convert_to_ms(int iba_time)
920 {
921 /* approximate conversion to ms from 4.096us x 2^iba_time */
922 return 1 << max(iba_time - 8, 0);
923 }
924
925 /*
926 * calculate: 4.096x2^ack_timeout = 4.096x2^ack_delay + 2x4.096x2^life_time
927 * Because of how ack_timeout is stored, adding one doubles the timeout.
928 * To avoid large timeouts, select the max(ack_delay, life_time + 1), and
929 * increment it (round up) only if the other is within 50%.
930 */
cm_ack_timeout(u8 ca_ack_delay,u8 packet_life_time)931 static u8 cm_ack_timeout(u8 ca_ack_delay, u8 packet_life_time)
932 {
933 int ack_timeout = packet_life_time + 1;
934
935 if (ack_timeout >= ca_ack_delay)
936 ack_timeout += (ca_ack_delay >= (ack_timeout - 1));
937 else
938 ack_timeout = ca_ack_delay +
939 (ack_timeout >= (ca_ack_delay - 1));
940
941 return min(31, ack_timeout);
942 }
943
cm_remove_remote(struct cm_id_private * cm_id_priv)944 static void cm_remove_remote(struct cm_id_private *cm_id_priv)
945 {
946 struct cm_timewait_info *timewait_info = cm_id_priv->timewait_info;
947
948 if (timewait_info->inserted_remote_id) {
949 rb_erase(&timewait_info->remote_id_node, &cm.remote_id_table);
950 timewait_info->inserted_remote_id = 0;
951 }
952
953 if (timewait_info->inserted_remote_qp) {
954 rb_erase(&timewait_info->remote_qp_node, &cm.remote_qp_table);
955 timewait_info->inserted_remote_qp = 0;
956 }
957 }
958
cm_create_timewait_info(__be32 local_id)959 static struct cm_timewait_info *cm_create_timewait_info(__be32 local_id)
960 {
961 struct cm_timewait_info *timewait_info;
962
963 timewait_info = kzalloc(sizeof *timewait_info, GFP_KERNEL);
964 if (!timewait_info)
965 return ERR_PTR(-ENOMEM);
966
967 timewait_info->work.local_id = local_id;
968 INIT_DELAYED_WORK(&timewait_info->work.work, cm_work_handler);
969 timewait_info->work.cm_event.event = IB_CM_TIMEWAIT_EXIT;
970 return timewait_info;
971 }
972
cm_enter_timewait(struct cm_id_private * cm_id_priv)973 static void cm_enter_timewait(struct cm_id_private *cm_id_priv)
974 {
975 int wait_time;
976 unsigned long flags;
977 struct cm_device *cm_dev;
978
979 lockdep_assert_held(&cm_id_priv->lock);
980
981 cm_dev = ib_get_client_data(cm_id_priv->id.device, &cm_client);
982 if (!cm_dev)
983 return;
984
985 spin_lock_irqsave(&cm.lock, flags);
986 cm_remove_remote(cm_id_priv);
987 list_add_tail(&cm_id_priv->timewait_info->list, &cm.timewait_list);
988 spin_unlock_irqrestore(&cm.lock, flags);
989
990 /*
991 * The cm_id could be destroyed by the user before we exit timewait.
992 * To protect against this, we search for the cm_id after exiting
993 * timewait before notifying the user that we've exited timewait.
994 */
995 cm_id_priv->id.state = IB_CM_TIMEWAIT;
996 wait_time = cm_convert_to_ms(cm_id_priv->av.timeout);
997
998 /* Check if the device started its remove_one */
999 spin_lock_irqsave(&cm.lock, flags);
1000 if (!cm_dev->going_down)
1001 queue_delayed_work(cm.wq, &cm_id_priv->timewait_info->work.work,
1002 msecs_to_jiffies(wait_time));
1003 spin_unlock_irqrestore(&cm.lock, flags);
1004
1005 /*
1006 * The timewait_info is converted into a work and gets freed during
1007 * cm_free_work() in cm_timewait_handler().
1008 */
1009 BUILD_BUG_ON(offsetof(struct cm_timewait_info, work) != 0);
1010 cm_id_priv->timewait_info = NULL;
1011 }
1012
cm_reset_to_idle(struct cm_id_private * cm_id_priv)1013 static void cm_reset_to_idle(struct cm_id_private *cm_id_priv)
1014 {
1015 unsigned long flags;
1016
1017 lockdep_assert_held(&cm_id_priv->lock);
1018
1019 cm_id_priv->id.state = IB_CM_IDLE;
1020 if (cm_id_priv->timewait_info) {
1021 spin_lock_irqsave(&cm.lock, flags);
1022 cm_remove_remote(cm_id_priv);
1023 spin_unlock_irqrestore(&cm.lock, flags);
1024 kfree(cm_id_priv->timewait_info);
1025 cm_id_priv->timewait_info = NULL;
1026 }
1027 }
1028
cm_destroy_id_wait_timeout(struct ib_cm_id * cm_id,enum ib_cm_state old_state)1029 static noinline void cm_destroy_id_wait_timeout(struct ib_cm_id *cm_id,
1030 enum ib_cm_state old_state)
1031 {
1032 struct cm_id_private *cm_id_priv;
1033
1034 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1035 pr_err("%s: cm_id=%p timed out. state %d -> %d, refcnt=%d\n", __func__,
1036 cm_id, old_state, cm_id->state, refcount_read(&cm_id_priv->refcount));
1037 }
1038
cm_destroy_id(struct ib_cm_id * cm_id,int err)1039 static void cm_destroy_id(struct ib_cm_id *cm_id, int err)
1040 {
1041 struct cm_id_private *cm_id_priv;
1042 enum ib_cm_state old_state;
1043 struct cm_work *work;
1044 int ret;
1045
1046 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1047 spin_lock_irq(&cm_id_priv->lock);
1048 old_state = cm_id->state;
1049 retest:
1050 switch (cm_id->state) {
1051 case IB_CM_LISTEN:
1052 spin_lock(&cm.lock);
1053 if (--cm_id_priv->listen_sharecount > 0) {
1054 /* The id is still shared. */
1055 WARN_ON(refcount_read(&cm_id_priv->refcount) == 1);
1056 spin_unlock(&cm.lock);
1057 spin_unlock_irq(&cm_id_priv->lock);
1058 cm_deref_id(cm_id_priv);
1059 return;
1060 }
1061 cm_id->state = IB_CM_IDLE;
1062 rb_erase(&cm_id_priv->service_node, &cm.listen_service_table);
1063 RB_CLEAR_NODE(&cm_id_priv->service_node);
1064 spin_unlock(&cm.lock);
1065 break;
1066 case IB_CM_SIDR_REQ_SENT:
1067 cm_id->state = IB_CM_IDLE;
1068 ib_cancel_mad(cm_id_priv->msg);
1069 break;
1070 case IB_CM_SIDR_REQ_RCVD:
1071 cm_send_sidr_rep_locked(cm_id_priv,
1072 &(struct ib_cm_sidr_rep_param){
1073 .status = IB_SIDR_REJECT });
1074 /* cm_send_sidr_rep_locked will not move to IDLE if it fails */
1075 cm_id->state = IB_CM_IDLE;
1076 break;
1077 case IB_CM_REQ_SENT:
1078 case IB_CM_MRA_REQ_RCVD:
1079 ib_cancel_mad(cm_id_priv->msg);
1080 cm_send_rej_locked(cm_id_priv, IB_CM_REJ_TIMEOUT,
1081 &cm_id_priv->id.device->node_guid,
1082 sizeof(cm_id_priv->id.device->node_guid),
1083 NULL, 0);
1084 break;
1085 case IB_CM_REQ_RCVD:
1086 if (err == -ENOMEM) {
1087 /* Do not reject to allow future retries. */
1088 cm_reset_to_idle(cm_id_priv);
1089 } else {
1090 cm_send_rej_locked(cm_id_priv,
1091 IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
1092 NULL, 0);
1093 }
1094 break;
1095 case IB_CM_REP_SENT:
1096 case IB_CM_MRA_REP_RCVD:
1097 ib_cancel_mad(cm_id_priv->msg);
1098 cm_send_rej_locked(cm_id_priv, IB_CM_REJ_CONSUMER_DEFINED, NULL,
1099 0, NULL, 0);
1100 goto retest;
1101 case IB_CM_MRA_REQ_SENT:
1102 case IB_CM_REP_RCVD:
1103 case IB_CM_MRA_REP_SENT:
1104 cm_send_rej_locked(cm_id_priv, IB_CM_REJ_CONSUMER_DEFINED, NULL,
1105 0, NULL, 0);
1106 break;
1107 case IB_CM_ESTABLISHED:
1108 if (cm_id_priv->qp_type == IB_QPT_XRC_TGT) {
1109 cm_id->state = IB_CM_IDLE;
1110 break;
1111 }
1112 cm_send_dreq_locked(cm_id_priv, NULL, 0);
1113 goto retest;
1114 case IB_CM_DREQ_SENT:
1115 ib_cancel_mad(cm_id_priv->msg);
1116 cm_enter_timewait(cm_id_priv);
1117 goto retest;
1118 case IB_CM_DREQ_RCVD:
1119 cm_send_drep_locked(cm_id_priv, NULL, 0);
1120 WARN_ON(cm_id->state != IB_CM_TIMEWAIT);
1121 goto retest;
1122 case IB_CM_TIMEWAIT:
1123 /*
1124 * The cm_acquire_id in cm_timewait_handler will stop working
1125 * once we do xa_erase below, so just move to idle here for
1126 * consistency.
1127 */
1128 cm_id->state = IB_CM_IDLE;
1129 break;
1130 case IB_CM_IDLE:
1131 break;
1132 }
1133 WARN_ON(cm_id->state != IB_CM_IDLE);
1134
1135 spin_lock(&cm.lock);
1136 /* Required for cleanup paths related cm_req_handler() */
1137 if (cm_id_priv->timewait_info) {
1138 cm_remove_remote(cm_id_priv);
1139 kfree(cm_id_priv->timewait_info);
1140 cm_id_priv->timewait_info = NULL;
1141 }
1142
1143 WARN_ON(cm_id_priv->listen_sharecount);
1144 WARN_ON(!RB_EMPTY_NODE(&cm_id_priv->service_node));
1145 if (!RB_EMPTY_NODE(&cm_id_priv->sidr_id_node))
1146 rb_erase(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
1147 spin_unlock(&cm.lock);
1148 spin_unlock_irq(&cm_id_priv->lock);
1149
1150 xa_erase(&cm.local_id_table, cm_local_id(cm_id->local_id));
1151 cm_deref_id(cm_id_priv);
1152 do {
1153 ret = wait_for_completion_timeout(&cm_id_priv->comp,
1154 msecs_to_jiffies(
1155 CM_DESTROY_ID_WAIT_TIMEOUT));
1156 if (!ret) /* timeout happened */
1157 cm_destroy_id_wait_timeout(cm_id, old_state);
1158 } while (!ret);
1159
1160 while ((work = cm_dequeue_work(cm_id_priv)) != NULL)
1161 cm_free_work(work);
1162
1163 cm_destroy_av(&cm_id_priv->av);
1164 cm_destroy_av(&cm_id_priv->alt_av);
1165 kfree(cm_id_priv->private_data);
1166 kfree_rcu(cm_id_priv, rcu);
1167 }
1168
ib_destroy_cm_id(struct ib_cm_id * cm_id)1169 void ib_destroy_cm_id(struct ib_cm_id *cm_id)
1170 {
1171 cm_destroy_id(cm_id, 0);
1172 }
1173 EXPORT_SYMBOL(ib_destroy_cm_id);
1174
cm_init_listen(struct cm_id_private * cm_id_priv,__be64 service_id)1175 static int cm_init_listen(struct cm_id_private *cm_id_priv, __be64 service_id)
1176 {
1177 if ((service_id & IB_SERVICE_ID_AGN_MASK) == IB_CM_ASSIGN_SERVICE_ID &&
1178 (service_id != IB_CM_ASSIGN_SERVICE_ID))
1179 return -EINVAL;
1180
1181 if (service_id == IB_CM_ASSIGN_SERVICE_ID)
1182 cm_id_priv->id.service_id = cpu_to_be64(cm.listen_service_id++);
1183 else
1184 cm_id_priv->id.service_id = service_id;
1185
1186 return 0;
1187 }
1188
1189 /**
1190 * ib_cm_listen - Initiates listening on the specified service ID for
1191 * connection and service ID resolution requests.
1192 * @cm_id: Connection identifier associated with the listen request.
1193 * @service_id: Service identifier matched against incoming connection
1194 * and service ID resolution requests. The service ID should be specified
1195 * network-byte order. If set to IB_CM_ASSIGN_SERVICE_ID, the CM will
1196 * assign a service ID to the caller.
1197 */
ib_cm_listen(struct ib_cm_id * cm_id,__be64 service_id)1198 int ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id)
1199 {
1200 struct cm_id_private *cm_id_priv =
1201 container_of(cm_id, struct cm_id_private, id);
1202 unsigned long flags;
1203 int ret;
1204
1205 spin_lock_irqsave(&cm_id_priv->lock, flags);
1206 if (cm_id_priv->id.state != IB_CM_IDLE) {
1207 ret = -EINVAL;
1208 goto out;
1209 }
1210
1211 ret = cm_init_listen(cm_id_priv, service_id);
1212 if (ret)
1213 goto out;
1214
1215 if (!cm_insert_listen(cm_id_priv, NULL)) {
1216 ret = -EBUSY;
1217 goto out;
1218 }
1219
1220 cm_id_priv->id.state = IB_CM_LISTEN;
1221 ret = 0;
1222
1223 out:
1224 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1225 return ret;
1226 }
1227 EXPORT_SYMBOL(ib_cm_listen);
1228
1229 /**
1230 * ib_cm_insert_listen - Create a new listening ib_cm_id and listen on
1231 * the given service ID.
1232 *
1233 * If there's an existing ID listening on that same device and service ID,
1234 * return it.
1235 *
1236 * @device: Device associated with the cm_id. All related communication will
1237 * be associated with the specified device.
1238 * @cm_handler: Callback invoked to notify the user of CM events.
1239 * @service_id: Service identifier matched against incoming connection
1240 * and service ID resolution requests. The service ID should be specified
1241 * network-byte order. If set to IB_CM_ASSIGN_SERVICE_ID, the CM will
1242 * assign a service ID to the caller.
1243 *
1244 * Callers should call ib_destroy_cm_id when done with the listener ID.
1245 */
ib_cm_insert_listen(struct ib_device * device,ib_cm_handler cm_handler,__be64 service_id)1246 struct ib_cm_id *ib_cm_insert_listen(struct ib_device *device,
1247 ib_cm_handler cm_handler,
1248 __be64 service_id)
1249 {
1250 struct cm_id_private *listen_id_priv;
1251 struct cm_id_private *cm_id_priv;
1252 int err = 0;
1253
1254 /* Create an ID in advance, since the creation may sleep */
1255 cm_id_priv = cm_alloc_id_priv(device, cm_handler, NULL);
1256 if (IS_ERR(cm_id_priv))
1257 return ERR_CAST(cm_id_priv);
1258
1259 err = cm_init_listen(cm_id_priv, service_id);
1260 if (err) {
1261 ib_destroy_cm_id(&cm_id_priv->id);
1262 return ERR_PTR(err);
1263 }
1264
1265 spin_lock_irq(&cm_id_priv->lock);
1266 listen_id_priv = cm_insert_listen(cm_id_priv, cm_handler);
1267 if (listen_id_priv != cm_id_priv) {
1268 spin_unlock_irq(&cm_id_priv->lock);
1269 ib_destroy_cm_id(&cm_id_priv->id);
1270 if (!listen_id_priv)
1271 return ERR_PTR(-EINVAL);
1272 return &listen_id_priv->id;
1273 }
1274 cm_id_priv->id.state = IB_CM_LISTEN;
1275 spin_unlock_irq(&cm_id_priv->lock);
1276
1277 /*
1278 * A listen ID does not need to be in the xarray since it does not
1279 * receive mads, is not placed in the remote_id or remote_qpn rbtree,
1280 * and does not enter timewait.
1281 */
1282
1283 return &cm_id_priv->id;
1284 }
1285 EXPORT_SYMBOL(ib_cm_insert_listen);
1286
cm_form_tid(struct cm_id_private * cm_id_priv)1287 static __be64 cm_form_tid(struct cm_id_private *cm_id_priv)
1288 {
1289 u64 hi_tid = 0, low_tid;
1290
1291 lockdep_assert_held(&cm_id_priv->lock);
1292
1293 low_tid = (u64)cm_id_priv->id.local_id;
1294 if (!cm_id_priv->av.port)
1295 return cpu_to_be64(low_tid);
1296
1297 spin_lock(&cm_id_priv->av.port->cm_dev->mad_agent_lock);
1298 if (cm_id_priv->av.port->mad_agent)
1299 hi_tid = ((u64)cm_id_priv->av.port->mad_agent->hi_tid) << 32;
1300 spin_unlock(&cm_id_priv->av.port->cm_dev->mad_agent_lock);
1301 return cpu_to_be64(hi_tid | low_tid);
1302 }
1303
cm_format_mad_hdr(struct ib_mad_hdr * hdr,__be16 attr_id,__be64 tid)1304 static void cm_format_mad_hdr(struct ib_mad_hdr *hdr,
1305 __be16 attr_id, __be64 tid)
1306 {
1307 hdr->base_version = IB_MGMT_BASE_VERSION;
1308 hdr->mgmt_class = IB_MGMT_CLASS_CM;
1309 hdr->class_version = IB_CM_CLASS_VERSION;
1310 hdr->method = IB_MGMT_METHOD_SEND;
1311 hdr->attr_id = attr_id;
1312 hdr->tid = tid;
1313 }
1314
cm_format_mad_ece_hdr(struct ib_mad_hdr * hdr,__be16 attr_id,__be64 tid,u32 attr_mod)1315 static void cm_format_mad_ece_hdr(struct ib_mad_hdr *hdr, __be16 attr_id,
1316 __be64 tid, u32 attr_mod)
1317 {
1318 cm_format_mad_hdr(hdr, attr_id, tid);
1319 hdr->attr_mod = cpu_to_be32(attr_mod);
1320 }
1321
cm_format_req(struct cm_req_msg * req_msg,struct cm_id_private * cm_id_priv,struct ib_cm_req_param * param)1322 static void cm_format_req(struct cm_req_msg *req_msg,
1323 struct cm_id_private *cm_id_priv,
1324 struct ib_cm_req_param *param)
1325 {
1326 struct sa_path_rec *pri_path = param->primary_path;
1327 struct sa_path_rec *alt_path = param->alternate_path;
1328 bool pri_ext = false;
1329 __be16 lid;
1330
1331 if (pri_path->rec_type == SA_PATH_REC_TYPE_OPA)
1332 pri_ext = opa_is_extended_lid(pri_path->opa.dlid,
1333 pri_path->opa.slid);
1334
1335 cm_format_mad_ece_hdr(&req_msg->hdr, CM_REQ_ATTR_ID,
1336 cm_form_tid(cm_id_priv), param->ece.attr_mod);
1337
1338 IBA_SET(CM_REQ_LOCAL_COMM_ID, req_msg,
1339 be32_to_cpu(cm_id_priv->id.local_id));
1340 IBA_SET(CM_REQ_SERVICE_ID, req_msg, be64_to_cpu(param->service_id));
1341 IBA_SET(CM_REQ_LOCAL_CA_GUID, req_msg,
1342 be64_to_cpu(cm_id_priv->id.device->node_guid));
1343 IBA_SET(CM_REQ_LOCAL_QPN, req_msg, param->qp_num);
1344 IBA_SET(CM_REQ_INITIATOR_DEPTH, req_msg, param->initiator_depth);
1345 IBA_SET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg,
1346 param->remote_cm_response_timeout);
1347 cm_req_set_qp_type(req_msg, param->qp_type);
1348 IBA_SET(CM_REQ_END_TO_END_FLOW_CONTROL, req_msg, param->flow_control);
1349 IBA_SET(CM_REQ_STARTING_PSN, req_msg, param->starting_psn);
1350 IBA_SET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg,
1351 param->local_cm_response_timeout);
1352 IBA_SET(CM_REQ_PARTITION_KEY, req_msg,
1353 be16_to_cpu(param->primary_path->pkey));
1354 IBA_SET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg,
1355 param->primary_path->mtu);
1356 IBA_SET(CM_REQ_MAX_CM_RETRIES, req_msg, param->max_cm_retries);
1357
1358 if (param->qp_type != IB_QPT_XRC_INI) {
1359 IBA_SET(CM_REQ_RESPONDER_RESOURCES, req_msg,
1360 param->responder_resources);
1361 IBA_SET(CM_REQ_RETRY_COUNT, req_msg, param->retry_count);
1362 IBA_SET(CM_REQ_RNR_RETRY_COUNT, req_msg,
1363 param->rnr_retry_count);
1364 IBA_SET(CM_REQ_SRQ, req_msg, param->srq);
1365 }
1366
1367 *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg) =
1368 pri_path->sgid;
1369 *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg) =
1370 pri_path->dgid;
1371 if (pri_ext) {
1372 IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg)
1373 ->global.interface_id =
1374 OPA_MAKE_ID(be32_to_cpu(pri_path->opa.slid));
1375 IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg)
1376 ->global.interface_id =
1377 OPA_MAKE_ID(be32_to_cpu(pri_path->opa.dlid));
1378 }
1379 if (pri_path->hop_limit <= 1) {
1380 IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
1381 be16_to_cpu(pri_ext ? 0 :
1382 htons(ntohl(sa_path_get_slid(
1383 pri_path)))));
1384 IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,
1385 be16_to_cpu(pri_ext ? 0 :
1386 htons(ntohl(sa_path_get_dlid(
1387 pri_path)))));
1388 } else {
1389
1390 if (param->primary_path_inbound) {
1391 lid = param->primary_path_inbound->ib.dlid;
1392 IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
1393 be16_to_cpu(lid));
1394 } else
1395 IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
1396 be16_to_cpu(IB_LID_PERMISSIVE));
1397
1398 /* Work-around until there's a way to obtain remote LID info */
1399 IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,
1400 be16_to_cpu(IB_LID_PERMISSIVE));
1401 }
1402 IBA_SET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg,
1403 be32_to_cpu(pri_path->flow_label));
1404 IBA_SET(CM_REQ_PRIMARY_PACKET_RATE, req_msg, pri_path->rate);
1405 IBA_SET(CM_REQ_PRIMARY_TRAFFIC_CLASS, req_msg, pri_path->traffic_class);
1406 IBA_SET(CM_REQ_PRIMARY_HOP_LIMIT, req_msg, pri_path->hop_limit);
1407 IBA_SET(CM_REQ_PRIMARY_SL, req_msg, pri_path->sl);
1408 IBA_SET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg,
1409 (pri_path->hop_limit <= 1));
1410 IBA_SET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, req_msg,
1411 cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
1412 pri_path->packet_life_time));
1413
1414 if (alt_path) {
1415 bool alt_ext = false;
1416
1417 if (alt_path->rec_type == SA_PATH_REC_TYPE_OPA)
1418 alt_ext = opa_is_extended_lid(alt_path->opa.dlid,
1419 alt_path->opa.slid);
1420
1421 *IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg) =
1422 alt_path->sgid;
1423 *IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg) =
1424 alt_path->dgid;
1425 if (alt_ext) {
1426 IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID,
1427 req_msg)
1428 ->global.interface_id =
1429 OPA_MAKE_ID(be32_to_cpu(alt_path->opa.slid));
1430 IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_REMOTE_PORT_GID,
1431 req_msg)
1432 ->global.interface_id =
1433 OPA_MAKE_ID(be32_to_cpu(alt_path->opa.dlid));
1434 }
1435 if (alt_path->hop_limit <= 1) {
1436 IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,
1437 be16_to_cpu(
1438 alt_ext ? 0 :
1439 htons(ntohl(sa_path_get_slid(
1440 alt_path)))));
1441 IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,
1442 be16_to_cpu(
1443 alt_ext ? 0 :
1444 htons(ntohl(sa_path_get_dlid(
1445 alt_path)))));
1446 } else {
1447 IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,
1448 be16_to_cpu(IB_LID_PERMISSIVE));
1449 IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,
1450 be16_to_cpu(IB_LID_PERMISSIVE));
1451 }
1452 IBA_SET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg,
1453 be32_to_cpu(alt_path->flow_label));
1454 IBA_SET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg, alt_path->rate);
1455 IBA_SET(CM_REQ_ALTERNATE_TRAFFIC_CLASS, req_msg,
1456 alt_path->traffic_class);
1457 IBA_SET(CM_REQ_ALTERNATE_HOP_LIMIT, req_msg,
1458 alt_path->hop_limit);
1459 IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, alt_path->sl);
1460 IBA_SET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg,
1461 (alt_path->hop_limit <= 1));
1462 IBA_SET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, req_msg,
1463 cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
1464 alt_path->packet_life_time));
1465 }
1466 IBA_SET(CM_REQ_VENDOR_ID, req_msg, param->ece.vendor_id);
1467
1468 if (param->private_data && param->private_data_len)
1469 IBA_SET_MEM(CM_REQ_PRIVATE_DATA, req_msg, param->private_data,
1470 param->private_data_len);
1471 }
1472
cm_validate_req_param(struct ib_cm_req_param * param)1473 static int cm_validate_req_param(struct ib_cm_req_param *param)
1474 {
1475 if (!param->primary_path)
1476 return -EINVAL;
1477
1478 if (param->qp_type != IB_QPT_RC && param->qp_type != IB_QPT_UC &&
1479 param->qp_type != IB_QPT_XRC_INI)
1480 return -EINVAL;
1481
1482 if (param->private_data &&
1483 param->private_data_len > IB_CM_REQ_PRIVATE_DATA_SIZE)
1484 return -EINVAL;
1485
1486 if (param->alternate_path &&
1487 (param->alternate_path->pkey != param->primary_path->pkey ||
1488 param->alternate_path->mtu != param->primary_path->mtu))
1489 return -EINVAL;
1490
1491 return 0;
1492 }
1493
ib_send_cm_req(struct ib_cm_id * cm_id,struct ib_cm_req_param * param)1494 int ib_send_cm_req(struct ib_cm_id *cm_id,
1495 struct ib_cm_req_param *param)
1496 {
1497 struct cm_av av = {}, alt_av = {};
1498 struct cm_id_private *cm_id_priv;
1499 struct ib_mad_send_buf *msg;
1500 struct cm_req_msg *req_msg;
1501 unsigned long flags;
1502 int ret;
1503
1504 ret = cm_validate_req_param(param);
1505 if (ret)
1506 return ret;
1507
1508 /* Verify that we're not in timewait. */
1509 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1510 spin_lock_irqsave(&cm_id_priv->lock, flags);
1511 if (cm_id->state != IB_CM_IDLE || WARN_ON(cm_id_priv->timewait_info)) {
1512 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1513 return -EINVAL;
1514 }
1515 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1516
1517 cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv->
1518 id.local_id);
1519 if (IS_ERR(cm_id_priv->timewait_info)) {
1520 ret = PTR_ERR(cm_id_priv->timewait_info);
1521 cm_id_priv->timewait_info = NULL;
1522 return ret;
1523 }
1524
1525 ret = cm_init_av_by_path(param->primary_path,
1526 param->ppath_sgid_attr, &av);
1527 if (ret)
1528 return ret;
1529 if (param->alternate_path) {
1530 ret = cm_init_av_by_path(param->alternate_path, NULL,
1531 &alt_av);
1532 if (ret) {
1533 cm_destroy_av(&av);
1534 return ret;
1535 }
1536 }
1537 cm_id->service_id = param->service_id;
1538 cm_id_priv->timeout_ms = cm_convert_to_ms(
1539 param->primary_path->packet_life_time) * 2 +
1540 cm_convert_to_ms(
1541 param->remote_cm_response_timeout);
1542 cm_id_priv->max_cm_retries = param->max_cm_retries;
1543 cm_id_priv->initiator_depth = param->initiator_depth;
1544 cm_id_priv->responder_resources = param->responder_resources;
1545 cm_id_priv->retry_count = param->retry_count;
1546 cm_id_priv->path_mtu = param->primary_path->mtu;
1547 cm_id_priv->pkey = param->primary_path->pkey;
1548 cm_id_priv->qp_type = param->qp_type;
1549
1550 spin_lock_irqsave(&cm_id_priv->lock, flags);
1551
1552 cm_move_av_from_path(&cm_id_priv->av, &av);
1553 if (param->primary_path_outbound)
1554 cm_id_priv->av.dlid_datapath =
1555 be16_to_cpu(param->primary_path_outbound->ib.dlid);
1556
1557 if (param->alternate_path)
1558 cm_move_av_from_path(&cm_id_priv->alt_av, &alt_av);
1559
1560 msg = cm_alloc_priv_msg(cm_id_priv);
1561 if (IS_ERR(msg)) {
1562 ret = PTR_ERR(msg);
1563 goto out_unlock;
1564 }
1565
1566 req_msg = (struct cm_req_msg *)msg->mad;
1567 cm_format_req(req_msg, cm_id_priv, param);
1568 cm_id_priv->tid = req_msg->hdr.tid;
1569 msg->timeout_ms = cm_id_priv->timeout_ms;
1570 msg->context[1] = (void *)(unsigned long)IB_CM_REQ_SENT;
1571
1572 cm_id_priv->local_qpn = cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg));
1573 cm_id_priv->rq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg));
1574
1575 trace_icm_send_req(&cm_id_priv->id);
1576 ret = ib_post_send_mad(msg, NULL);
1577 if (ret)
1578 goto out_free;
1579 BUG_ON(cm_id->state != IB_CM_IDLE);
1580 cm_id->state = IB_CM_REQ_SENT;
1581 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1582 return 0;
1583 out_free:
1584 cm_free_priv_msg(msg);
1585 out_unlock:
1586 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1587 return ret;
1588 }
1589 EXPORT_SYMBOL(ib_send_cm_req);
1590
cm_issue_rej(struct cm_port * port,struct ib_mad_recv_wc * mad_recv_wc,enum ib_cm_rej_reason reason,enum cm_msg_response msg_rejected,void * ari,u8 ari_length)1591 static int cm_issue_rej(struct cm_port *port,
1592 struct ib_mad_recv_wc *mad_recv_wc,
1593 enum ib_cm_rej_reason reason,
1594 enum cm_msg_response msg_rejected,
1595 void *ari, u8 ari_length)
1596 {
1597 struct ib_mad_send_buf *msg = NULL;
1598 struct cm_rej_msg *rej_msg, *rcv_msg;
1599 int ret;
1600
1601 ret = cm_alloc_response_msg(port, mad_recv_wc, &msg);
1602 if (ret)
1603 return ret;
1604
1605 /* We just need common CM header information. Cast to any message. */
1606 rcv_msg = (struct cm_rej_msg *) mad_recv_wc->recv_buf.mad;
1607 rej_msg = (struct cm_rej_msg *) msg->mad;
1608
1609 cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, rcv_msg->hdr.tid);
1610 IBA_SET(CM_REJ_REMOTE_COMM_ID, rej_msg,
1611 IBA_GET(CM_REJ_LOCAL_COMM_ID, rcv_msg));
1612 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1613 IBA_GET(CM_REJ_REMOTE_COMM_ID, rcv_msg));
1614 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, msg_rejected);
1615 IBA_SET(CM_REJ_REASON, rej_msg, reason);
1616
1617 if (ari && ari_length) {
1618 IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length);
1619 IBA_SET_MEM(CM_REJ_ARI, rej_msg, ari, ari_length);
1620 }
1621
1622 trace_icm_issue_rej(
1623 IBA_GET(CM_REJ_LOCAL_COMM_ID, rcv_msg),
1624 IBA_GET(CM_REJ_REMOTE_COMM_ID, rcv_msg));
1625 ret = ib_post_send_mad(msg, NULL);
1626 if (ret)
1627 cm_free_response_msg(msg);
1628
1629 return ret;
1630 }
1631
cm_req_has_alt_path(struct cm_req_msg * req_msg)1632 static bool cm_req_has_alt_path(struct cm_req_msg *req_msg)
1633 {
1634 return ((cpu_to_be16(
1635 IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg))) ||
1636 (ib_is_opa_gid(IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID,
1637 req_msg))));
1638 }
1639
cm_path_set_rec_type(struct ib_device * ib_device,u32 port_num,struct sa_path_rec * path,union ib_gid * gid)1640 static void cm_path_set_rec_type(struct ib_device *ib_device, u32 port_num,
1641 struct sa_path_rec *path, union ib_gid *gid)
1642 {
1643 if (ib_is_opa_gid(gid) && rdma_cap_opa_ah(ib_device, port_num))
1644 path->rec_type = SA_PATH_REC_TYPE_OPA;
1645 else
1646 path->rec_type = SA_PATH_REC_TYPE_IB;
1647 }
1648
cm_format_path_lid_from_req(struct cm_req_msg * req_msg,struct sa_path_rec * primary_path,struct sa_path_rec * alt_path,struct ib_wc * wc)1649 static void cm_format_path_lid_from_req(struct cm_req_msg *req_msg,
1650 struct sa_path_rec *primary_path,
1651 struct sa_path_rec *alt_path,
1652 struct ib_wc *wc)
1653 {
1654 u32 lid;
1655
1656 if (primary_path->rec_type != SA_PATH_REC_TYPE_OPA) {
1657 sa_path_set_dlid(primary_path, wc->slid);
1658 sa_path_set_slid(primary_path,
1659 IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID,
1660 req_msg));
1661 } else {
1662 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1663 CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg));
1664 sa_path_set_dlid(primary_path, lid);
1665
1666 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1667 CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg));
1668 sa_path_set_slid(primary_path, lid);
1669 }
1670
1671 if (!cm_req_has_alt_path(req_msg))
1672 return;
1673
1674 if (alt_path->rec_type != SA_PATH_REC_TYPE_OPA) {
1675 sa_path_set_dlid(alt_path,
1676 IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID,
1677 req_msg));
1678 sa_path_set_slid(alt_path,
1679 IBA_GET(CM_REQ_ALTERNATE_REMOTE_PORT_LID,
1680 req_msg));
1681 } else {
1682 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1683 CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg));
1684 sa_path_set_dlid(alt_path, lid);
1685
1686 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1687 CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg));
1688 sa_path_set_slid(alt_path, lid);
1689 }
1690 }
1691
cm_format_paths_from_req(struct cm_req_msg * req_msg,struct sa_path_rec * primary_path,struct sa_path_rec * alt_path,struct ib_wc * wc)1692 static void cm_format_paths_from_req(struct cm_req_msg *req_msg,
1693 struct sa_path_rec *primary_path,
1694 struct sa_path_rec *alt_path,
1695 struct ib_wc *wc)
1696 {
1697 primary_path->dgid =
1698 *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg);
1699 primary_path->sgid =
1700 *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg);
1701 primary_path->flow_label =
1702 cpu_to_be32(IBA_GET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg));
1703 primary_path->hop_limit = IBA_GET(CM_REQ_PRIMARY_HOP_LIMIT, req_msg);
1704 primary_path->traffic_class =
1705 IBA_GET(CM_REQ_PRIMARY_TRAFFIC_CLASS, req_msg);
1706 primary_path->reversible = 1;
1707 primary_path->pkey =
1708 cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));
1709 primary_path->sl = IBA_GET(CM_REQ_PRIMARY_SL, req_msg);
1710 primary_path->mtu_selector = IB_SA_EQ;
1711 primary_path->mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);
1712 primary_path->rate_selector = IB_SA_EQ;
1713 primary_path->rate = IBA_GET(CM_REQ_PRIMARY_PACKET_RATE, req_msg);
1714 primary_path->packet_life_time_selector = IB_SA_EQ;
1715 primary_path->packet_life_time =
1716 IBA_GET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, req_msg);
1717 primary_path->packet_life_time -= (primary_path->packet_life_time > 0);
1718 primary_path->service_id =
1719 cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));
1720 if (sa_path_is_roce(primary_path))
1721 primary_path->roce.route_resolved = false;
1722
1723 if (cm_req_has_alt_path(req_msg)) {
1724 alt_path->dgid = *IBA_GET_MEM_PTR(
1725 CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg);
1726 alt_path->sgid = *IBA_GET_MEM_PTR(
1727 CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg);
1728 alt_path->flow_label = cpu_to_be32(
1729 IBA_GET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg));
1730 alt_path->hop_limit =
1731 IBA_GET(CM_REQ_ALTERNATE_HOP_LIMIT, req_msg);
1732 alt_path->traffic_class =
1733 IBA_GET(CM_REQ_ALTERNATE_TRAFFIC_CLASS, req_msg);
1734 alt_path->reversible = 1;
1735 alt_path->pkey =
1736 cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));
1737 alt_path->sl = IBA_GET(CM_REQ_ALTERNATE_SL, req_msg);
1738 alt_path->mtu_selector = IB_SA_EQ;
1739 alt_path->mtu =
1740 IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);
1741 alt_path->rate_selector = IB_SA_EQ;
1742 alt_path->rate = IBA_GET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg);
1743 alt_path->packet_life_time_selector = IB_SA_EQ;
1744 alt_path->packet_life_time =
1745 IBA_GET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, req_msg);
1746 alt_path->packet_life_time -= (alt_path->packet_life_time > 0);
1747 alt_path->service_id =
1748 cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));
1749
1750 if (sa_path_is_roce(alt_path))
1751 alt_path->roce.route_resolved = false;
1752 }
1753 cm_format_path_lid_from_req(req_msg, primary_path, alt_path, wc);
1754 }
1755
cm_get_bth_pkey(struct cm_work * work)1756 static u16 cm_get_bth_pkey(struct cm_work *work)
1757 {
1758 struct ib_device *ib_dev = work->port->cm_dev->ib_device;
1759 u32 port_num = work->port->port_num;
1760 u16 pkey_index = work->mad_recv_wc->wc->pkey_index;
1761 u16 pkey;
1762 int ret;
1763
1764 ret = ib_get_cached_pkey(ib_dev, port_num, pkey_index, &pkey);
1765 if (ret) {
1766 dev_warn_ratelimited(&ib_dev->dev, "ib_cm: Couldn't retrieve pkey for incoming request (port %u, pkey index %u). %d\n",
1767 port_num, pkey_index, ret);
1768 return 0;
1769 }
1770
1771 return pkey;
1772 }
1773
1774 /**
1775 * cm_opa_to_ib_sgid - Convert OPA SGID to IB SGID
1776 * ULPs (such as IPoIB) do not understand OPA GIDs and will
1777 * reject them as the local_gid will not match the sgid. Therefore,
1778 * change the pathrec's SGID to an IB SGID.
1779 *
1780 * @work: Work completion
1781 * @path: Path record
1782 */
cm_opa_to_ib_sgid(struct cm_work * work,struct sa_path_rec * path)1783 static void cm_opa_to_ib_sgid(struct cm_work *work,
1784 struct sa_path_rec *path)
1785 {
1786 struct ib_device *dev = work->port->cm_dev->ib_device;
1787 u32 port_num = work->port->port_num;
1788
1789 if (rdma_cap_opa_ah(dev, port_num) &&
1790 (ib_is_opa_gid(&path->sgid))) {
1791 union ib_gid sgid;
1792
1793 if (rdma_query_gid(dev, port_num, 0, &sgid)) {
1794 dev_warn(&dev->dev,
1795 "Error updating sgid in CM request\n");
1796 return;
1797 }
1798
1799 path->sgid = sgid;
1800 }
1801 }
1802
cm_format_req_event(struct cm_work * work,struct cm_id_private * cm_id_priv,struct ib_cm_id * listen_id)1803 static void cm_format_req_event(struct cm_work *work,
1804 struct cm_id_private *cm_id_priv,
1805 struct ib_cm_id *listen_id)
1806 {
1807 struct cm_req_msg *req_msg;
1808 struct ib_cm_req_event_param *param;
1809
1810 req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1811 param = &work->cm_event.param.req_rcvd;
1812 param->listen_id = listen_id;
1813 param->bth_pkey = cm_get_bth_pkey(work);
1814 param->port = cm_id_priv->av.port->port_num;
1815 param->primary_path = &work->path[0];
1816 cm_opa_to_ib_sgid(work, param->primary_path);
1817 if (cm_req_has_alt_path(req_msg)) {
1818 param->alternate_path = &work->path[1];
1819 cm_opa_to_ib_sgid(work, param->alternate_path);
1820 } else {
1821 param->alternate_path = NULL;
1822 }
1823 param->remote_ca_guid =
1824 cpu_to_be64(IBA_GET(CM_REQ_LOCAL_CA_GUID, req_msg));
1825 param->remote_qkey = IBA_GET(CM_REQ_LOCAL_Q_KEY, req_msg);
1826 param->remote_qpn = IBA_GET(CM_REQ_LOCAL_QPN, req_msg);
1827 param->qp_type = cm_req_get_qp_type(req_msg);
1828 param->starting_psn = IBA_GET(CM_REQ_STARTING_PSN, req_msg);
1829 param->responder_resources = IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg);
1830 param->initiator_depth = IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg);
1831 param->local_cm_response_timeout =
1832 IBA_GET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg);
1833 param->flow_control = IBA_GET(CM_REQ_END_TO_END_FLOW_CONTROL, req_msg);
1834 param->remote_cm_response_timeout =
1835 IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg);
1836 param->retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg);
1837 param->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg);
1838 param->srq = IBA_GET(CM_REQ_SRQ, req_msg);
1839 param->ppath_sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr;
1840 param->ece.vendor_id = IBA_GET(CM_REQ_VENDOR_ID, req_msg);
1841 param->ece.attr_mod = be32_to_cpu(req_msg->hdr.attr_mod);
1842
1843 work->cm_event.private_data =
1844 IBA_GET_MEM_PTR(CM_REQ_PRIVATE_DATA, req_msg);
1845 }
1846
cm_process_work(struct cm_id_private * cm_id_priv,struct cm_work * work)1847 static void cm_process_work(struct cm_id_private *cm_id_priv,
1848 struct cm_work *work)
1849 {
1850 int ret;
1851
1852 /* We will typically only have the current event to report. */
1853 ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &work->cm_event);
1854 cm_free_work(work);
1855
1856 while (!ret && !atomic_add_negative(-1, &cm_id_priv->work_count)) {
1857 spin_lock_irq(&cm_id_priv->lock);
1858 work = cm_dequeue_work(cm_id_priv);
1859 spin_unlock_irq(&cm_id_priv->lock);
1860 if (!work)
1861 return;
1862
1863 ret = cm_id_priv->id.cm_handler(&cm_id_priv->id,
1864 &work->cm_event);
1865 cm_free_work(work);
1866 }
1867 cm_deref_id(cm_id_priv);
1868 if (ret)
1869 cm_destroy_id(&cm_id_priv->id, ret);
1870 }
1871
cm_format_mra(struct cm_mra_msg * mra_msg,struct cm_id_private * cm_id_priv,enum cm_msg_response msg_mraed,u8 service_timeout,const void * private_data,u8 private_data_len)1872 static void cm_format_mra(struct cm_mra_msg *mra_msg,
1873 struct cm_id_private *cm_id_priv,
1874 enum cm_msg_response msg_mraed, u8 service_timeout,
1875 const void *private_data, u8 private_data_len)
1876 {
1877 cm_format_mad_hdr(&mra_msg->hdr, CM_MRA_ATTR_ID, cm_id_priv->tid);
1878 IBA_SET(CM_MRA_MESSAGE_MRAED, mra_msg, msg_mraed);
1879 IBA_SET(CM_MRA_LOCAL_COMM_ID, mra_msg,
1880 be32_to_cpu(cm_id_priv->id.local_id));
1881 IBA_SET(CM_MRA_REMOTE_COMM_ID, mra_msg,
1882 be32_to_cpu(cm_id_priv->id.remote_id));
1883 IBA_SET(CM_MRA_SERVICE_TIMEOUT, mra_msg, service_timeout);
1884
1885 if (private_data && private_data_len)
1886 IBA_SET_MEM(CM_MRA_PRIVATE_DATA, mra_msg, private_data,
1887 private_data_len);
1888 }
1889
cm_format_rej(struct cm_rej_msg * rej_msg,struct cm_id_private * cm_id_priv,enum ib_cm_rej_reason reason,void * ari,u8 ari_length,const void * private_data,u8 private_data_len,enum ib_cm_state state)1890 static void cm_format_rej(struct cm_rej_msg *rej_msg,
1891 struct cm_id_private *cm_id_priv,
1892 enum ib_cm_rej_reason reason, void *ari,
1893 u8 ari_length, const void *private_data,
1894 u8 private_data_len, enum ib_cm_state state)
1895 {
1896 lockdep_assert_held(&cm_id_priv->lock);
1897
1898 cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, cm_id_priv->tid);
1899 IBA_SET(CM_REJ_REMOTE_COMM_ID, rej_msg,
1900 be32_to_cpu(cm_id_priv->id.remote_id));
1901
1902 switch (state) {
1903 case IB_CM_REQ_RCVD:
1904 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg, be32_to_cpu(0));
1905 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ);
1906 break;
1907 case IB_CM_MRA_REQ_SENT:
1908 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1909 be32_to_cpu(cm_id_priv->id.local_id));
1910 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ);
1911 break;
1912 case IB_CM_REP_RCVD:
1913 case IB_CM_MRA_REP_SENT:
1914 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1915 be32_to_cpu(cm_id_priv->id.local_id));
1916 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REP);
1917 break;
1918 default:
1919 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1920 be32_to_cpu(cm_id_priv->id.local_id));
1921 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg,
1922 CM_MSG_RESPONSE_OTHER);
1923 break;
1924 }
1925
1926 IBA_SET(CM_REJ_REASON, rej_msg, reason);
1927 if (ari && ari_length) {
1928 IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length);
1929 IBA_SET_MEM(CM_REJ_ARI, rej_msg, ari, ari_length);
1930 }
1931
1932 if (private_data && private_data_len)
1933 IBA_SET_MEM(CM_REJ_PRIVATE_DATA, rej_msg, private_data,
1934 private_data_len);
1935 }
1936
cm_dup_req_handler(struct cm_work * work,struct cm_id_private * cm_id_priv)1937 static void cm_dup_req_handler(struct cm_work *work,
1938 struct cm_id_private *cm_id_priv)
1939 {
1940 struct ib_mad_send_buf *msg = NULL;
1941 int ret;
1942
1943 atomic_long_inc(
1944 &work->port->counters[CM_RECV_DUPLICATES][CM_REQ_COUNTER]);
1945
1946 /* Quick state check to discard duplicate REQs. */
1947 spin_lock_irq(&cm_id_priv->lock);
1948 if (cm_id_priv->id.state == IB_CM_REQ_RCVD) {
1949 spin_unlock_irq(&cm_id_priv->lock);
1950 return;
1951 }
1952 spin_unlock_irq(&cm_id_priv->lock);
1953
1954 ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg);
1955 if (ret)
1956 return;
1957
1958 spin_lock_irq(&cm_id_priv->lock);
1959 switch (cm_id_priv->id.state) {
1960 case IB_CM_MRA_REQ_SENT:
1961 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
1962 CM_MSG_RESPONSE_REQ, cm_id_priv->service_timeout,
1963 cm_id_priv->private_data,
1964 cm_id_priv->private_data_len);
1965 break;
1966 case IB_CM_TIMEWAIT:
1967 cm_format_rej((struct cm_rej_msg *)msg->mad, cm_id_priv,
1968 IB_CM_REJ_STALE_CONN, NULL, 0, NULL, 0,
1969 IB_CM_TIMEWAIT);
1970 break;
1971 default:
1972 goto unlock;
1973 }
1974 spin_unlock_irq(&cm_id_priv->lock);
1975
1976 trace_icm_send_dup_req(&cm_id_priv->id);
1977 ret = ib_post_send_mad(msg, NULL);
1978 if (ret)
1979 goto free;
1980 return;
1981
1982 unlock: spin_unlock_irq(&cm_id_priv->lock);
1983 free: cm_free_response_msg(msg);
1984 }
1985
cm_match_req(struct cm_work * work,struct cm_id_private * cm_id_priv)1986 static struct cm_id_private *cm_match_req(struct cm_work *work,
1987 struct cm_id_private *cm_id_priv)
1988 {
1989 struct cm_id_private *listen_cm_id_priv, *cur_cm_id_priv;
1990 struct cm_timewait_info *timewait_info;
1991 struct cm_req_msg *req_msg;
1992
1993 req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1994
1995 /* Check for possible duplicate REQ. */
1996 spin_lock_irq(&cm.lock);
1997 timewait_info = cm_insert_remote_id(cm_id_priv->timewait_info);
1998 if (timewait_info) {
1999 cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
2000 timewait_info->work.remote_id);
2001 spin_unlock_irq(&cm.lock);
2002 if (cur_cm_id_priv) {
2003 cm_dup_req_handler(work, cur_cm_id_priv);
2004 cm_deref_id(cur_cm_id_priv);
2005 }
2006 return NULL;
2007 }
2008
2009 /* Check for stale connections. */
2010 timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info);
2011 if (timewait_info) {
2012 cm_remove_remote(cm_id_priv);
2013 cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
2014 timewait_info->work.remote_id);
2015
2016 spin_unlock_irq(&cm.lock);
2017 cm_issue_rej(work->port, work->mad_recv_wc,
2018 IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REQ,
2019 NULL, 0);
2020 if (cur_cm_id_priv) {
2021 ib_send_cm_dreq(&cur_cm_id_priv->id, NULL, 0);
2022 cm_deref_id(cur_cm_id_priv);
2023 }
2024 return NULL;
2025 }
2026
2027 /* Find matching listen request. */
2028 listen_cm_id_priv = cm_find_listen(
2029 cm_id_priv->id.device,
2030 cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg)));
2031 if (!listen_cm_id_priv) {
2032 cm_remove_remote(cm_id_priv);
2033 spin_unlock_irq(&cm.lock);
2034 cm_issue_rej(work->port, work->mad_recv_wc,
2035 IB_CM_REJ_INVALID_SERVICE_ID, CM_MSG_RESPONSE_REQ,
2036 NULL, 0);
2037 return NULL;
2038 }
2039 spin_unlock_irq(&cm.lock);
2040 return listen_cm_id_priv;
2041 }
2042
2043 /*
2044 * Work-around for inter-subnet connections. If the LIDs are permissive,
2045 * we need to override the LID/SL data in the REQ with the LID information
2046 * in the work completion.
2047 */
cm_process_routed_req(struct cm_req_msg * req_msg,struct ib_wc * wc)2048 static void cm_process_routed_req(struct cm_req_msg *req_msg, struct ib_wc *wc)
2049 {
2050 if (!IBA_GET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg)) {
2051 if (cpu_to_be16(IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID,
2052 req_msg)) == IB_LID_PERMISSIVE) {
2053 IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
2054 be16_to_cpu(ib_lid_be16(wc->slid)));
2055 IBA_SET(CM_REQ_PRIMARY_SL, req_msg, wc->sl);
2056 }
2057
2058 if (cpu_to_be16(IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID,
2059 req_msg)) == IB_LID_PERMISSIVE)
2060 IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,
2061 wc->dlid_path_bits);
2062 }
2063
2064 if (!IBA_GET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg)) {
2065 if (cpu_to_be16(IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID,
2066 req_msg)) == IB_LID_PERMISSIVE) {
2067 IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,
2068 be16_to_cpu(ib_lid_be16(wc->slid)));
2069 IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, wc->sl);
2070 }
2071
2072 if (cpu_to_be16(IBA_GET(CM_REQ_ALTERNATE_REMOTE_PORT_LID,
2073 req_msg)) == IB_LID_PERMISSIVE)
2074 IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,
2075 wc->dlid_path_bits);
2076 }
2077 }
2078
cm_req_handler(struct cm_work * work)2079 static int cm_req_handler(struct cm_work *work)
2080 {
2081 struct cm_id_private *cm_id_priv, *listen_cm_id_priv;
2082 struct cm_req_msg *req_msg;
2083 const struct ib_global_route *grh;
2084 const struct ib_gid_attr *gid_attr;
2085 int ret;
2086
2087 req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
2088
2089 cm_id_priv =
2090 cm_alloc_id_priv(work->port->cm_dev->ib_device, NULL, NULL);
2091 if (IS_ERR(cm_id_priv))
2092 return PTR_ERR(cm_id_priv);
2093
2094 cm_id_priv->id.remote_id =
2095 cpu_to_be32(IBA_GET(CM_REQ_LOCAL_COMM_ID, req_msg));
2096 cm_id_priv->id.service_id =
2097 cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));
2098 cm_id_priv->tid = req_msg->hdr.tid;
2099 cm_id_priv->timeout_ms = cm_convert_to_ms(
2100 IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg));
2101 cm_id_priv->max_cm_retries = IBA_GET(CM_REQ_MAX_CM_RETRIES, req_msg);
2102 cm_id_priv->remote_qpn =
2103 cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg));
2104 cm_id_priv->initiator_depth =
2105 IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg);
2106 cm_id_priv->responder_resources =
2107 IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg);
2108 cm_id_priv->path_mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);
2109 cm_id_priv->pkey = cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));
2110 cm_id_priv->sq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg));
2111 cm_id_priv->retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg);
2112 cm_id_priv->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg);
2113 cm_id_priv->qp_type = cm_req_get_qp_type(req_msg);
2114
2115 ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
2116 work->mad_recv_wc->recv_buf.grh,
2117 &cm_id_priv->av);
2118 if (ret)
2119 goto destroy;
2120 cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv->
2121 id.local_id);
2122 if (IS_ERR(cm_id_priv->timewait_info)) {
2123 ret = PTR_ERR(cm_id_priv->timewait_info);
2124 cm_id_priv->timewait_info = NULL;
2125 goto destroy;
2126 }
2127 cm_id_priv->timewait_info->work.remote_id = cm_id_priv->id.remote_id;
2128 cm_id_priv->timewait_info->remote_ca_guid =
2129 cpu_to_be64(IBA_GET(CM_REQ_LOCAL_CA_GUID, req_msg));
2130 cm_id_priv->timewait_info->remote_qpn = cm_id_priv->remote_qpn;
2131
2132 /*
2133 * Note that the ID pointer is not in the xarray at this point,
2134 * so this set is only visible to the local thread.
2135 */
2136 cm_id_priv->id.state = IB_CM_REQ_RCVD;
2137
2138 listen_cm_id_priv = cm_match_req(work, cm_id_priv);
2139 if (!listen_cm_id_priv) {
2140 trace_icm_no_listener_err(&cm_id_priv->id);
2141 cm_id_priv->id.state = IB_CM_IDLE;
2142 ret = -EINVAL;
2143 goto destroy;
2144 }
2145
2146 memset(&work->path[0], 0, sizeof(work->path[0]));
2147 if (cm_req_has_alt_path(req_msg))
2148 memset(&work->path[1], 0, sizeof(work->path[1]));
2149 grh = rdma_ah_read_grh(&cm_id_priv->av.ah_attr);
2150 gid_attr = grh->sgid_attr;
2151
2152 if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE) {
2153 work->path[0].rec_type =
2154 sa_conv_gid_to_pathrec_type(gid_attr->gid_type);
2155 } else {
2156 cm_process_routed_req(req_msg, work->mad_recv_wc->wc);
2157 cm_path_set_rec_type(
2158 work->port->cm_dev->ib_device, work->port->port_num,
2159 &work->path[0],
2160 IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID,
2161 req_msg));
2162 }
2163 if (cm_req_has_alt_path(req_msg))
2164 work->path[1].rec_type = work->path[0].rec_type;
2165 cm_format_paths_from_req(req_msg, &work->path[0],
2166 &work->path[1], work->mad_recv_wc->wc);
2167 if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE)
2168 sa_path_set_dmac(&work->path[0],
2169 cm_id_priv->av.ah_attr.roce.dmac);
2170 work->path[0].hop_limit = grh->hop_limit;
2171
2172 /* This destroy call is needed to pair with cm_init_av_for_response */
2173 cm_destroy_av(&cm_id_priv->av);
2174 ret = cm_init_av_by_path(&work->path[0], gid_attr, &cm_id_priv->av);
2175 if (ret) {
2176 int err;
2177
2178 err = rdma_query_gid(work->port->cm_dev->ib_device,
2179 work->port->port_num, 0,
2180 &work->path[0].sgid);
2181 if (err)
2182 ib_send_cm_rej(&cm_id_priv->id, IB_CM_REJ_INVALID_GID,
2183 NULL, 0, NULL, 0);
2184 else
2185 ib_send_cm_rej(&cm_id_priv->id, IB_CM_REJ_INVALID_GID,
2186 &work->path[0].sgid,
2187 sizeof(work->path[0].sgid),
2188 NULL, 0);
2189 goto rejected;
2190 }
2191 if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_IB)
2192 cm_id_priv->av.dlid_datapath =
2193 IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg);
2194
2195 if (cm_req_has_alt_path(req_msg)) {
2196 ret = cm_init_av_by_path(&work->path[1], NULL,
2197 &cm_id_priv->alt_av);
2198 if (ret) {
2199 ib_send_cm_rej(&cm_id_priv->id,
2200 IB_CM_REJ_INVALID_ALT_GID,
2201 &work->path[0].sgid,
2202 sizeof(work->path[0].sgid), NULL, 0);
2203 goto rejected;
2204 }
2205 }
2206
2207 cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler;
2208 cm_id_priv->id.context = listen_cm_id_priv->id.context;
2209 cm_format_req_event(work, cm_id_priv, &listen_cm_id_priv->id);
2210
2211 /* Now MAD handlers can see the new ID */
2212 spin_lock_irq(&cm_id_priv->lock);
2213 cm_finalize_id(cm_id_priv);
2214
2215 /* Refcount belongs to the event, pairs with cm_process_work() */
2216 refcount_inc(&cm_id_priv->refcount);
2217 cm_queue_work_unlock(cm_id_priv, work);
2218 /*
2219 * Since this ID was just created and was not made visible to other MAD
2220 * handlers until the cm_finalize_id() above we know that the
2221 * cm_process_work() will deliver the event and the listen_cm_id
2222 * embedded in the event can be derefed here.
2223 */
2224 cm_deref_id(listen_cm_id_priv);
2225 return 0;
2226
2227 rejected:
2228 cm_deref_id(listen_cm_id_priv);
2229 destroy:
2230 ib_destroy_cm_id(&cm_id_priv->id);
2231 return ret;
2232 }
2233
cm_format_rep(struct cm_rep_msg * rep_msg,struct cm_id_private * cm_id_priv,struct ib_cm_rep_param * param)2234 static void cm_format_rep(struct cm_rep_msg *rep_msg,
2235 struct cm_id_private *cm_id_priv,
2236 struct ib_cm_rep_param *param)
2237 {
2238 cm_format_mad_ece_hdr(&rep_msg->hdr, CM_REP_ATTR_ID, cm_id_priv->tid,
2239 param->ece.attr_mod);
2240 IBA_SET(CM_REP_LOCAL_COMM_ID, rep_msg,
2241 be32_to_cpu(cm_id_priv->id.local_id));
2242 IBA_SET(CM_REP_REMOTE_COMM_ID, rep_msg,
2243 be32_to_cpu(cm_id_priv->id.remote_id));
2244 IBA_SET(CM_REP_STARTING_PSN, rep_msg, param->starting_psn);
2245 IBA_SET(CM_REP_RESPONDER_RESOURCES, rep_msg,
2246 param->responder_resources);
2247 IBA_SET(CM_REP_TARGET_ACK_DELAY, rep_msg,
2248 cm_id_priv->av.port->cm_dev->ack_delay);
2249 IBA_SET(CM_REP_FAILOVER_ACCEPTED, rep_msg, param->failover_accepted);
2250 IBA_SET(CM_REP_RNR_RETRY_COUNT, rep_msg, param->rnr_retry_count);
2251 IBA_SET(CM_REP_LOCAL_CA_GUID, rep_msg,
2252 be64_to_cpu(cm_id_priv->id.device->node_guid));
2253
2254 if (cm_id_priv->qp_type != IB_QPT_XRC_TGT) {
2255 IBA_SET(CM_REP_INITIATOR_DEPTH, rep_msg,
2256 param->initiator_depth);
2257 IBA_SET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg,
2258 param->flow_control);
2259 IBA_SET(CM_REP_SRQ, rep_msg, param->srq);
2260 IBA_SET(CM_REP_LOCAL_QPN, rep_msg, param->qp_num);
2261 } else {
2262 IBA_SET(CM_REP_SRQ, rep_msg, 1);
2263 IBA_SET(CM_REP_LOCAL_EE_CONTEXT_NUMBER, rep_msg, param->qp_num);
2264 }
2265
2266 IBA_SET(CM_REP_VENDOR_ID_L, rep_msg, param->ece.vendor_id);
2267 IBA_SET(CM_REP_VENDOR_ID_M, rep_msg, param->ece.vendor_id >> 8);
2268 IBA_SET(CM_REP_VENDOR_ID_H, rep_msg, param->ece.vendor_id >> 16);
2269
2270 if (param->private_data && param->private_data_len)
2271 IBA_SET_MEM(CM_REP_PRIVATE_DATA, rep_msg, param->private_data,
2272 param->private_data_len);
2273 }
2274
ib_send_cm_rep(struct ib_cm_id * cm_id,struct ib_cm_rep_param * param)2275 int ib_send_cm_rep(struct ib_cm_id *cm_id,
2276 struct ib_cm_rep_param *param)
2277 {
2278 struct cm_id_private *cm_id_priv;
2279 struct ib_mad_send_buf *msg;
2280 struct cm_rep_msg *rep_msg;
2281 unsigned long flags;
2282 int ret;
2283
2284 if (param->private_data &&
2285 param->private_data_len > IB_CM_REP_PRIVATE_DATA_SIZE)
2286 return -EINVAL;
2287
2288 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2289 spin_lock_irqsave(&cm_id_priv->lock, flags);
2290 if (cm_id->state != IB_CM_REQ_RCVD &&
2291 cm_id->state != IB_CM_MRA_REQ_SENT) {
2292 trace_icm_send_rep_err(cm_id_priv->id.local_id, cm_id->state);
2293 ret = -EINVAL;
2294 goto out;
2295 }
2296
2297 msg = cm_alloc_priv_msg(cm_id_priv);
2298 if (IS_ERR(msg)) {
2299 ret = PTR_ERR(msg);
2300 goto out;
2301 }
2302
2303 rep_msg = (struct cm_rep_msg *) msg->mad;
2304 cm_format_rep(rep_msg, cm_id_priv, param);
2305 msg->timeout_ms = cm_id_priv->timeout_ms;
2306 msg->context[1] = (void *) (unsigned long) IB_CM_REP_SENT;
2307
2308 trace_icm_send_rep(cm_id);
2309 ret = ib_post_send_mad(msg, NULL);
2310 if (ret)
2311 goto out_free;
2312
2313 cm_id->state = IB_CM_REP_SENT;
2314 cm_id_priv->initiator_depth = param->initiator_depth;
2315 cm_id_priv->responder_resources = param->responder_resources;
2316 cm_id_priv->rq_psn = cpu_to_be32(IBA_GET(CM_REP_STARTING_PSN, rep_msg));
2317 WARN_ONCE(param->qp_num & 0xFF000000,
2318 "IBTA declares QPN to be 24 bits, but it is 0x%X\n",
2319 param->qp_num);
2320 cm_id_priv->local_qpn = cpu_to_be32(param->qp_num & 0xFFFFFF);
2321 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2322 return 0;
2323
2324 out_free:
2325 cm_free_priv_msg(msg);
2326 out:
2327 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2328 return ret;
2329 }
2330 EXPORT_SYMBOL(ib_send_cm_rep);
2331
cm_format_rtu(struct cm_rtu_msg * rtu_msg,struct cm_id_private * cm_id_priv,const void * private_data,u8 private_data_len)2332 static void cm_format_rtu(struct cm_rtu_msg *rtu_msg,
2333 struct cm_id_private *cm_id_priv,
2334 const void *private_data,
2335 u8 private_data_len)
2336 {
2337 cm_format_mad_hdr(&rtu_msg->hdr, CM_RTU_ATTR_ID, cm_id_priv->tid);
2338 IBA_SET(CM_RTU_LOCAL_COMM_ID, rtu_msg,
2339 be32_to_cpu(cm_id_priv->id.local_id));
2340 IBA_SET(CM_RTU_REMOTE_COMM_ID, rtu_msg,
2341 be32_to_cpu(cm_id_priv->id.remote_id));
2342
2343 if (private_data && private_data_len)
2344 IBA_SET_MEM(CM_RTU_PRIVATE_DATA, rtu_msg, private_data,
2345 private_data_len);
2346 }
2347
ib_send_cm_rtu(struct ib_cm_id * cm_id,const void * private_data,u8 private_data_len)2348 int ib_send_cm_rtu(struct ib_cm_id *cm_id,
2349 const void *private_data,
2350 u8 private_data_len)
2351 {
2352 struct cm_id_private *cm_id_priv;
2353 struct ib_mad_send_buf *msg;
2354 unsigned long flags;
2355 void *data;
2356 int ret;
2357
2358 if (private_data && private_data_len > IB_CM_RTU_PRIVATE_DATA_SIZE)
2359 return -EINVAL;
2360
2361 data = cm_copy_private_data(private_data, private_data_len);
2362 if (IS_ERR(data))
2363 return PTR_ERR(data);
2364
2365 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2366 spin_lock_irqsave(&cm_id_priv->lock, flags);
2367 if (cm_id->state != IB_CM_REP_RCVD &&
2368 cm_id->state != IB_CM_MRA_REP_SENT) {
2369 trace_icm_send_cm_rtu_err(cm_id);
2370 ret = -EINVAL;
2371 goto error;
2372 }
2373
2374 msg = cm_alloc_msg(cm_id_priv);
2375 if (IS_ERR(msg)) {
2376 ret = PTR_ERR(msg);
2377 goto error;
2378 }
2379
2380 cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv,
2381 private_data, private_data_len);
2382
2383 trace_icm_send_rtu(cm_id);
2384 ret = ib_post_send_mad(msg, NULL);
2385 if (ret) {
2386 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2387 cm_free_msg(msg);
2388 kfree(data);
2389 return ret;
2390 }
2391
2392 cm_id->state = IB_CM_ESTABLISHED;
2393 cm_set_private_data(cm_id_priv, data, private_data_len);
2394 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2395 return 0;
2396
2397 error: spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2398 kfree(data);
2399 return ret;
2400 }
2401 EXPORT_SYMBOL(ib_send_cm_rtu);
2402
cm_format_rep_event(struct cm_work * work,enum ib_qp_type qp_type)2403 static void cm_format_rep_event(struct cm_work *work, enum ib_qp_type qp_type)
2404 {
2405 struct cm_rep_msg *rep_msg;
2406 struct ib_cm_rep_event_param *param;
2407
2408 rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad;
2409 param = &work->cm_event.param.rep_rcvd;
2410 param->remote_ca_guid =
2411 cpu_to_be64(IBA_GET(CM_REP_LOCAL_CA_GUID, rep_msg));
2412 param->remote_qkey = IBA_GET(CM_REP_LOCAL_Q_KEY, rep_msg);
2413 param->remote_qpn = be32_to_cpu(cm_rep_get_qpn(rep_msg, qp_type));
2414 param->starting_psn = IBA_GET(CM_REP_STARTING_PSN, rep_msg);
2415 param->responder_resources = IBA_GET(CM_REP_INITIATOR_DEPTH, rep_msg);
2416 param->initiator_depth = IBA_GET(CM_REP_RESPONDER_RESOURCES, rep_msg);
2417 param->target_ack_delay = IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg);
2418 param->failover_accepted = IBA_GET(CM_REP_FAILOVER_ACCEPTED, rep_msg);
2419 param->flow_control = IBA_GET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg);
2420 param->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg);
2421 param->srq = IBA_GET(CM_REP_SRQ, rep_msg);
2422 param->ece.vendor_id = IBA_GET(CM_REP_VENDOR_ID_H, rep_msg) << 16;
2423 param->ece.vendor_id |= IBA_GET(CM_REP_VENDOR_ID_M, rep_msg) << 8;
2424 param->ece.vendor_id |= IBA_GET(CM_REP_VENDOR_ID_L, rep_msg);
2425 param->ece.attr_mod = be32_to_cpu(rep_msg->hdr.attr_mod);
2426
2427 work->cm_event.private_data =
2428 IBA_GET_MEM_PTR(CM_REP_PRIVATE_DATA, rep_msg);
2429 }
2430
cm_dup_rep_handler(struct cm_work * work)2431 static void cm_dup_rep_handler(struct cm_work *work)
2432 {
2433 struct cm_id_private *cm_id_priv;
2434 struct cm_rep_msg *rep_msg;
2435 struct ib_mad_send_buf *msg = NULL;
2436 int ret;
2437
2438 rep_msg = (struct cm_rep_msg *) work->mad_recv_wc->recv_buf.mad;
2439 cm_id_priv = cm_acquire_id(
2440 cpu_to_be32(IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)),
2441 cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg)));
2442 if (!cm_id_priv)
2443 return;
2444
2445 atomic_long_inc(
2446 &work->port->counters[CM_RECV_DUPLICATES][CM_REP_COUNTER]);
2447 ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg);
2448 if (ret)
2449 goto deref;
2450
2451 spin_lock_irq(&cm_id_priv->lock);
2452 if (cm_id_priv->id.state == IB_CM_ESTABLISHED)
2453 cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv,
2454 cm_id_priv->private_data,
2455 cm_id_priv->private_data_len);
2456 else if (cm_id_priv->id.state == IB_CM_MRA_REP_SENT)
2457 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
2458 CM_MSG_RESPONSE_REP, cm_id_priv->service_timeout,
2459 cm_id_priv->private_data,
2460 cm_id_priv->private_data_len);
2461 else
2462 goto unlock;
2463 spin_unlock_irq(&cm_id_priv->lock);
2464
2465 trace_icm_send_dup_rep(&cm_id_priv->id);
2466 ret = ib_post_send_mad(msg, NULL);
2467 if (ret)
2468 goto free;
2469 goto deref;
2470
2471 unlock: spin_unlock_irq(&cm_id_priv->lock);
2472 free: cm_free_response_msg(msg);
2473 deref: cm_deref_id(cm_id_priv);
2474 }
2475
cm_rep_handler(struct cm_work * work)2476 static int cm_rep_handler(struct cm_work *work)
2477 {
2478 struct cm_id_private *cm_id_priv;
2479 struct cm_rep_msg *rep_msg;
2480 int ret;
2481 struct cm_id_private *cur_cm_id_priv;
2482 struct cm_timewait_info *timewait_info;
2483
2484 rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad;
2485 cm_id_priv = cm_acquire_id(
2486 cpu_to_be32(IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)), 0);
2487 if (!cm_id_priv) {
2488 cm_dup_rep_handler(work);
2489 trace_icm_remote_no_priv_err(
2490 IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2491 return -EINVAL;
2492 }
2493
2494 cm_format_rep_event(work, cm_id_priv->qp_type);
2495
2496 spin_lock_irq(&cm_id_priv->lock);
2497 switch (cm_id_priv->id.state) {
2498 case IB_CM_REQ_SENT:
2499 case IB_CM_MRA_REQ_RCVD:
2500 break;
2501 default:
2502 ret = -EINVAL;
2503 trace_icm_rep_unknown_err(
2504 IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg),
2505 IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg),
2506 cm_id_priv->id.state);
2507 spin_unlock_irq(&cm_id_priv->lock);
2508 goto error;
2509 }
2510
2511 cm_id_priv->timewait_info->work.remote_id =
2512 cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg));
2513 cm_id_priv->timewait_info->remote_ca_guid =
2514 cpu_to_be64(IBA_GET(CM_REP_LOCAL_CA_GUID, rep_msg));
2515 cm_id_priv->timewait_info->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type);
2516
2517 spin_lock(&cm.lock);
2518 /* Check for duplicate REP. */
2519 if (cm_insert_remote_id(cm_id_priv->timewait_info)) {
2520 spin_unlock(&cm.lock);
2521 spin_unlock_irq(&cm_id_priv->lock);
2522 ret = -EINVAL;
2523 trace_icm_insert_failed_err(
2524 IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2525 goto error;
2526 }
2527 /* Check for a stale connection. */
2528 timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info);
2529 if (timewait_info) {
2530 cm_remove_remote(cm_id_priv);
2531 cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
2532 timewait_info->work.remote_id);
2533
2534 spin_unlock(&cm.lock);
2535 spin_unlock_irq(&cm_id_priv->lock);
2536 cm_issue_rej(work->port, work->mad_recv_wc,
2537 IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REP,
2538 NULL, 0);
2539 ret = -EINVAL;
2540 trace_icm_staleconn_err(
2541 IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg),
2542 IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2543
2544 if (cur_cm_id_priv) {
2545 ib_send_cm_dreq(&cur_cm_id_priv->id, NULL, 0);
2546 cm_deref_id(cur_cm_id_priv);
2547 }
2548
2549 goto error;
2550 }
2551 spin_unlock(&cm.lock);
2552
2553 cm_id_priv->id.state = IB_CM_REP_RCVD;
2554 cm_id_priv->id.remote_id =
2555 cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg));
2556 cm_id_priv->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type);
2557 cm_id_priv->initiator_depth =
2558 IBA_GET(CM_REP_RESPONDER_RESOURCES, rep_msg);
2559 cm_id_priv->responder_resources =
2560 IBA_GET(CM_REP_INITIATOR_DEPTH, rep_msg);
2561 cm_id_priv->sq_psn = cpu_to_be32(IBA_GET(CM_REP_STARTING_PSN, rep_msg));
2562 cm_id_priv->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg);
2563 cm_id_priv->target_ack_delay =
2564 IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg);
2565 cm_id_priv->av.timeout =
2566 cm_ack_timeout(cm_id_priv->target_ack_delay,
2567 cm_id_priv->av.timeout - 1);
2568 cm_id_priv->alt_av.timeout =
2569 cm_ack_timeout(cm_id_priv->target_ack_delay,
2570 cm_id_priv->alt_av.timeout - 1);
2571
2572 ib_cancel_mad(cm_id_priv->msg);
2573 cm_queue_work_unlock(cm_id_priv, work);
2574 return 0;
2575
2576 error:
2577 cm_deref_id(cm_id_priv);
2578 return ret;
2579 }
2580
cm_establish_handler(struct cm_work * work)2581 static int cm_establish_handler(struct cm_work *work)
2582 {
2583 struct cm_id_private *cm_id_priv;
2584
2585 /* See comment in cm_establish about lookup. */
2586 cm_id_priv = cm_acquire_id(work->local_id, work->remote_id);
2587 if (!cm_id_priv)
2588 return -EINVAL;
2589
2590 spin_lock_irq(&cm_id_priv->lock);
2591 if (cm_id_priv->id.state != IB_CM_ESTABLISHED) {
2592 spin_unlock_irq(&cm_id_priv->lock);
2593 goto out;
2594 }
2595
2596 ib_cancel_mad(cm_id_priv->msg);
2597 cm_queue_work_unlock(cm_id_priv, work);
2598 return 0;
2599 out:
2600 cm_deref_id(cm_id_priv);
2601 return -EINVAL;
2602 }
2603
cm_rtu_handler(struct cm_work * work)2604 static int cm_rtu_handler(struct cm_work *work)
2605 {
2606 struct cm_id_private *cm_id_priv;
2607 struct cm_rtu_msg *rtu_msg;
2608
2609 rtu_msg = (struct cm_rtu_msg *)work->mad_recv_wc->recv_buf.mad;
2610 cm_id_priv = cm_acquire_id(
2611 cpu_to_be32(IBA_GET(CM_RTU_REMOTE_COMM_ID, rtu_msg)),
2612 cpu_to_be32(IBA_GET(CM_RTU_LOCAL_COMM_ID, rtu_msg)));
2613 if (!cm_id_priv)
2614 return -EINVAL;
2615
2616 work->cm_event.private_data =
2617 IBA_GET_MEM_PTR(CM_RTU_PRIVATE_DATA, rtu_msg);
2618
2619 spin_lock_irq(&cm_id_priv->lock);
2620 if (cm_id_priv->id.state != IB_CM_REP_SENT &&
2621 cm_id_priv->id.state != IB_CM_MRA_REP_RCVD) {
2622 spin_unlock_irq(&cm_id_priv->lock);
2623 atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
2624 [CM_RTU_COUNTER]);
2625 goto out;
2626 }
2627 cm_id_priv->id.state = IB_CM_ESTABLISHED;
2628
2629 ib_cancel_mad(cm_id_priv->msg);
2630 cm_queue_work_unlock(cm_id_priv, work);
2631 return 0;
2632 out:
2633 cm_deref_id(cm_id_priv);
2634 return -EINVAL;
2635 }
2636
cm_format_dreq(struct cm_dreq_msg * dreq_msg,struct cm_id_private * cm_id_priv,const void * private_data,u8 private_data_len)2637 static void cm_format_dreq(struct cm_dreq_msg *dreq_msg,
2638 struct cm_id_private *cm_id_priv,
2639 const void *private_data,
2640 u8 private_data_len)
2641 {
2642 cm_format_mad_hdr(&dreq_msg->hdr, CM_DREQ_ATTR_ID,
2643 cm_form_tid(cm_id_priv));
2644 IBA_SET(CM_DREQ_LOCAL_COMM_ID, dreq_msg,
2645 be32_to_cpu(cm_id_priv->id.local_id));
2646 IBA_SET(CM_DREQ_REMOTE_COMM_ID, dreq_msg,
2647 be32_to_cpu(cm_id_priv->id.remote_id));
2648 IBA_SET(CM_DREQ_REMOTE_QPN_EECN, dreq_msg,
2649 be32_to_cpu(cm_id_priv->remote_qpn));
2650
2651 if (private_data && private_data_len)
2652 IBA_SET_MEM(CM_DREQ_PRIVATE_DATA, dreq_msg, private_data,
2653 private_data_len);
2654 }
2655
cm_send_dreq_locked(struct cm_id_private * cm_id_priv,const void * private_data,u8 private_data_len)2656 static int cm_send_dreq_locked(struct cm_id_private *cm_id_priv,
2657 const void *private_data, u8 private_data_len)
2658 {
2659 struct ib_mad_send_buf *msg;
2660 int ret;
2661
2662 lockdep_assert_held(&cm_id_priv->lock);
2663
2664 if (private_data && private_data_len > IB_CM_DREQ_PRIVATE_DATA_SIZE)
2665 return -EINVAL;
2666
2667 if (cm_id_priv->id.state != IB_CM_ESTABLISHED) {
2668 trace_icm_dreq_skipped(&cm_id_priv->id);
2669 return -EINVAL;
2670 }
2671
2672 if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT ||
2673 cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
2674 ib_cancel_mad(cm_id_priv->msg);
2675
2676 msg = cm_alloc_priv_msg(cm_id_priv);
2677 if (IS_ERR(msg)) {
2678 cm_enter_timewait(cm_id_priv);
2679 return PTR_ERR(msg);
2680 }
2681
2682 cm_format_dreq((struct cm_dreq_msg *) msg->mad, cm_id_priv,
2683 private_data, private_data_len);
2684 msg->timeout_ms = cm_id_priv->timeout_ms;
2685 msg->context[1] = (void *) (unsigned long) IB_CM_DREQ_SENT;
2686
2687 trace_icm_send_dreq(&cm_id_priv->id);
2688 ret = ib_post_send_mad(msg, NULL);
2689 if (ret) {
2690 cm_enter_timewait(cm_id_priv);
2691 cm_free_priv_msg(msg);
2692 return ret;
2693 }
2694
2695 cm_id_priv->id.state = IB_CM_DREQ_SENT;
2696 return 0;
2697 }
2698
ib_send_cm_dreq(struct ib_cm_id * cm_id,const void * private_data,u8 private_data_len)2699 int ib_send_cm_dreq(struct ib_cm_id *cm_id, const void *private_data,
2700 u8 private_data_len)
2701 {
2702 struct cm_id_private *cm_id_priv =
2703 container_of(cm_id, struct cm_id_private, id);
2704 unsigned long flags;
2705 int ret;
2706
2707 spin_lock_irqsave(&cm_id_priv->lock, flags);
2708 ret = cm_send_dreq_locked(cm_id_priv, private_data, private_data_len);
2709 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2710 return ret;
2711 }
2712 EXPORT_SYMBOL(ib_send_cm_dreq);
2713
cm_format_drep(struct cm_drep_msg * drep_msg,struct cm_id_private * cm_id_priv,const void * private_data,u8 private_data_len)2714 static void cm_format_drep(struct cm_drep_msg *drep_msg,
2715 struct cm_id_private *cm_id_priv,
2716 const void *private_data,
2717 u8 private_data_len)
2718 {
2719 cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, cm_id_priv->tid);
2720 IBA_SET(CM_DREP_LOCAL_COMM_ID, drep_msg,
2721 be32_to_cpu(cm_id_priv->id.local_id));
2722 IBA_SET(CM_DREP_REMOTE_COMM_ID, drep_msg,
2723 be32_to_cpu(cm_id_priv->id.remote_id));
2724
2725 if (private_data && private_data_len)
2726 IBA_SET_MEM(CM_DREP_PRIVATE_DATA, drep_msg, private_data,
2727 private_data_len);
2728 }
2729
cm_send_drep_locked(struct cm_id_private * cm_id_priv,void * private_data,u8 private_data_len)2730 static int cm_send_drep_locked(struct cm_id_private *cm_id_priv,
2731 void *private_data, u8 private_data_len)
2732 {
2733 struct ib_mad_send_buf *msg;
2734 int ret;
2735
2736 lockdep_assert_held(&cm_id_priv->lock);
2737
2738 if (private_data && private_data_len > IB_CM_DREP_PRIVATE_DATA_SIZE)
2739 return -EINVAL;
2740
2741 if (cm_id_priv->id.state != IB_CM_DREQ_RCVD) {
2742 trace_icm_send_drep_err(&cm_id_priv->id);
2743 kfree(private_data);
2744 return -EINVAL;
2745 }
2746
2747 cm_set_private_data(cm_id_priv, private_data, private_data_len);
2748 cm_enter_timewait(cm_id_priv);
2749
2750 msg = cm_alloc_msg(cm_id_priv);
2751 if (IS_ERR(msg))
2752 return PTR_ERR(msg);
2753
2754 cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv,
2755 private_data, private_data_len);
2756
2757 trace_icm_send_drep(&cm_id_priv->id);
2758 ret = ib_post_send_mad(msg, NULL);
2759 if (ret) {
2760 cm_free_msg(msg);
2761 return ret;
2762 }
2763 return 0;
2764 }
2765
ib_send_cm_drep(struct ib_cm_id * cm_id,const void * private_data,u8 private_data_len)2766 int ib_send_cm_drep(struct ib_cm_id *cm_id, const void *private_data,
2767 u8 private_data_len)
2768 {
2769 struct cm_id_private *cm_id_priv =
2770 container_of(cm_id, struct cm_id_private, id);
2771 unsigned long flags;
2772 void *data;
2773 int ret;
2774
2775 data = cm_copy_private_data(private_data, private_data_len);
2776 if (IS_ERR(data))
2777 return PTR_ERR(data);
2778
2779 spin_lock_irqsave(&cm_id_priv->lock, flags);
2780 ret = cm_send_drep_locked(cm_id_priv, data, private_data_len);
2781 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2782 return ret;
2783 }
2784 EXPORT_SYMBOL(ib_send_cm_drep);
2785
cm_issue_drep(struct cm_port * port,struct ib_mad_recv_wc * mad_recv_wc)2786 static int cm_issue_drep(struct cm_port *port,
2787 struct ib_mad_recv_wc *mad_recv_wc)
2788 {
2789 struct ib_mad_send_buf *msg = NULL;
2790 struct cm_dreq_msg *dreq_msg;
2791 struct cm_drep_msg *drep_msg;
2792 int ret;
2793
2794 ret = cm_alloc_response_msg(port, mad_recv_wc, &msg);
2795 if (ret)
2796 return ret;
2797
2798 dreq_msg = (struct cm_dreq_msg *) mad_recv_wc->recv_buf.mad;
2799 drep_msg = (struct cm_drep_msg *) msg->mad;
2800
2801 cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, dreq_msg->hdr.tid);
2802 IBA_SET(CM_DREP_REMOTE_COMM_ID, drep_msg,
2803 IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg));
2804 IBA_SET(CM_DREP_LOCAL_COMM_ID, drep_msg,
2805 IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg));
2806
2807 trace_icm_issue_drep(
2808 IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg),
2809 IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg));
2810 ret = ib_post_send_mad(msg, NULL);
2811 if (ret)
2812 cm_free_response_msg(msg);
2813
2814 return ret;
2815 }
2816
cm_dreq_handler(struct cm_work * work)2817 static int cm_dreq_handler(struct cm_work *work)
2818 {
2819 struct cm_id_private *cm_id_priv;
2820 struct cm_dreq_msg *dreq_msg;
2821 struct ib_mad_send_buf *msg = NULL;
2822
2823 dreq_msg = (struct cm_dreq_msg *)work->mad_recv_wc->recv_buf.mad;
2824 cm_id_priv = cm_acquire_id(
2825 cpu_to_be32(IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg)),
2826 cpu_to_be32(IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg)));
2827 if (!cm_id_priv) {
2828 atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
2829 [CM_DREQ_COUNTER]);
2830 cm_issue_drep(work->port, work->mad_recv_wc);
2831 trace_icm_no_priv_err(
2832 IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg),
2833 IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg));
2834 return -EINVAL;
2835 }
2836
2837 work->cm_event.private_data =
2838 IBA_GET_MEM_PTR(CM_DREQ_PRIVATE_DATA, dreq_msg);
2839
2840 spin_lock_irq(&cm_id_priv->lock);
2841 if (cm_id_priv->local_qpn !=
2842 cpu_to_be32(IBA_GET(CM_DREQ_REMOTE_QPN_EECN, dreq_msg)))
2843 goto unlock;
2844
2845 switch (cm_id_priv->id.state) {
2846 case IB_CM_REP_SENT:
2847 case IB_CM_DREQ_SENT:
2848 case IB_CM_MRA_REP_RCVD:
2849 ib_cancel_mad(cm_id_priv->msg);
2850 break;
2851 case IB_CM_ESTABLISHED:
2852 if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT ||
2853 cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
2854 ib_cancel_mad(cm_id_priv->msg);
2855 break;
2856 case IB_CM_TIMEWAIT:
2857 atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
2858 [CM_DREQ_COUNTER]);
2859 msg = cm_alloc_response_msg_no_ah(work->port, work->mad_recv_wc);
2860 if (IS_ERR(msg))
2861 goto unlock;
2862
2863 cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv,
2864 cm_id_priv->private_data,
2865 cm_id_priv->private_data_len);
2866 spin_unlock_irq(&cm_id_priv->lock);
2867
2868 if (cm_create_response_msg_ah(work->port, work->mad_recv_wc, msg) ||
2869 ib_post_send_mad(msg, NULL))
2870 cm_free_response_msg(msg);
2871 goto deref;
2872 case IB_CM_DREQ_RCVD:
2873 atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
2874 [CM_DREQ_COUNTER]);
2875 goto unlock;
2876 default:
2877 trace_icm_dreq_unknown_err(&cm_id_priv->id);
2878 goto unlock;
2879 }
2880 cm_id_priv->id.state = IB_CM_DREQ_RCVD;
2881 cm_id_priv->tid = dreq_msg->hdr.tid;
2882 cm_queue_work_unlock(cm_id_priv, work);
2883 return 0;
2884
2885 unlock: spin_unlock_irq(&cm_id_priv->lock);
2886 deref: cm_deref_id(cm_id_priv);
2887 return -EINVAL;
2888 }
2889
cm_drep_handler(struct cm_work * work)2890 static int cm_drep_handler(struct cm_work *work)
2891 {
2892 struct cm_id_private *cm_id_priv;
2893 struct cm_drep_msg *drep_msg;
2894
2895 drep_msg = (struct cm_drep_msg *)work->mad_recv_wc->recv_buf.mad;
2896 cm_id_priv = cm_acquire_id(
2897 cpu_to_be32(IBA_GET(CM_DREP_REMOTE_COMM_ID, drep_msg)),
2898 cpu_to_be32(IBA_GET(CM_DREP_LOCAL_COMM_ID, drep_msg)));
2899 if (!cm_id_priv)
2900 return -EINVAL;
2901
2902 work->cm_event.private_data =
2903 IBA_GET_MEM_PTR(CM_DREP_PRIVATE_DATA, drep_msg);
2904
2905 spin_lock_irq(&cm_id_priv->lock);
2906 if (cm_id_priv->id.state != IB_CM_DREQ_SENT &&
2907 cm_id_priv->id.state != IB_CM_DREQ_RCVD) {
2908 spin_unlock_irq(&cm_id_priv->lock);
2909 goto out;
2910 }
2911 cm_enter_timewait(cm_id_priv);
2912
2913 ib_cancel_mad(cm_id_priv->msg);
2914 cm_queue_work_unlock(cm_id_priv, work);
2915 return 0;
2916 out:
2917 cm_deref_id(cm_id_priv);
2918 return -EINVAL;
2919 }
2920
cm_send_rej_locked(struct cm_id_private * cm_id_priv,enum ib_cm_rej_reason reason,void * ari,u8 ari_length,const void * private_data,u8 private_data_len)2921 static int cm_send_rej_locked(struct cm_id_private *cm_id_priv,
2922 enum ib_cm_rej_reason reason, void *ari,
2923 u8 ari_length, const void *private_data,
2924 u8 private_data_len)
2925 {
2926 enum ib_cm_state state = cm_id_priv->id.state;
2927 struct ib_mad_send_buf *msg;
2928 int ret;
2929
2930 lockdep_assert_held(&cm_id_priv->lock);
2931
2932 if ((private_data && private_data_len > IB_CM_REJ_PRIVATE_DATA_SIZE) ||
2933 (ari && ari_length > IB_CM_REJ_ARI_LENGTH))
2934 return -EINVAL;
2935
2936 trace_icm_send_rej(&cm_id_priv->id, reason);
2937
2938 switch (state) {
2939 case IB_CM_REQ_SENT:
2940 case IB_CM_MRA_REQ_RCVD:
2941 case IB_CM_REQ_RCVD:
2942 case IB_CM_MRA_REQ_SENT:
2943 case IB_CM_REP_RCVD:
2944 case IB_CM_MRA_REP_SENT:
2945 cm_reset_to_idle(cm_id_priv);
2946 msg = cm_alloc_msg(cm_id_priv);
2947 if (IS_ERR(msg))
2948 return PTR_ERR(msg);
2949 cm_format_rej((struct cm_rej_msg *)msg->mad, cm_id_priv, reason,
2950 ari, ari_length, private_data, private_data_len,
2951 state);
2952 break;
2953 case IB_CM_REP_SENT:
2954 case IB_CM_MRA_REP_RCVD:
2955 cm_enter_timewait(cm_id_priv);
2956 msg = cm_alloc_msg(cm_id_priv);
2957 if (IS_ERR(msg))
2958 return PTR_ERR(msg);
2959 cm_format_rej((struct cm_rej_msg *)msg->mad, cm_id_priv, reason,
2960 ari, ari_length, private_data, private_data_len,
2961 state);
2962 break;
2963 default:
2964 trace_icm_send_unknown_rej_err(&cm_id_priv->id);
2965 return -EINVAL;
2966 }
2967
2968 ret = ib_post_send_mad(msg, NULL);
2969 if (ret) {
2970 cm_free_msg(msg);
2971 return ret;
2972 }
2973
2974 return 0;
2975 }
2976
ib_send_cm_rej(struct ib_cm_id * cm_id,enum ib_cm_rej_reason reason,void * ari,u8 ari_length,const void * private_data,u8 private_data_len)2977 int ib_send_cm_rej(struct ib_cm_id *cm_id, enum ib_cm_rej_reason reason,
2978 void *ari, u8 ari_length, const void *private_data,
2979 u8 private_data_len)
2980 {
2981 struct cm_id_private *cm_id_priv =
2982 container_of(cm_id, struct cm_id_private, id);
2983 unsigned long flags;
2984 int ret;
2985
2986 spin_lock_irqsave(&cm_id_priv->lock, flags);
2987 ret = cm_send_rej_locked(cm_id_priv, reason, ari, ari_length,
2988 private_data, private_data_len);
2989 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2990 return ret;
2991 }
2992 EXPORT_SYMBOL(ib_send_cm_rej);
2993
cm_format_rej_event(struct cm_work * work)2994 static void cm_format_rej_event(struct cm_work *work)
2995 {
2996 struct cm_rej_msg *rej_msg;
2997 struct ib_cm_rej_event_param *param;
2998
2999 rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad;
3000 param = &work->cm_event.param.rej_rcvd;
3001 param->ari = IBA_GET_MEM_PTR(CM_REJ_ARI, rej_msg);
3002 param->ari_length = IBA_GET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg);
3003 param->reason = IBA_GET(CM_REJ_REASON, rej_msg);
3004 work->cm_event.private_data =
3005 IBA_GET_MEM_PTR(CM_REJ_PRIVATE_DATA, rej_msg);
3006 }
3007
cm_acquire_rejected_id(struct cm_rej_msg * rej_msg)3008 static struct cm_id_private *cm_acquire_rejected_id(struct cm_rej_msg *rej_msg)
3009 {
3010 struct cm_id_private *cm_id_priv;
3011 __be32 remote_id;
3012
3013 remote_id = cpu_to_be32(IBA_GET(CM_REJ_LOCAL_COMM_ID, rej_msg));
3014
3015 if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_TIMEOUT) {
3016 cm_id_priv = cm_find_remote_id(
3017 *((__be64 *)IBA_GET_MEM_PTR(CM_REJ_ARI, rej_msg)),
3018 remote_id);
3019 } else if (IBA_GET(CM_REJ_MESSAGE_REJECTED, rej_msg) ==
3020 CM_MSG_RESPONSE_REQ)
3021 cm_id_priv = cm_acquire_id(
3022 cpu_to_be32(IBA_GET(CM_REJ_REMOTE_COMM_ID, rej_msg)),
3023 0);
3024 else
3025 cm_id_priv = cm_acquire_id(
3026 cpu_to_be32(IBA_GET(CM_REJ_REMOTE_COMM_ID, rej_msg)),
3027 remote_id);
3028
3029 return cm_id_priv;
3030 }
3031
cm_rej_handler(struct cm_work * work)3032 static int cm_rej_handler(struct cm_work *work)
3033 {
3034 struct cm_id_private *cm_id_priv;
3035 struct cm_rej_msg *rej_msg;
3036
3037 rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad;
3038 cm_id_priv = cm_acquire_rejected_id(rej_msg);
3039 if (!cm_id_priv)
3040 return -EINVAL;
3041
3042 cm_format_rej_event(work);
3043
3044 spin_lock_irq(&cm_id_priv->lock);
3045 switch (cm_id_priv->id.state) {
3046 case IB_CM_REQ_SENT:
3047 case IB_CM_MRA_REQ_RCVD:
3048 case IB_CM_REP_SENT:
3049 case IB_CM_MRA_REP_RCVD:
3050 ib_cancel_mad(cm_id_priv->msg);
3051 fallthrough;
3052 case IB_CM_REQ_RCVD:
3053 case IB_CM_MRA_REQ_SENT:
3054 if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_STALE_CONN)
3055 cm_enter_timewait(cm_id_priv);
3056 else
3057 cm_reset_to_idle(cm_id_priv);
3058 break;
3059 case IB_CM_DREQ_SENT:
3060 ib_cancel_mad(cm_id_priv->msg);
3061 fallthrough;
3062 case IB_CM_REP_RCVD:
3063 case IB_CM_MRA_REP_SENT:
3064 cm_enter_timewait(cm_id_priv);
3065 break;
3066 case IB_CM_ESTABLISHED:
3067 if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT ||
3068 cm_id_priv->id.lap_state == IB_CM_LAP_SENT) {
3069 if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT)
3070 ib_cancel_mad(cm_id_priv->msg);
3071 cm_enter_timewait(cm_id_priv);
3072 break;
3073 }
3074 fallthrough;
3075 default:
3076 trace_icm_rej_unknown_err(&cm_id_priv->id);
3077 spin_unlock_irq(&cm_id_priv->lock);
3078 goto out;
3079 }
3080
3081 cm_queue_work_unlock(cm_id_priv, work);
3082 return 0;
3083 out:
3084 cm_deref_id(cm_id_priv);
3085 return -EINVAL;
3086 }
3087
ib_send_cm_mra(struct ib_cm_id * cm_id,u8 service_timeout,const void * private_data,u8 private_data_len)3088 int ib_send_cm_mra(struct ib_cm_id *cm_id,
3089 u8 service_timeout,
3090 const void *private_data,
3091 u8 private_data_len)
3092 {
3093 struct cm_id_private *cm_id_priv;
3094 struct ib_mad_send_buf *msg;
3095 enum ib_cm_state cm_state;
3096 enum ib_cm_lap_state lap_state;
3097 enum cm_msg_response msg_response;
3098 void *data;
3099 unsigned long flags;
3100 int ret;
3101
3102 if (private_data && private_data_len > IB_CM_MRA_PRIVATE_DATA_SIZE)
3103 return -EINVAL;
3104
3105 data = cm_copy_private_data(private_data, private_data_len);
3106 if (IS_ERR(data))
3107 return PTR_ERR(data);
3108
3109 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3110
3111 spin_lock_irqsave(&cm_id_priv->lock, flags);
3112 switch (cm_id_priv->id.state) {
3113 case IB_CM_REQ_RCVD:
3114 cm_state = IB_CM_MRA_REQ_SENT;
3115 lap_state = cm_id->lap_state;
3116 msg_response = CM_MSG_RESPONSE_REQ;
3117 break;
3118 case IB_CM_REP_RCVD:
3119 cm_state = IB_CM_MRA_REP_SENT;
3120 lap_state = cm_id->lap_state;
3121 msg_response = CM_MSG_RESPONSE_REP;
3122 break;
3123 case IB_CM_ESTABLISHED:
3124 if (cm_id->lap_state == IB_CM_LAP_RCVD) {
3125 cm_state = cm_id->state;
3126 lap_state = IB_CM_MRA_LAP_SENT;
3127 msg_response = CM_MSG_RESPONSE_OTHER;
3128 break;
3129 }
3130 fallthrough;
3131 default:
3132 trace_icm_send_mra_unknown_err(&cm_id_priv->id);
3133 ret = -EINVAL;
3134 goto error_unlock;
3135 }
3136
3137 if (!(service_timeout & IB_CM_MRA_FLAG_DELAY)) {
3138 msg = cm_alloc_msg(cm_id_priv);
3139 if (IS_ERR(msg)) {
3140 ret = PTR_ERR(msg);
3141 goto error_unlock;
3142 }
3143
3144 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
3145 msg_response, service_timeout,
3146 private_data, private_data_len);
3147 trace_icm_send_mra(cm_id);
3148 ret = ib_post_send_mad(msg, NULL);
3149 if (ret)
3150 goto error_free_msg;
3151 }
3152
3153 cm_id->state = cm_state;
3154 cm_id->lap_state = lap_state;
3155 cm_id_priv->service_timeout = service_timeout;
3156 cm_set_private_data(cm_id_priv, data, private_data_len);
3157 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3158 return 0;
3159
3160 error_free_msg:
3161 cm_free_msg(msg);
3162 error_unlock:
3163 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3164 kfree(data);
3165 return ret;
3166 }
3167 EXPORT_SYMBOL(ib_send_cm_mra);
3168
cm_acquire_mraed_id(struct cm_mra_msg * mra_msg)3169 static struct cm_id_private *cm_acquire_mraed_id(struct cm_mra_msg *mra_msg)
3170 {
3171 switch (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg)) {
3172 case CM_MSG_RESPONSE_REQ:
3173 return cm_acquire_id(
3174 cpu_to_be32(IBA_GET(CM_MRA_REMOTE_COMM_ID, mra_msg)),
3175 0);
3176 case CM_MSG_RESPONSE_REP:
3177 case CM_MSG_RESPONSE_OTHER:
3178 return cm_acquire_id(
3179 cpu_to_be32(IBA_GET(CM_MRA_REMOTE_COMM_ID, mra_msg)),
3180 cpu_to_be32(IBA_GET(CM_MRA_LOCAL_COMM_ID, mra_msg)));
3181 default:
3182 return NULL;
3183 }
3184 }
3185
cm_mra_handler(struct cm_work * work)3186 static int cm_mra_handler(struct cm_work *work)
3187 {
3188 struct cm_id_private *cm_id_priv;
3189 struct cm_mra_msg *mra_msg;
3190 int timeout;
3191
3192 mra_msg = (struct cm_mra_msg *)work->mad_recv_wc->recv_buf.mad;
3193 cm_id_priv = cm_acquire_mraed_id(mra_msg);
3194 if (!cm_id_priv)
3195 return -EINVAL;
3196
3197 work->cm_event.private_data =
3198 IBA_GET_MEM_PTR(CM_MRA_PRIVATE_DATA, mra_msg);
3199 work->cm_event.param.mra_rcvd.service_timeout =
3200 IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg);
3201 timeout = cm_convert_to_ms(IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg)) +
3202 cm_convert_to_ms(cm_id_priv->av.timeout);
3203
3204 spin_lock_irq(&cm_id_priv->lock);
3205 switch (cm_id_priv->id.state) {
3206 case IB_CM_REQ_SENT:
3207 if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=
3208 CM_MSG_RESPONSE_REQ ||
3209 ib_modify_mad(cm_id_priv->msg, timeout))
3210 goto out;
3211 cm_id_priv->id.state = IB_CM_MRA_REQ_RCVD;
3212 break;
3213 case IB_CM_REP_SENT:
3214 if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=
3215 CM_MSG_RESPONSE_REP ||
3216 ib_modify_mad(cm_id_priv->msg, timeout))
3217 goto out;
3218 cm_id_priv->id.state = IB_CM_MRA_REP_RCVD;
3219 break;
3220 case IB_CM_ESTABLISHED:
3221 if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=
3222 CM_MSG_RESPONSE_OTHER ||
3223 cm_id_priv->id.lap_state != IB_CM_LAP_SENT ||
3224 ib_modify_mad(cm_id_priv->msg, timeout)) {
3225 if (cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
3226 atomic_long_inc(
3227 &work->port->counters[CM_RECV_DUPLICATES]
3228 [CM_MRA_COUNTER]);
3229 goto out;
3230 }
3231 cm_id_priv->id.lap_state = IB_CM_MRA_LAP_RCVD;
3232 break;
3233 case IB_CM_MRA_REQ_RCVD:
3234 case IB_CM_MRA_REP_RCVD:
3235 atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
3236 [CM_MRA_COUNTER]);
3237 fallthrough;
3238 default:
3239 trace_icm_mra_unknown_err(&cm_id_priv->id);
3240 goto out;
3241 }
3242
3243 cm_id_priv->msg->context[1] = (void *) (unsigned long)
3244 cm_id_priv->id.state;
3245 cm_queue_work_unlock(cm_id_priv, work);
3246 return 0;
3247 out:
3248 spin_unlock_irq(&cm_id_priv->lock);
3249 cm_deref_id(cm_id_priv);
3250 return -EINVAL;
3251 }
3252
cm_format_path_lid_from_lap(struct cm_lap_msg * lap_msg,struct sa_path_rec * path)3253 static void cm_format_path_lid_from_lap(struct cm_lap_msg *lap_msg,
3254 struct sa_path_rec *path)
3255 {
3256 u32 lid;
3257
3258 if (path->rec_type != SA_PATH_REC_TYPE_OPA) {
3259 sa_path_set_dlid(path, IBA_GET(CM_LAP_ALTERNATE_LOCAL_PORT_LID,
3260 lap_msg));
3261 sa_path_set_slid(path, IBA_GET(CM_LAP_ALTERNATE_REMOTE_PORT_LID,
3262 lap_msg));
3263 } else {
3264 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
3265 CM_LAP_ALTERNATE_LOCAL_PORT_GID, lap_msg));
3266 sa_path_set_dlid(path, lid);
3267
3268 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
3269 CM_LAP_ALTERNATE_REMOTE_PORT_GID, lap_msg));
3270 sa_path_set_slid(path, lid);
3271 }
3272 }
3273
cm_format_path_from_lap(struct cm_id_private * cm_id_priv,struct sa_path_rec * path,struct cm_lap_msg * lap_msg)3274 static void cm_format_path_from_lap(struct cm_id_private *cm_id_priv,
3275 struct sa_path_rec *path,
3276 struct cm_lap_msg *lap_msg)
3277 {
3278 path->dgid = *IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_LOCAL_PORT_GID, lap_msg);
3279 path->sgid =
3280 *IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_REMOTE_PORT_GID, lap_msg);
3281 path->flow_label =
3282 cpu_to_be32(IBA_GET(CM_LAP_ALTERNATE_FLOW_LABEL, lap_msg));
3283 path->hop_limit = IBA_GET(CM_LAP_ALTERNATE_HOP_LIMIT, lap_msg);
3284 path->traffic_class = IBA_GET(CM_LAP_ALTERNATE_TRAFFIC_CLASS, lap_msg);
3285 path->reversible = 1;
3286 path->pkey = cm_id_priv->pkey;
3287 path->sl = IBA_GET(CM_LAP_ALTERNATE_SL, lap_msg);
3288 path->mtu_selector = IB_SA_EQ;
3289 path->mtu = cm_id_priv->path_mtu;
3290 path->rate_selector = IB_SA_EQ;
3291 path->rate = IBA_GET(CM_LAP_ALTERNATE_PACKET_RATE, lap_msg);
3292 path->packet_life_time_selector = IB_SA_EQ;
3293 path->packet_life_time =
3294 IBA_GET(CM_LAP_ALTERNATE_LOCAL_ACK_TIMEOUT, lap_msg);
3295 path->packet_life_time -= (path->packet_life_time > 0);
3296 cm_format_path_lid_from_lap(lap_msg, path);
3297 }
3298
cm_lap_handler(struct cm_work * work)3299 static int cm_lap_handler(struct cm_work *work)
3300 {
3301 struct cm_id_private *cm_id_priv;
3302 struct cm_lap_msg *lap_msg;
3303 struct ib_cm_lap_event_param *param;
3304 struct ib_mad_send_buf *msg = NULL;
3305 struct rdma_ah_attr ah_attr;
3306 struct cm_av alt_av = {};
3307 int ret;
3308
3309 /* Currently Alternate path messages are not supported for
3310 * RoCE link layer.
3311 */
3312 if (rdma_protocol_roce(work->port->cm_dev->ib_device,
3313 work->port->port_num))
3314 return -EINVAL;
3315
3316 /* todo: verify LAP request and send reject APR if invalid. */
3317 lap_msg = (struct cm_lap_msg *)work->mad_recv_wc->recv_buf.mad;
3318 cm_id_priv = cm_acquire_id(
3319 cpu_to_be32(IBA_GET(CM_LAP_REMOTE_COMM_ID, lap_msg)),
3320 cpu_to_be32(IBA_GET(CM_LAP_LOCAL_COMM_ID, lap_msg)));
3321 if (!cm_id_priv)
3322 return -EINVAL;
3323
3324 param = &work->cm_event.param.lap_rcvd;
3325 memset(&work->path[0], 0, sizeof(work->path[1]));
3326 cm_path_set_rec_type(work->port->cm_dev->ib_device,
3327 work->port->port_num, &work->path[0],
3328 IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_LOCAL_PORT_GID,
3329 lap_msg));
3330 param->alternate_path = &work->path[0];
3331 cm_format_path_from_lap(cm_id_priv, param->alternate_path, lap_msg);
3332 work->cm_event.private_data =
3333 IBA_GET_MEM_PTR(CM_LAP_PRIVATE_DATA, lap_msg);
3334
3335 ret = ib_init_ah_attr_from_wc(work->port->cm_dev->ib_device,
3336 work->port->port_num,
3337 work->mad_recv_wc->wc,
3338 work->mad_recv_wc->recv_buf.grh,
3339 &ah_attr);
3340 if (ret)
3341 goto deref;
3342
3343 ret = cm_init_av_by_path(param->alternate_path, NULL, &alt_av);
3344 if (ret) {
3345 rdma_destroy_ah_attr(&ah_attr);
3346 goto deref;
3347 }
3348
3349 spin_lock_irq(&cm_id_priv->lock);
3350 cm_init_av_for_lap(work->port, work->mad_recv_wc->wc,
3351 &ah_attr, &cm_id_priv->av);
3352 cm_move_av_from_path(&cm_id_priv->alt_av, &alt_av);
3353
3354 if (cm_id_priv->id.state != IB_CM_ESTABLISHED)
3355 goto unlock;
3356
3357 switch (cm_id_priv->id.lap_state) {
3358 case IB_CM_LAP_UNINIT:
3359 case IB_CM_LAP_IDLE:
3360 break;
3361 case IB_CM_MRA_LAP_SENT:
3362 atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
3363 [CM_LAP_COUNTER]);
3364 msg = cm_alloc_response_msg_no_ah(work->port, work->mad_recv_wc);
3365 if (IS_ERR(msg))
3366 goto unlock;
3367
3368 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
3369 CM_MSG_RESPONSE_OTHER,
3370 cm_id_priv->service_timeout,
3371 cm_id_priv->private_data,
3372 cm_id_priv->private_data_len);
3373 spin_unlock_irq(&cm_id_priv->lock);
3374
3375 if (cm_create_response_msg_ah(work->port, work->mad_recv_wc, msg) ||
3376 ib_post_send_mad(msg, NULL))
3377 cm_free_response_msg(msg);
3378 goto deref;
3379 case IB_CM_LAP_RCVD:
3380 atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
3381 [CM_LAP_COUNTER]);
3382 goto unlock;
3383 default:
3384 goto unlock;
3385 }
3386
3387 cm_id_priv->id.lap_state = IB_CM_LAP_RCVD;
3388 cm_id_priv->tid = lap_msg->hdr.tid;
3389 cm_queue_work_unlock(cm_id_priv, work);
3390 return 0;
3391
3392 unlock: spin_unlock_irq(&cm_id_priv->lock);
3393 deref: cm_deref_id(cm_id_priv);
3394 return -EINVAL;
3395 }
3396
cm_apr_handler(struct cm_work * work)3397 static int cm_apr_handler(struct cm_work *work)
3398 {
3399 struct cm_id_private *cm_id_priv;
3400 struct cm_apr_msg *apr_msg;
3401
3402 /* Currently Alternate path messages are not supported for
3403 * RoCE link layer.
3404 */
3405 if (rdma_protocol_roce(work->port->cm_dev->ib_device,
3406 work->port->port_num))
3407 return -EINVAL;
3408
3409 apr_msg = (struct cm_apr_msg *)work->mad_recv_wc->recv_buf.mad;
3410 cm_id_priv = cm_acquire_id(
3411 cpu_to_be32(IBA_GET(CM_APR_REMOTE_COMM_ID, apr_msg)),
3412 cpu_to_be32(IBA_GET(CM_APR_LOCAL_COMM_ID, apr_msg)));
3413 if (!cm_id_priv)
3414 return -EINVAL; /* Unmatched reply. */
3415
3416 work->cm_event.param.apr_rcvd.ap_status =
3417 IBA_GET(CM_APR_AR_STATUS, apr_msg);
3418 work->cm_event.param.apr_rcvd.apr_info =
3419 IBA_GET_MEM_PTR(CM_APR_ADDITIONAL_INFORMATION, apr_msg);
3420 work->cm_event.param.apr_rcvd.info_len =
3421 IBA_GET(CM_APR_ADDITIONAL_INFORMATION_LENGTH, apr_msg);
3422 work->cm_event.private_data =
3423 IBA_GET_MEM_PTR(CM_APR_PRIVATE_DATA, apr_msg);
3424
3425 spin_lock_irq(&cm_id_priv->lock);
3426 if (cm_id_priv->id.state != IB_CM_ESTABLISHED ||
3427 (cm_id_priv->id.lap_state != IB_CM_LAP_SENT &&
3428 cm_id_priv->id.lap_state != IB_CM_MRA_LAP_RCVD)) {
3429 spin_unlock_irq(&cm_id_priv->lock);
3430 goto out;
3431 }
3432 cm_id_priv->id.lap_state = IB_CM_LAP_IDLE;
3433 ib_cancel_mad(cm_id_priv->msg);
3434 cm_queue_work_unlock(cm_id_priv, work);
3435 return 0;
3436 out:
3437 cm_deref_id(cm_id_priv);
3438 return -EINVAL;
3439 }
3440
cm_timewait_handler(struct cm_work * work)3441 static int cm_timewait_handler(struct cm_work *work)
3442 {
3443 struct cm_timewait_info *timewait_info;
3444 struct cm_id_private *cm_id_priv;
3445
3446 timewait_info = container_of(work, struct cm_timewait_info, work);
3447 spin_lock_irq(&cm.lock);
3448 list_del(&timewait_info->list);
3449 spin_unlock_irq(&cm.lock);
3450
3451 cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
3452 timewait_info->work.remote_id);
3453 if (!cm_id_priv)
3454 return -EINVAL;
3455
3456 spin_lock_irq(&cm_id_priv->lock);
3457 if (cm_id_priv->id.state != IB_CM_TIMEWAIT ||
3458 cm_id_priv->remote_qpn != timewait_info->remote_qpn) {
3459 spin_unlock_irq(&cm_id_priv->lock);
3460 goto out;
3461 }
3462 cm_id_priv->id.state = IB_CM_IDLE;
3463 cm_queue_work_unlock(cm_id_priv, work);
3464 return 0;
3465 out:
3466 cm_deref_id(cm_id_priv);
3467 return -EINVAL;
3468 }
3469
cm_format_sidr_req(struct cm_sidr_req_msg * sidr_req_msg,struct cm_id_private * cm_id_priv,struct ib_cm_sidr_req_param * param)3470 static void cm_format_sidr_req(struct cm_sidr_req_msg *sidr_req_msg,
3471 struct cm_id_private *cm_id_priv,
3472 struct ib_cm_sidr_req_param *param)
3473 {
3474 cm_format_mad_hdr(&sidr_req_msg->hdr, CM_SIDR_REQ_ATTR_ID,
3475 cm_form_tid(cm_id_priv));
3476 IBA_SET(CM_SIDR_REQ_REQUESTID, sidr_req_msg,
3477 be32_to_cpu(cm_id_priv->id.local_id));
3478 IBA_SET(CM_SIDR_REQ_PARTITION_KEY, sidr_req_msg,
3479 be16_to_cpu(param->path->pkey));
3480 IBA_SET(CM_SIDR_REQ_SERVICEID, sidr_req_msg,
3481 be64_to_cpu(param->service_id));
3482
3483 if (param->private_data && param->private_data_len)
3484 IBA_SET_MEM(CM_SIDR_REQ_PRIVATE_DATA, sidr_req_msg,
3485 param->private_data, param->private_data_len);
3486 }
3487
ib_send_cm_sidr_req(struct ib_cm_id * cm_id,struct ib_cm_sidr_req_param * param)3488 int ib_send_cm_sidr_req(struct ib_cm_id *cm_id,
3489 struct ib_cm_sidr_req_param *param)
3490 {
3491 struct cm_id_private *cm_id_priv;
3492 struct ib_mad_send_buf *msg;
3493 struct cm_av av = {};
3494 unsigned long flags;
3495 int ret;
3496
3497 if (!param->path || (param->private_data &&
3498 param->private_data_len > IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE))
3499 return -EINVAL;
3500
3501 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3502 ret = cm_init_av_by_path(param->path, param->sgid_attr, &av);
3503 if (ret)
3504 return ret;
3505
3506 spin_lock_irqsave(&cm_id_priv->lock, flags);
3507 cm_move_av_from_path(&cm_id_priv->av, &av);
3508 cm_id->service_id = param->service_id;
3509 cm_id_priv->timeout_ms = param->timeout_ms;
3510 cm_id_priv->max_cm_retries = param->max_cm_retries;
3511 if (cm_id->state != IB_CM_IDLE) {
3512 ret = -EINVAL;
3513 goto out_unlock;
3514 }
3515
3516 msg = cm_alloc_priv_msg(cm_id_priv);
3517 if (IS_ERR(msg)) {
3518 ret = PTR_ERR(msg);
3519 goto out_unlock;
3520 }
3521
3522 cm_format_sidr_req((struct cm_sidr_req_msg *)msg->mad, cm_id_priv,
3523 param);
3524 msg->timeout_ms = cm_id_priv->timeout_ms;
3525 msg->context[1] = (void *)(unsigned long)IB_CM_SIDR_REQ_SENT;
3526
3527 trace_icm_send_sidr_req(&cm_id_priv->id);
3528 ret = ib_post_send_mad(msg, NULL);
3529 if (ret)
3530 goto out_free;
3531 cm_id->state = IB_CM_SIDR_REQ_SENT;
3532 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3533 return 0;
3534 out_free:
3535 cm_free_priv_msg(msg);
3536 out_unlock:
3537 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3538 return ret;
3539 }
3540 EXPORT_SYMBOL(ib_send_cm_sidr_req);
3541
cm_format_sidr_req_event(struct cm_work * work,const struct cm_id_private * rx_cm_id,struct ib_cm_id * listen_id)3542 static void cm_format_sidr_req_event(struct cm_work *work,
3543 const struct cm_id_private *rx_cm_id,
3544 struct ib_cm_id *listen_id)
3545 {
3546 struct cm_sidr_req_msg *sidr_req_msg;
3547 struct ib_cm_sidr_req_event_param *param;
3548
3549 sidr_req_msg = (struct cm_sidr_req_msg *)
3550 work->mad_recv_wc->recv_buf.mad;
3551 param = &work->cm_event.param.sidr_req_rcvd;
3552 param->pkey = IBA_GET(CM_SIDR_REQ_PARTITION_KEY, sidr_req_msg);
3553 param->listen_id = listen_id;
3554 param->service_id =
3555 cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg));
3556 param->bth_pkey = cm_get_bth_pkey(work);
3557 param->port = work->port->port_num;
3558 param->sgid_attr = rx_cm_id->av.ah_attr.grh.sgid_attr;
3559 work->cm_event.private_data =
3560 IBA_GET_MEM_PTR(CM_SIDR_REQ_PRIVATE_DATA, sidr_req_msg);
3561 }
3562
cm_sidr_req_handler(struct cm_work * work)3563 static int cm_sidr_req_handler(struct cm_work *work)
3564 {
3565 struct cm_id_private *cm_id_priv, *listen_cm_id_priv;
3566 struct cm_sidr_req_msg *sidr_req_msg;
3567 struct ib_wc *wc;
3568 int ret;
3569
3570 cm_id_priv =
3571 cm_alloc_id_priv(work->port->cm_dev->ib_device, NULL, NULL);
3572 if (IS_ERR(cm_id_priv))
3573 return PTR_ERR(cm_id_priv);
3574
3575 /* Record SGID/SLID and request ID for lookup. */
3576 sidr_req_msg = (struct cm_sidr_req_msg *)
3577 work->mad_recv_wc->recv_buf.mad;
3578
3579 cm_id_priv->id.remote_id =
3580 cpu_to_be32(IBA_GET(CM_SIDR_REQ_REQUESTID, sidr_req_msg));
3581 cm_id_priv->id.service_id =
3582 cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg));
3583 cm_id_priv->tid = sidr_req_msg->hdr.tid;
3584
3585 wc = work->mad_recv_wc->wc;
3586 cm_id_priv->sidr_slid = wc->slid;
3587 ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
3588 work->mad_recv_wc->recv_buf.grh,
3589 &cm_id_priv->av);
3590 if (ret)
3591 goto out;
3592
3593 spin_lock_irq(&cm.lock);
3594 listen_cm_id_priv = cm_insert_remote_sidr(cm_id_priv);
3595 if (listen_cm_id_priv) {
3596 spin_unlock_irq(&cm.lock);
3597 atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
3598 [CM_SIDR_REQ_COUNTER]);
3599 goto out; /* Duplicate message. */
3600 }
3601 cm_id_priv->id.state = IB_CM_SIDR_REQ_RCVD;
3602 listen_cm_id_priv = cm_find_listen(cm_id_priv->id.device,
3603 cm_id_priv->id.service_id);
3604 if (!listen_cm_id_priv) {
3605 spin_unlock_irq(&cm.lock);
3606 ib_send_cm_sidr_rep(&cm_id_priv->id,
3607 &(struct ib_cm_sidr_rep_param){
3608 .status = IB_SIDR_UNSUPPORTED });
3609 goto out; /* No match. */
3610 }
3611 spin_unlock_irq(&cm.lock);
3612
3613 cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler;
3614 cm_id_priv->id.context = listen_cm_id_priv->id.context;
3615
3616 /*
3617 * A SIDR ID does not need to be in the xarray since it does not receive
3618 * mads, is not placed in the remote_id or remote_qpn rbtree, and does
3619 * not enter timewait.
3620 */
3621
3622 cm_format_sidr_req_event(work, cm_id_priv, &listen_cm_id_priv->id);
3623 ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &work->cm_event);
3624 cm_free_work(work);
3625 /*
3626 * A pointer to the listen_cm_id is held in the event, so this deref
3627 * must be after the event is delivered above.
3628 */
3629 cm_deref_id(listen_cm_id_priv);
3630 if (ret)
3631 cm_destroy_id(&cm_id_priv->id, ret);
3632 return 0;
3633 out:
3634 ib_destroy_cm_id(&cm_id_priv->id);
3635 return -EINVAL;
3636 }
3637
cm_format_sidr_rep(struct cm_sidr_rep_msg * sidr_rep_msg,struct cm_id_private * cm_id_priv,struct ib_cm_sidr_rep_param * param)3638 static void cm_format_sidr_rep(struct cm_sidr_rep_msg *sidr_rep_msg,
3639 struct cm_id_private *cm_id_priv,
3640 struct ib_cm_sidr_rep_param *param)
3641 {
3642 cm_format_mad_ece_hdr(&sidr_rep_msg->hdr, CM_SIDR_REP_ATTR_ID,
3643 cm_id_priv->tid, param->ece.attr_mod);
3644 IBA_SET(CM_SIDR_REP_REQUESTID, sidr_rep_msg,
3645 be32_to_cpu(cm_id_priv->id.remote_id));
3646 IBA_SET(CM_SIDR_REP_STATUS, sidr_rep_msg, param->status);
3647 IBA_SET(CM_SIDR_REP_QPN, sidr_rep_msg, param->qp_num);
3648 IBA_SET(CM_SIDR_REP_SERVICEID, sidr_rep_msg,
3649 be64_to_cpu(cm_id_priv->id.service_id));
3650 IBA_SET(CM_SIDR_REP_Q_KEY, sidr_rep_msg, param->qkey);
3651 IBA_SET(CM_SIDR_REP_VENDOR_ID_L, sidr_rep_msg,
3652 param->ece.vendor_id & 0xFF);
3653 IBA_SET(CM_SIDR_REP_VENDOR_ID_H, sidr_rep_msg,
3654 (param->ece.vendor_id >> 8) & 0xFF);
3655
3656 if (param->info && param->info_length)
3657 IBA_SET_MEM(CM_SIDR_REP_ADDITIONAL_INFORMATION, sidr_rep_msg,
3658 param->info, param->info_length);
3659
3660 if (param->private_data && param->private_data_len)
3661 IBA_SET_MEM(CM_SIDR_REP_PRIVATE_DATA, sidr_rep_msg,
3662 param->private_data, param->private_data_len);
3663 }
3664
cm_send_sidr_rep_locked(struct cm_id_private * cm_id_priv,struct ib_cm_sidr_rep_param * param)3665 static int cm_send_sidr_rep_locked(struct cm_id_private *cm_id_priv,
3666 struct ib_cm_sidr_rep_param *param)
3667 {
3668 struct ib_mad_send_buf *msg;
3669 unsigned long flags;
3670 int ret;
3671
3672 lockdep_assert_held(&cm_id_priv->lock);
3673
3674 if ((param->info && param->info_length > IB_CM_SIDR_REP_INFO_LENGTH) ||
3675 (param->private_data &&
3676 param->private_data_len > IB_CM_SIDR_REP_PRIVATE_DATA_SIZE))
3677 return -EINVAL;
3678
3679 if (cm_id_priv->id.state != IB_CM_SIDR_REQ_RCVD)
3680 return -EINVAL;
3681
3682 msg = cm_alloc_msg(cm_id_priv);
3683 if (IS_ERR(msg))
3684 return PTR_ERR(msg);
3685
3686 cm_format_sidr_rep((struct cm_sidr_rep_msg *) msg->mad, cm_id_priv,
3687 param);
3688 trace_icm_send_sidr_rep(&cm_id_priv->id);
3689 ret = ib_post_send_mad(msg, NULL);
3690 if (ret) {
3691 cm_free_msg(msg);
3692 return ret;
3693 }
3694 cm_id_priv->id.state = IB_CM_IDLE;
3695 spin_lock_irqsave(&cm.lock, flags);
3696 if (!RB_EMPTY_NODE(&cm_id_priv->sidr_id_node)) {
3697 rb_erase(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
3698 RB_CLEAR_NODE(&cm_id_priv->sidr_id_node);
3699 }
3700 spin_unlock_irqrestore(&cm.lock, flags);
3701 return 0;
3702 }
3703
ib_send_cm_sidr_rep(struct ib_cm_id * cm_id,struct ib_cm_sidr_rep_param * param)3704 int ib_send_cm_sidr_rep(struct ib_cm_id *cm_id,
3705 struct ib_cm_sidr_rep_param *param)
3706 {
3707 struct cm_id_private *cm_id_priv =
3708 container_of(cm_id, struct cm_id_private, id);
3709 unsigned long flags;
3710 int ret;
3711
3712 spin_lock_irqsave(&cm_id_priv->lock, flags);
3713 ret = cm_send_sidr_rep_locked(cm_id_priv, param);
3714 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3715 return ret;
3716 }
3717 EXPORT_SYMBOL(ib_send_cm_sidr_rep);
3718
cm_format_sidr_rep_event(struct cm_work * work,const struct cm_id_private * cm_id_priv)3719 static void cm_format_sidr_rep_event(struct cm_work *work,
3720 const struct cm_id_private *cm_id_priv)
3721 {
3722 struct cm_sidr_rep_msg *sidr_rep_msg;
3723 struct ib_cm_sidr_rep_event_param *param;
3724
3725 sidr_rep_msg = (struct cm_sidr_rep_msg *)
3726 work->mad_recv_wc->recv_buf.mad;
3727 param = &work->cm_event.param.sidr_rep_rcvd;
3728 param->status = IBA_GET(CM_SIDR_REP_STATUS, sidr_rep_msg);
3729 param->qkey = IBA_GET(CM_SIDR_REP_Q_KEY, sidr_rep_msg);
3730 param->qpn = IBA_GET(CM_SIDR_REP_QPN, sidr_rep_msg);
3731 param->info = IBA_GET_MEM_PTR(CM_SIDR_REP_ADDITIONAL_INFORMATION,
3732 sidr_rep_msg);
3733 param->info_len = IBA_GET(CM_SIDR_REP_ADDITIONAL_INFORMATION_LENGTH,
3734 sidr_rep_msg);
3735 param->sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr;
3736 work->cm_event.private_data =
3737 IBA_GET_MEM_PTR(CM_SIDR_REP_PRIVATE_DATA, sidr_rep_msg);
3738 }
3739
cm_sidr_rep_handler(struct cm_work * work)3740 static int cm_sidr_rep_handler(struct cm_work *work)
3741 {
3742 struct cm_sidr_rep_msg *sidr_rep_msg;
3743 struct cm_id_private *cm_id_priv;
3744
3745 sidr_rep_msg = (struct cm_sidr_rep_msg *)
3746 work->mad_recv_wc->recv_buf.mad;
3747 cm_id_priv = cm_acquire_id(
3748 cpu_to_be32(IBA_GET(CM_SIDR_REP_REQUESTID, sidr_rep_msg)), 0);
3749 if (!cm_id_priv)
3750 return -EINVAL; /* Unmatched reply. */
3751
3752 spin_lock_irq(&cm_id_priv->lock);
3753 if (cm_id_priv->id.state != IB_CM_SIDR_REQ_SENT) {
3754 spin_unlock_irq(&cm_id_priv->lock);
3755 goto out;
3756 }
3757 cm_id_priv->id.state = IB_CM_IDLE;
3758 ib_cancel_mad(cm_id_priv->msg);
3759 spin_unlock_irq(&cm_id_priv->lock);
3760
3761 cm_format_sidr_rep_event(work, cm_id_priv);
3762 cm_process_work(cm_id_priv, work);
3763 return 0;
3764 out:
3765 cm_deref_id(cm_id_priv);
3766 return -EINVAL;
3767 }
3768
cm_process_send_error(struct cm_id_private * cm_id_priv,struct ib_mad_send_buf * msg,enum ib_cm_state state,enum ib_wc_status wc_status)3769 static void cm_process_send_error(struct cm_id_private *cm_id_priv,
3770 struct ib_mad_send_buf *msg,
3771 enum ib_cm_state state,
3772 enum ib_wc_status wc_status)
3773 {
3774 struct ib_cm_event cm_event = {};
3775 int ret;
3776
3777 /* Discard old sends or ones without a response. */
3778 spin_lock_irq(&cm_id_priv->lock);
3779 if (msg != cm_id_priv->msg) {
3780 spin_unlock_irq(&cm_id_priv->lock);
3781 cm_free_msg(msg);
3782 return;
3783 }
3784 cm_free_priv_msg(msg);
3785
3786 if (state != cm_id_priv->id.state || wc_status == IB_WC_SUCCESS ||
3787 wc_status == IB_WC_WR_FLUSH_ERR)
3788 goto out_unlock;
3789
3790 trace_icm_mad_send_err(state, wc_status);
3791 switch (state) {
3792 case IB_CM_REQ_SENT:
3793 case IB_CM_MRA_REQ_RCVD:
3794 cm_reset_to_idle(cm_id_priv);
3795 cm_event.event = IB_CM_REQ_ERROR;
3796 break;
3797 case IB_CM_REP_SENT:
3798 case IB_CM_MRA_REP_RCVD:
3799 cm_reset_to_idle(cm_id_priv);
3800 cm_event.event = IB_CM_REP_ERROR;
3801 break;
3802 case IB_CM_DREQ_SENT:
3803 cm_enter_timewait(cm_id_priv);
3804 cm_event.event = IB_CM_DREQ_ERROR;
3805 break;
3806 case IB_CM_SIDR_REQ_SENT:
3807 cm_id_priv->id.state = IB_CM_IDLE;
3808 cm_event.event = IB_CM_SIDR_REQ_ERROR;
3809 break;
3810 default:
3811 goto out_unlock;
3812 }
3813 spin_unlock_irq(&cm_id_priv->lock);
3814 cm_event.param.send_status = wc_status;
3815
3816 /* No other events can occur on the cm_id at this point. */
3817 ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &cm_event);
3818 if (ret)
3819 ib_destroy_cm_id(&cm_id_priv->id);
3820 return;
3821 out_unlock:
3822 spin_unlock_irq(&cm_id_priv->lock);
3823 }
3824
cm_send_handler(struct ib_mad_agent * mad_agent,struct ib_mad_send_wc * mad_send_wc)3825 static void cm_send_handler(struct ib_mad_agent *mad_agent,
3826 struct ib_mad_send_wc *mad_send_wc)
3827 {
3828 struct ib_mad_send_buf *msg = mad_send_wc->send_buf;
3829 struct cm_id_private *cm_id_priv = msg->context[0];
3830 enum ib_cm_state state =
3831 (enum ib_cm_state)(unsigned long)msg->context[1];
3832 struct cm_port *port;
3833 u16 attr_index;
3834
3835 port = mad_agent->context;
3836 attr_index = be16_to_cpu(((struct ib_mad_hdr *)
3837 msg->mad)->attr_id) - CM_ATTR_ID_OFFSET;
3838
3839 /*
3840 * If the send was in response to a received message (context[0] is not
3841 * set to a cm_id), and is not a REJ, then it is a send that was
3842 * manually retried.
3843 */
3844 if (!cm_id_priv && (attr_index != CM_REJ_COUNTER))
3845 msg->retries = 1;
3846
3847 atomic_long_add(1 + msg->retries, &port->counters[CM_XMIT][attr_index]);
3848 if (msg->retries)
3849 atomic_long_add(msg->retries,
3850 &port->counters[CM_XMIT_RETRIES][attr_index]);
3851
3852 if (cm_id_priv)
3853 cm_process_send_error(cm_id_priv, msg, state,
3854 mad_send_wc->status);
3855 else
3856 cm_free_response_msg(msg);
3857 }
3858
cm_work_handler(struct work_struct * _work)3859 static void cm_work_handler(struct work_struct *_work)
3860 {
3861 struct cm_work *work = container_of(_work, struct cm_work, work.work);
3862 int ret;
3863
3864 switch (work->cm_event.event) {
3865 case IB_CM_REQ_RECEIVED:
3866 ret = cm_req_handler(work);
3867 break;
3868 case IB_CM_MRA_RECEIVED:
3869 ret = cm_mra_handler(work);
3870 break;
3871 case IB_CM_REJ_RECEIVED:
3872 ret = cm_rej_handler(work);
3873 break;
3874 case IB_CM_REP_RECEIVED:
3875 ret = cm_rep_handler(work);
3876 break;
3877 case IB_CM_RTU_RECEIVED:
3878 ret = cm_rtu_handler(work);
3879 break;
3880 case IB_CM_USER_ESTABLISHED:
3881 ret = cm_establish_handler(work);
3882 break;
3883 case IB_CM_DREQ_RECEIVED:
3884 ret = cm_dreq_handler(work);
3885 break;
3886 case IB_CM_DREP_RECEIVED:
3887 ret = cm_drep_handler(work);
3888 break;
3889 case IB_CM_SIDR_REQ_RECEIVED:
3890 ret = cm_sidr_req_handler(work);
3891 break;
3892 case IB_CM_SIDR_REP_RECEIVED:
3893 ret = cm_sidr_rep_handler(work);
3894 break;
3895 case IB_CM_LAP_RECEIVED:
3896 ret = cm_lap_handler(work);
3897 break;
3898 case IB_CM_APR_RECEIVED:
3899 ret = cm_apr_handler(work);
3900 break;
3901 case IB_CM_TIMEWAIT_EXIT:
3902 ret = cm_timewait_handler(work);
3903 break;
3904 default:
3905 trace_icm_handler_err(work->cm_event.event);
3906 ret = -EINVAL;
3907 break;
3908 }
3909 if (ret)
3910 cm_free_work(work);
3911 }
3912
cm_establish(struct ib_cm_id * cm_id)3913 static int cm_establish(struct ib_cm_id *cm_id)
3914 {
3915 struct cm_id_private *cm_id_priv;
3916 struct cm_work *work;
3917 unsigned long flags;
3918 int ret = 0;
3919 struct cm_device *cm_dev;
3920
3921 cm_dev = ib_get_client_data(cm_id->device, &cm_client);
3922 if (!cm_dev)
3923 return -ENODEV;
3924
3925 work = kmalloc(sizeof *work, GFP_ATOMIC);
3926 if (!work)
3927 return -ENOMEM;
3928
3929 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3930 spin_lock_irqsave(&cm_id_priv->lock, flags);
3931 switch (cm_id->state) {
3932 case IB_CM_REP_SENT:
3933 case IB_CM_MRA_REP_RCVD:
3934 cm_id->state = IB_CM_ESTABLISHED;
3935 break;
3936 case IB_CM_ESTABLISHED:
3937 ret = -EISCONN;
3938 break;
3939 default:
3940 trace_icm_establish_err(cm_id);
3941 ret = -EINVAL;
3942 break;
3943 }
3944 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3945
3946 if (ret) {
3947 kfree(work);
3948 goto out;
3949 }
3950
3951 /*
3952 * The CM worker thread may try to destroy the cm_id before it
3953 * can execute this work item. To prevent potential deadlock,
3954 * we need to find the cm_id once we're in the context of the
3955 * worker thread, rather than holding a reference on it.
3956 */
3957 INIT_DELAYED_WORK(&work->work, cm_work_handler);
3958 work->local_id = cm_id->local_id;
3959 work->remote_id = cm_id->remote_id;
3960 work->mad_recv_wc = NULL;
3961 work->cm_event.event = IB_CM_USER_ESTABLISHED;
3962
3963 /* Check if the device started its remove_one */
3964 spin_lock_irqsave(&cm.lock, flags);
3965 if (!cm_dev->going_down) {
3966 queue_delayed_work(cm.wq, &work->work, 0);
3967 } else {
3968 kfree(work);
3969 ret = -ENODEV;
3970 }
3971 spin_unlock_irqrestore(&cm.lock, flags);
3972
3973 out:
3974 return ret;
3975 }
3976
cm_migrate(struct ib_cm_id * cm_id)3977 static int cm_migrate(struct ib_cm_id *cm_id)
3978 {
3979 struct cm_id_private *cm_id_priv;
3980 unsigned long flags;
3981 int ret = 0;
3982
3983 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3984 spin_lock_irqsave(&cm_id_priv->lock, flags);
3985 if (cm_id->state == IB_CM_ESTABLISHED &&
3986 (cm_id->lap_state == IB_CM_LAP_UNINIT ||
3987 cm_id->lap_state == IB_CM_LAP_IDLE)) {
3988 cm_id->lap_state = IB_CM_LAP_IDLE;
3989 cm_id_priv->av = cm_id_priv->alt_av;
3990 } else
3991 ret = -EINVAL;
3992 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3993
3994 return ret;
3995 }
3996
ib_cm_notify(struct ib_cm_id * cm_id,enum ib_event_type event)3997 int ib_cm_notify(struct ib_cm_id *cm_id, enum ib_event_type event)
3998 {
3999 int ret;
4000
4001 switch (event) {
4002 case IB_EVENT_COMM_EST:
4003 ret = cm_establish(cm_id);
4004 break;
4005 case IB_EVENT_PATH_MIG:
4006 ret = cm_migrate(cm_id);
4007 break;
4008 default:
4009 ret = -EINVAL;
4010 }
4011 return ret;
4012 }
4013 EXPORT_SYMBOL(ib_cm_notify);
4014
cm_recv_handler(struct ib_mad_agent * mad_agent,struct ib_mad_send_buf * send_buf,struct ib_mad_recv_wc * mad_recv_wc)4015 static void cm_recv_handler(struct ib_mad_agent *mad_agent,
4016 struct ib_mad_send_buf *send_buf,
4017 struct ib_mad_recv_wc *mad_recv_wc)
4018 {
4019 struct cm_port *port = mad_agent->context;
4020 struct cm_work *work;
4021 enum ib_cm_event_type event;
4022 bool alt_path = false;
4023 u16 attr_id;
4024 int paths = 0;
4025 int going_down = 0;
4026
4027 switch (mad_recv_wc->recv_buf.mad->mad_hdr.attr_id) {
4028 case CM_REQ_ATTR_ID:
4029 alt_path = cm_req_has_alt_path((struct cm_req_msg *)
4030 mad_recv_wc->recv_buf.mad);
4031 paths = 1 + (alt_path != 0);
4032 event = IB_CM_REQ_RECEIVED;
4033 break;
4034 case CM_MRA_ATTR_ID:
4035 event = IB_CM_MRA_RECEIVED;
4036 break;
4037 case CM_REJ_ATTR_ID:
4038 event = IB_CM_REJ_RECEIVED;
4039 break;
4040 case CM_REP_ATTR_ID:
4041 event = IB_CM_REP_RECEIVED;
4042 break;
4043 case CM_RTU_ATTR_ID:
4044 event = IB_CM_RTU_RECEIVED;
4045 break;
4046 case CM_DREQ_ATTR_ID:
4047 event = IB_CM_DREQ_RECEIVED;
4048 break;
4049 case CM_DREP_ATTR_ID:
4050 event = IB_CM_DREP_RECEIVED;
4051 break;
4052 case CM_SIDR_REQ_ATTR_ID:
4053 event = IB_CM_SIDR_REQ_RECEIVED;
4054 break;
4055 case CM_SIDR_REP_ATTR_ID:
4056 event = IB_CM_SIDR_REP_RECEIVED;
4057 break;
4058 case CM_LAP_ATTR_ID:
4059 paths = 1;
4060 event = IB_CM_LAP_RECEIVED;
4061 break;
4062 case CM_APR_ATTR_ID:
4063 event = IB_CM_APR_RECEIVED;
4064 break;
4065 default:
4066 ib_free_recv_mad(mad_recv_wc);
4067 return;
4068 }
4069
4070 attr_id = be16_to_cpu(mad_recv_wc->recv_buf.mad->mad_hdr.attr_id);
4071 atomic_long_inc(&port->counters[CM_RECV][attr_id - CM_ATTR_ID_OFFSET]);
4072
4073 work = kmalloc(struct_size(work, path, paths), GFP_KERNEL);
4074 if (!work) {
4075 ib_free_recv_mad(mad_recv_wc);
4076 return;
4077 }
4078
4079 INIT_DELAYED_WORK(&work->work, cm_work_handler);
4080 work->cm_event.event = event;
4081 work->mad_recv_wc = mad_recv_wc;
4082 work->port = port;
4083
4084 /* Check if the device started its remove_one */
4085 spin_lock_irq(&cm.lock);
4086 if (!port->cm_dev->going_down)
4087 queue_delayed_work(cm.wq, &work->work, 0);
4088 else
4089 going_down = 1;
4090 spin_unlock_irq(&cm.lock);
4091
4092 if (going_down) {
4093 kfree(work);
4094 ib_free_recv_mad(mad_recv_wc);
4095 }
4096 }
4097
cm_init_qp_init_attr(struct cm_id_private * cm_id_priv,struct ib_qp_attr * qp_attr,int * qp_attr_mask)4098 static int cm_init_qp_init_attr(struct cm_id_private *cm_id_priv,
4099 struct ib_qp_attr *qp_attr,
4100 int *qp_attr_mask)
4101 {
4102 unsigned long flags;
4103 int ret;
4104
4105 spin_lock_irqsave(&cm_id_priv->lock, flags);
4106 switch (cm_id_priv->id.state) {
4107 case IB_CM_REQ_SENT:
4108 case IB_CM_MRA_REQ_RCVD:
4109 case IB_CM_REQ_RCVD:
4110 case IB_CM_MRA_REQ_SENT:
4111 case IB_CM_REP_RCVD:
4112 case IB_CM_MRA_REP_SENT:
4113 case IB_CM_REP_SENT:
4114 case IB_CM_MRA_REP_RCVD:
4115 case IB_CM_ESTABLISHED:
4116 *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS |
4117 IB_QP_PKEY_INDEX | IB_QP_PORT;
4118 qp_attr->qp_access_flags = IB_ACCESS_REMOTE_WRITE;
4119 if (cm_id_priv->responder_resources) {
4120 struct ib_device *ib_dev = cm_id_priv->id.device;
4121 u64 support_flush = ib_dev->attrs.device_cap_flags &
4122 (IB_DEVICE_FLUSH_GLOBAL | IB_DEVICE_FLUSH_PERSISTENT);
4123 u32 flushable = support_flush ?
4124 (IB_ACCESS_FLUSH_GLOBAL |
4125 IB_ACCESS_FLUSH_PERSISTENT) : 0;
4126
4127 qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_READ |
4128 IB_ACCESS_REMOTE_ATOMIC |
4129 flushable;
4130 }
4131 qp_attr->pkey_index = cm_id_priv->av.pkey_index;
4132 if (cm_id_priv->av.port)
4133 qp_attr->port_num = cm_id_priv->av.port->port_num;
4134 ret = 0;
4135 break;
4136 default:
4137 trace_icm_qp_init_err(&cm_id_priv->id);
4138 ret = -EINVAL;
4139 break;
4140 }
4141 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4142 return ret;
4143 }
4144
cm_init_qp_rtr_attr(struct cm_id_private * cm_id_priv,struct ib_qp_attr * qp_attr,int * qp_attr_mask)4145 static int cm_init_qp_rtr_attr(struct cm_id_private *cm_id_priv,
4146 struct ib_qp_attr *qp_attr,
4147 int *qp_attr_mask)
4148 {
4149 unsigned long flags;
4150 int ret;
4151
4152 spin_lock_irqsave(&cm_id_priv->lock, flags);
4153 switch (cm_id_priv->id.state) {
4154 case IB_CM_REQ_RCVD:
4155 case IB_CM_MRA_REQ_SENT:
4156 case IB_CM_REP_RCVD:
4157 case IB_CM_MRA_REP_SENT:
4158 case IB_CM_REP_SENT:
4159 case IB_CM_MRA_REP_RCVD:
4160 case IB_CM_ESTABLISHED:
4161 *qp_attr_mask = IB_QP_STATE | IB_QP_AV | IB_QP_PATH_MTU |
4162 IB_QP_DEST_QPN | IB_QP_RQ_PSN;
4163 qp_attr->ah_attr = cm_id_priv->av.ah_attr;
4164 if ((qp_attr->ah_attr.type == RDMA_AH_ATTR_TYPE_IB) &&
4165 cm_id_priv->av.dlid_datapath &&
4166 (cm_id_priv->av.dlid_datapath != 0xffff))
4167 qp_attr->ah_attr.ib.dlid = cm_id_priv->av.dlid_datapath;
4168 qp_attr->path_mtu = cm_id_priv->path_mtu;
4169 qp_attr->dest_qp_num = be32_to_cpu(cm_id_priv->remote_qpn);
4170 qp_attr->rq_psn = be32_to_cpu(cm_id_priv->rq_psn);
4171 if (cm_id_priv->qp_type == IB_QPT_RC ||
4172 cm_id_priv->qp_type == IB_QPT_XRC_TGT) {
4173 *qp_attr_mask |= IB_QP_MAX_DEST_RD_ATOMIC |
4174 IB_QP_MIN_RNR_TIMER;
4175 qp_attr->max_dest_rd_atomic =
4176 cm_id_priv->responder_resources;
4177 qp_attr->min_rnr_timer = 0;
4178 }
4179 if (rdma_ah_get_dlid(&cm_id_priv->alt_av.ah_attr) &&
4180 cm_id_priv->alt_av.port) {
4181 *qp_attr_mask |= IB_QP_ALT_PATH;
4182 qp_attr->alt_port_num = cm_id_priv->alt_av.port->port_num;
4183 qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index;
4184 qp_attr->alt_timeout = cm_id_priv->alt_av.timeout;
4185 qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr;
4186 }
4187 ret = 0;
4188 break;
4189 default:
4190 trace_icm_qp_rtr_err(&cm_id_priv->id);
4191 ret = -EINVAL;
4192 break;
4193 }
4194 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4195 return ret;
4196 }
4197
cm_init_qp_rts_attr(struct cm_id_private * cm_id_priv,struct ib_qp_attr * qp_attr,int * qp_attr_mask)4198 static int cm_init_qp_rts_attr(struct cm_id_private *cm_id_priv,
4199 struct ib_qp_attr *qp_attr,
4200 int *qp_attr_mask)
4201 {
4202 unsigned long flags;
4203 int ret;
4204
4205 spin_lock_irqsave(&cm_id_priv->lock, flags);
4206 switch (cm_id_priv->id.state) {
4207 /* Allow transition to RTS before sending REP */
4208 case IB_CM_REQ_RCVD:
4209 case IB_CM_MRA_REQ_SENT:
4210
4211 case IB_CM_REP_RCVD:
4212 case IB_CM_MRA_REP_SENT:
4213 case IB_CM_REP_SENT:
4214 case IB_CM_MRA_REP_RCVD:
4215 case IB_CM_ESTABLISHED:
4216 if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT) {
4217 *qp_attr_mask = IB_QP_STATE | IB_QP_SQ_PSN;
4218 qp_attr->sq_psn = be32_to_cpu(cm_id_priv->sq_psn);
4219 switch (cm_id_priv->qp_type) {
4220 case IB_QPT_RC:
4221 case IB_QPT_XRC_INI:
4222 *qp_attr_mask |= IB_QP_RETRY_CNT | IB_QP_RNR_RETRY |
4223 IB_QP_MAX_QP_RD_ATOMIC;
4224 qp_attr->retry_cnt = cm_id_priv->retry_count;
4225 qp_attr->rnr_retry = cm_id_priv->rnr_retry_count;
4226 qp_attr->max_rd_atomic = cm_id_priv->initiator_depth;
4227 fallthrough;
4228 case IB_QPT_XRC_TGT:
4229 *qp_attr_mask |= IB_QP_TIMEOUT;
4230 qp_attr->timeout = cm_id_priv->av.timeout;
4231 break;
4232 default:
4233 break;
4234 }
4235 if (rdma_ah_get_dlid(&cm_id_priv->alt_av.ah_attr)) {
4236 *qp_attr_mask |= IB_QP_PATH_MIG_STATE;
4237 qp_attr->path_mig_state = IB_MIG_REARM;
4238 }
4239 } else {
4240 *qp_attr_mask = IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE;
4241 if (cm_id_priv->alt_av.port)
4242 qp_attr->alt_port_num =
4243 cm_id_priv->alt_av.port->port_num;
4244 qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index;
4245 qp_attr->alt_timeout = cm_id_priv->alt_av.timeout;
4246 qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr;
4247 qp_attr->path_mig_state = IB_MIG_REARM;
4248 }
4249 ret = 0;
4250 break;
4251 default:
4252 trace_icm_qp_rts_err(&cm_id_priv->id);
4253 ret = -EINVAL;
4254 break;
4255 }
4256 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4257 return ret;
4258 }
4259
ib_cm_init_qp_attr(struct ib_cm_id * cm_id,struct ib_qp_attr * qp_attr,int * qp_attr_mask)4260 int ib_cm_init_qp_attr(struct ib_cm_id *cm_id,
4261 struct ib_qp_attr *qp_attr,
4262 int *qp_attr_mask)
4263 {
4264 struct cm_id_private *cm_id_priv;
4265 int ret;
4266
4267 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
4268 switch (qp_attr->qp_state) {
4269 case IB_QPS_INIT:
4270 ret = cm_init_qp_init_attr(cm_id_priv, qp_attr, qp_attr_mask);
4271 break;
4272 case IB_QPS_RTR:
4273 ret = cm_init_qp_rtr_attr(cm_id_priv, qp_attr, qp_attr_mask);
4274 break;
4275 case IB_QPS_RTS:
4276 ret = cm_init_qp_rts_attr(cm_id_priv, qp_attr, qp_attr_mask);
4277 break;
4278 default:
4279 ret = -EINVAL;
4280 break;
4281 }
4282 return ret;
4283 }
4284 EXPORT_SYMBOL(ib_cm_init_qp_attr);
4285
cm_show_counter(struct ib_device * ibdev,u32 port_num,struct ib_port_attribute * attr,char * buf)4286 static ssize_t cm_show_counter(struct ib_device *ibdev, u32 port_num,
4287 struct ib_port_attribute *attr, char *buf)
4288 {
4289 struct cm_counter_attribute *cm_attr =
4290 container_of(attr, struct cm_counter_attribute, attr);
4291 struct cm_device *cm_dev = ib_get_client_data(ibdev, &cm_client);
4292
4293 if (WARN_ON(!cm_dev))
4294 return -EINVAL;
4295
4296 return sysfs_emit(
4297 buf, "%ld\n",
4298 atomic_long_read(
4299 &cm_dev->port[port_num - 1]
4300 ->counters[cm_attr->group][cm_attr->index]));
4301 }
4302
4303 #define CM_COUNTER_ATTR(_name, _group, _index) \
4304 { \
4305 .attr = __ATTR(_name, 0444, cm_show_counter, NULL), \
4306 .group = _group, .index = _index \
4307 }
4308
4309 #define CM_COUNTER_GROUP(_group, _name) \
4310 static struct cm_counter_attribute cm_counter_attr_##_group[] = { \
4311 CM_COUNTER_ATTR(req, _group, CM_REQ_COUNTER), \
4312 CM_COUNTER_ATTR(mra, _group, CM_MRA_COUNTER), \
4313 CM_COUNTER_ATTR(rej, _group, CM_REJ_COUNTER), \
4314 CM_COUNTER_ATTR(rep, _group, CM_REP_COUNTER), \
4315 CM_COUNTER_ATTR(rtu, _group, CM_RTU_COUNTER), \
4316 CM_COUNTER_ATTR(dreq, _group, CM_DREQ_COUNTER), \
4317 CM_COUNTER_ATTR(drep, _group, CM_DREP_COUNTER), \
4318 CM_COUNTER_ATTR(sidr_req, _group, CM_SIDR_REQ_COUNTER), \
4319 CM_COUNTER_ATTR(sidr_rep, _group, CM_SIDR_REP_COUNTER), \
4320 CM_COUNTER_ATTR(lap, _group, CM_LAP_COUNTER), \
4321 CM_COUNTER_ATTR(apr, _group, CM_APR_COUNTER), \
4322 }; \
4323 static struct attribute *cm_counter_attrs_##_group[] = { \
4324 &cm_counter_attr_##_group[0].attr.attr, \
4325 &cm_counter_attr_##_group[1].attr.attr, \
4326 &cm_counter_attr_##_group[2].attr.attr, \
4327 &cm_counter_attr_##_group[3].attr.attr, \
4328 &cm_counter_attr_##_group[4].attr.attr, \
4329 &cm_counter_attr_##_group[5].attr.attr, \
4330 &cm_counter_attr_##_group[6].attr.attr, \
4331 &cm_counter_attr_##_group[7].attr.attr, \
4332 &cm_counter_attr_##_group[8].attr.attr, \
4333 &cm_counter_attr_##_group[9].attr.attr, \
4334 &cm_counter_attr_##_group[10].attr.attr, \
4335 NULL, \
4336 }; \
4337 static const struct attribute_group cm_counter_group_##_group = { \
4338 .name = _name, \
4339 .attrs = cm_counter_attrs_##_group, \
4340 };
4341
4342 CM_COUNTER_GROUP(CM_XMIT, "cm_tx_msgs")
4343 CM_COUNTER_GROUP(CM_XMIT_RETRIES, "cm_tx_retries")
4344 CM_COUNTER_GROUP(CM_RECV, "cm_rx_msgs")
4345 CM_COUNTER_GROUP(CM_RECV_DUPLICATES, "cm_rx_duplicates")
4346
4347 static const struct attribute_group *cm_counter_groups[] = {
4348 &cm_counter_group_CM_XMIT,
4349 &cm_counter_group_CM_XMIT_RETRIES,
4350 &cm_counter_group_CM_RECV,
4351 &cm_counter_group_CM_RECV_DUPLICATES,
4352 NULL,
4353 };
4354
cm_add_one(struct ib_device * ib_device)4355 static int cm_add_one(struct ib_device *ib_device)
4356 {
4357 struct cm_device *cm_dev;
4358 struct cm_port *port;
4359 struct ib_mad_reg_req reg_req = {
4360 .mgmt_class = IB_MGMT_CLASS_CM,
4361 .mgmt_class_version = IB_CM_CLASS_VERSION,
4362 };
4363 struct ib_port_modify port_modify = {
4364 .set_port_cap_mask = IB_PORT_CM_SUP
4365 };
4366 unsigned long flags;
4367 int ret;
4368 int count = 0;
4369 u32 i;
4370
4371 cm_dev = kzalloc(struct_size(cm_dev, port, ib_device->phys_port_cnt),
4372 GFP_KERNEL);
4373 if (!cm_dev)
4374 return -ENOMEM;
4375
4376 kref_init(&cm_dev->kref);
4377 spin_lock_init(&cm_dev->mad_agent_lock);
4378 cm_dev->ib_device = ib_device;
4379 cm_dev->ack_delay = ib_device->attrs.local_ca_ack_delay;
4380 cm_dev->going_down = 0;
4381
4382 ib_set_client_data(ib_device, &cm_client, cm_dev);
4383
4384 set_bit(IB_MGMT_METHOD_SEND, reg_req.method_mask);
4385 rdma_for_each_port (ib_device, i) {
4386 if (!rdma_cap_ib_cm(ib_device, i))
4387 continue;
4388
4389 port = kzalloc(sizeof *port, GFP_KERNEL);
4390 if (!port) {
4391 ret = -ENOMEM;
4392 goto error1;
4393 }
4394
4395 cm_dev->port[i-1] = port;
4396 port->cm_dev = cm_dev;
4397 port->port_num = i;
4398
4399 ret = ib_port_register_client_groups(ib_device, i,
4400 cm_counter_groups);
4401 if (ret)
4402 goto error1;
4403
4404 port->mad_agent = ib_register_mad_agent(ib_device, i,
4405 IB_QPT_GSI,
4406 ®_req,
4407 0,
4408 cm_send_handler,
4409 cm_recv_handler,
4410 port,
4411 0);
4412 if (IS_ERR(port->mad_agent)) {
4413 ret = PTR_ERR(port->mad_agent);
4414 goto error2;
4415 }
4416
4417 ret = ib_modify_port(ib_device, i, 0, &port_modify);
4418 if (ret)
4419 goto error3;
4420
4421 count++;
4422 }
4423
4424 if (!count) {
4425 ret = -EOPNOTSUPP;
4426 goto free;
4427 }
4428
4429 write_lock_irqsave(&cm.device_lock, flags);
4430 list_add_tail(&cm_dev->list, &cm.device_list);
4431 write_unlock_irqrestore(&cm.device_lock, flags);
4432 return 0;
4433
4434 error3:
4435 ib_unregister_mad_agent(port->mad_agent);
4436 error2:
4437 ib_port_unregister_client_groups(ib_device, i, cm_counter_groups);
4438 error1:
4439 port_modify.set_port_cap_mask = 0;
4440 port_modify.clr_port_cap_mask = IB_PORT_CM_SUP;
4441 while (--i) {
4442 if (!rdma_cap_ib_cm(ib_device, i))
4443 continue;
4444
4445 port = cm_dev->port[i-1];
4446 ib_modify_port(ib_device, port->port_num, 0, &port_modify);
4447 ib_unregister_mad_agent(port->mad_agent);
4448 ib_port_unregister_client_groups(ib_device, i,
4449 cm_counter_groups);
4450 }
4451 free:
4452 cm_device_put(cm_dev);
4453 return ret;
4454 }
4455
cm_remove_one(struct ib_device * ib_device,void * client_data)4456 static void cm_remove_one(struct ib_device *ib_device, void *client_data)
4457 {
4458 struct cm_device *cm_dev = client_data;
4459 struct cm_port *port;
4460 struct ib_port_modify port_modify = {
4461 .clr_port_cap_mask = IB_PORT_CM_SUP
4462 };
4463 unsigned long flags;
4464 u32 i;
4465
4466 write_lock_irqsave(&cm.device_lock, flags);
4467 list_del(&cm_dev->list);
4468 write_unlock_irqrestore(&cm.device_lock, flags);
4469
4470 spin_lock_irq(&cm.lock);
4471 cm_dev->going_down = 1;
4472 spin_unlock_irq(&cm.lock);
4473
4474 rdma_for_each_port (ib_device, i) {
4475 struct ib_mad_agent *mad_agent;
4476
4477 if (!rdma_cap_ib_cm(ib_device, i))
4478 continue;
4479
4480 port = cm_dev->port[i-1];
4481 mad_agent = port->mad_agent;
4482 ib_modify_port(ib_device, port->port_num, 0, &port_modify);
4483 /*
4484 * We flush the queue here after the going_down set, this
4485 * verify that no new works will be queued in the recv handler,
4486 * after that we can call the unregister_mad_agent
4487 */
4488 flush_workqueue(cm.wq);
4489 /*
4490 * The above ensures no call paths from the work are running,
4491 * the remaining paths all take the mad_agent_lock.
4492 */
4493 spin_lock(&cm_dev->mad_agent_lock);
4494 port->mad_agent = NULL;
4495 spin_unlock(&cm_dev->mad_agent_lock);
4496 ib_unregister_mad_agent(mad_agent);
4497 ib_port_unregister_client_groups(ib_device, i,
4498 cm_counter_groups);
4499 }
4500
4501 cm_device_put(cm_dev);
4502 }
4503
ib_cm_init(void)4504 static int __init ib_cm_init(void)
4505 {
4506 int ret;
4507
4508 INIT_LIST_HEAD(&cm.device_list);
4509 rwlock_init(&cm.device_lock);
4510 spin_lock_init(&cm.lock);
4511 cm.listen_service_table = RB_ROOT;
4512 cm.listen_service_id = be64_to_cpu(IB_CM_ASSIGN_SERVICE_ID);
4513 cm.remote_id_table = RB_ROOT;
4514 cm.remote_qp_table = RB_ROOT;
4515 cm.remote_sidr_table = RB_ROOT;
4516 xa_init_flags(&cm.local_id_table, XA_FLAGS_ALLOC);
4517 get_random_bytes(&cm.random_id_operand, sizeof cm.random_id_operand);
4518 INIT_LIST_HEAD(&cm.timewait_list);
4519
4520 cm.wq = alloc_workqueue("ib_cm", 0, 1);
4521 if (!cm.wq) {
4522 ret = -ENOMEM;
4523 goto error2;
4524 }
4525
4526 ret = ib_register_client(&cm_client);
4527 if (ret)
4528 goto error3;
4529
4530 return 0;
4531 error3:
4532 destroy_workqueue(cm.wq);
4533 error2:
4534 return ret;
4535 }
4536
ib_cm_cleanup(void)4537 static void __exit ib_cm_cleanup(void)
4538 {
4539 struct cm_timewait_info *timewait_info, *tmp;
4540
4541 spin_lock_irq(&cm.lock);
4542 list_for_each_entry(timewait_info, &cm.timewait_list, list)
4543 cancel_delayed_work(&timewait_info->work.work);
4544 spin_unlock_irq(&cm.lock);
4545
4546 ib_unregister_client(&cm_client);
4547 destroy_workqueue(cm.wq);
4548
4549 list_for_each_entry_safe(timewait_info, tmp, &cm.timewait_list, list) {
4550 list_del(&timewait_info->list);
4551 kfree(timewait_info);
4552 }
4553
4554 WARN_ON(!xa_empty(&cm.local_id_table));
4555 }
4556
4557 module_init(ib_cm_init);
4558 module_exit(ib_cm_cleanup);
4559