1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 /* 26 * Copyright (c) 2009 Oracle. All rights reserved. 27 * 28 * This software is available to you under a choice of one of two 29 * licenses. You may choose to be licensed under the terms of the GNU 30 * General Public License (GPL) Version 2, available from the file 31 * COPYING in the main directory of this source tree, or the 32 * OpenIB.org BSD license below: 33 * 34 * Redistribution and use in source and binary forms, with or 35 * without modification, are permitted provided that the following 36 * conditions are met: 37 * 38 * - Redistributions of source code must retain the above 39 * copyright notice, this list of conditions and the following 40 * disclaimer. 41 * 42 * - Redistributions in binary form must reproduce the above 43 * copyright notice, this list of conditions and the following 44 * disclaimer in the documentation and/or other materials 45 * provided with the distribution. 46 * 47 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 48 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 49 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 50 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 51 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 52 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 53 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 54 * SOFTWARE. 55 * 56 */ 57 #include <sys/ib/clients/of/rdma/ib_verbs.h> 58 #include <sys/ib/clients/of/rdma/ib_addr.h> 59 #include <sys/ib/clients/of/rdma/rdma_cm.h> 60 61 #include <sys/ib/clients/rdsv3/ib.h> 62 #include <sys/ib/clients/rdsv3/rdma_transport.h> 63 #include <sys/ib/clients/rdsv3/rdsv3_debug.h> 64 65 kmutex_t rdsv3_rdma_listen_id_lock; 66 struct rdma_cm_id *rdsv3_rdma_listen_id = NULL; 67 68 int 69 rdsv3_rdma_cm_event_handler(struct rdma_cm_id *cm_id, 70 struct rdma_cm_event *event) 71 { 72 /* this can be null in the listening path */ 73 struct rdsv3_connection *conn = cm_id->context; 74 struct rdsv3_transport *trans; 75 int ret = 0; 76 77 RDSV3_DPRINTF2("rdsv3_rdma_cm_event_handler", 78 "conn %p id %p handling event %u", conn, cm_id, event->event); 79 80 trans = &rdsv3_ib_transport; 81 82 /* 83 * Prevent shutdown from tearing down the connection 84 * while we're executing. 85 */ 86 if (conn) { 87 mutex_enter(&conn->c_cm_lock); 88 89 /* 90 * If the connection is being shut down, bail out 91 * right away. We return 0 so cm_id doesn't get 92 * destroyed prematurely 93 */ 94 if (rdsv3_conn_state(conn) == RDSV3_CONN_DISCONNECTING) { 95 /* 96 * Reject incoming connections while we're tearing 97 * down an existing one. 98 */ 99 if (event->event == RDMA_CM_EVENT_CONNECT_REQUEST) 100 ret = 1; 101 RDSV3_DPRINTF2("rdsv3_rdma_cm_event_handler", 102 "conn %p id %p incoming event %u when " 103 "disconnecting", conn, cm_id, event->event); 104 goto out; 105 } 106 } 107 108 switch (event->event) { 109 case RDMA_CM_EVENT_CONNECT_REQUEST: 110 ret = trans->cm_handle_connect(cm_id, event); 111 break; 112 113 case RDMA_CM_EVENT_ADDR_RESOLVED: 114 /* XXX do we need to clean up if this fails? */ 115 ret = rdma_resolve_route(cm_id, 116 RDSV3_RDMA_RESOLVE_TIMEOUT_MS); 117 break; 118 119 case RDMA_CM_EVENT_ROUTE_RESOLVED: 120 /* XXX worry about racing with listen acceptance */ 121 ret = trans->cm_initiate_connect(cm_id); 122 break; 123 124 case RDMA_CM_EVENT_ESTABLISHED: 125 trans->cm_connect_complete(conn, event); 126 break; 127 128 case RDMA_CM_EVENT_ADDR_ERROR: 129 case RDMA_CM_EVENT_ROUTE_ERROR: 130 case RDMA_CM_EVENT_CONNECT_ERROR: 131 case RDMA_CM_EVENT_UNREACHABLE: 132 case RDMA_CM_EVENT_REJECTED: 133 case RDMA_CM_EVENT_DEVICE_REMOVAL: 134 case RDMA_CM_EVENT_ADDR_CHANGE: 135 if (conn) 136 rdsv3_conn_drop(conn); 137 break; 138 139 case RDMA_CM_EVENT_DISCONNECTED: 140 RDSV3_DPRINTF2("rdsv3_rdma_cm_event_handler", 141 "RDS/RDMA: DISCONNECT event - dropping connection " 142 "cm_id: %p", cm_id); 143 if (conn) { 144 RDSV3_DPRINTF2("rdsv3_rdma_cm_event_handler", 145 "RDS/RDMA: DISCONNECT event - dropping connection " 146 "%u.%u.%u.%u ->%u.%u.%u.%u", NIPQUAD(conn->c_laddr), 147 NIPQUAD(conn->c_faddr)); 148 rdsv3_conn_drop(conn); 149 } 150 break; 151 152 default: 153 /* things like device disconnect? */ 154 RDSV3_DPRINTF2("rdsv3_rdma_cm_event_handler", 155 "unknown event %u\n", event->event); 156 RDSV3_PANIC(); 157 break; 158 } 159 160 out: 161 if (conn) { 162 #ifndef __lock_lint 163 // struct rds_iw_connection *ic = conn->c_transport_data; 164 165 /* If we return non-zero, we must to hang on to the cm_id */ 166 // BUG_ON(ic->i_cm_id == cm_id && ret); 167 #endif 168 169 mutex_exit(&conn->c_cm_lock); 170 } 171 172 RDSV3_DPRINTF2("rdsv3_rdma_cm_event_handler", 173 "id %p event %u handling ret %d", cm_id, event->event, ret); 174 175 return (ret); 176 } 177 178 static int 179 rdsv3_rdma_listen_init(void) 180 { 181 struct sockaddr_in sin; 182 struct rdma_cm_id *cm_id; 183 int ret; 184 185 RDSV3_DPRINTF2("rdsv3_rdma_listen_init", "Enter"); 186 187 cm_id = rdma_create_id(rdsv3_rdma_cm_event_handler, NULL, RDMA_PS_TCP); 188 if (IS_ERR(cm_id)) { 189 ret = PTR_ERR(cm_id); 190 RDSV3_DPRINTF2("rdsv3_rdma_listen_init", 191 "RDS/RDMA: failed to setup listener, " 192 "rdma_create_id() returned %d", ret); 193 goto out; 194 } 195 196 sin.sin_family = PF_INET; 197 sin.sin_addr.s_addr = (uint32_t)htonl(INADDR_ANY); 198 sin.sin_port = (uint16_t)htons(RDSV3_PORT); 199 200 /* 201 * XXX I bet this binds the cm_id to a device. If we want to support 202 * fail-over we'll have to take this into consideration. 203 */ 204 ret = rdma_bind_addr(cm_id, (struct sockaddr *)&sin); 205 if (ret) { 206 RDSV3_DPRINTF2("rdsv3_rdma_listen_init", 207 "RDS/RDMA: failed to setup listener, " 208 "rdma_bind_addr() returned %d", ret); 209 goto out; 210 } 211 212 ret = rdma_listen(cm_id, 128); 213 if (ret) { 214 RDSV3_DPRINTF2("rdsv3_rdma_listen_init", 215 "RDS/RDMA: failed to setup listener, " 216 "rdma_listen() returned %d", ret); 217 goto out; 218 } 219 220 RDSV3_DPRINTF5("rdsv3_rdma_listen_init", 221 "cm %p listening on port %u", cm_id, RDSV3_PORT); 222 223 rdsv3_rdma_listen_id = cm_id; 224 cm_id = NULL; 225 226 RDSV3_DPRINTF2("rdsv3_rdma_listen_init", 227 "Return: rdsv3_rdma_listen_id: %p", rdsv3_rdma_listen_id); 228 out: 229 if (cm_id) 230 rdma_destroy_id(cm_id); 231 return (ret); 232 } 233 234 static void rdsv3_rdma_listen_stop(void) 235 { 236 RDSV3_DPRINTF2("rdsv3_rdma_listen_stop", "cm %p", rdsv3_rdma_listen_id); 237 rdma_destroy_id(rdsv3_rdma_listen_id); 238 239 RDSV3_DPRINTF2("rdsv3_rdma_listen_stop", "Return"); 240 } 241 242 /* 243 * This function can be called via two routes. 244 * 1. During attach on a worker thread. 245 * 2. From rdsv3_create() for 1st socket. 246 */ 247 void 248 rdsv3_rdma_init() 249 { 250 int ret; 251 252 RDSV3_DPRINTF2("rdsv3_rdma_init", "Enter"); 253 254 mutex_enter(&rdsv3_rdma_listen_id_lock); 255 if (rdsv3_rdma_listen_id != NULL) { 256 RDSV3_DPRINTF2("rdsv3_rdma_init", 257 "rdsv3_rdma_listen_id is already initialized: %p", 258 rdsv3_rdma_listen_id); 259 mutex_exit(&rdsv3_rdma_listen_id_lock); 260 return; 261 } 262 263 ret = rdsv3_rdma_listen_init(); 264 if (ret) { 265 mutex_exit(&rdsv3_rdma_listen_id_lock); 266 return; 267 } 268 269 ret = rdsv3_ib_init(); 270 if (ret) { 271 rdsv3_rdma_listen_stop(); 272 } 273 mutex_exit(&rdsv3_rdma_listen_id_lock); 274 275 RDSV3_DPRINTF2("rdsv3_rdma_init", "Return"); 276 } 277 278 /*ARGSUSED*/ 279 void 280 rdsv3_rdma_exit(void *arg) 281 { 282 RDSV3_DPRINTF2("rdsv3_rdma_exit", "Enter"); 283 284 /* stop listening first to ensure no new connections are attempted */ 285 if (rdsv3_rdma_listen_id) { 286 rdsv3_rdma_listen_stop(); 287 rdsv3_ib_exit(); 288 rdsv3_rdma_listen_id = NULL; 289 } 290 291 RDSV3_DPRINTF2("rdsv3_rdma_exit", "Return"); 292 } 293