xref: /freebsd/contrib/ofed/opensm/libvendor/osm_vendor_mlx_ts.c (revision d6b92ffa)

/*
 * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
 * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 */

/*  AUTHOR                 Edward Bortnikov
 *
 *  DESCRIPTION
 *     The lower-level MAD transport interface implementation
 *     that allows sending a single MAD/receiving a callback
 *     when a single MAD is received.
 */

#if HAVE_CONFIG_H
#  include <config.h>
#endif				/* HAVE_CONFIG_H */

#include <sys/types.h>
#include <sys/stat.h>
#include <sys/ioctl.h>
#include <fcntl.h>
#include <errno.h>
#include <stdlib.h>
#include <string.h>

#include <vendor/osm_vendor_api.h>
#include <vendor/osm_vendor_mlx_transport.h>
#include <vendor/osm_vendor_mlx_dispatcher.h>
#include <vendor/osm_vendor_mlx_svc.h>
#include <vendor/osm_ts_useraccess.h>

typedef struct _osmv_TOPSPIN_transport_mgr_ {
	int device_fd;
	osm_ts_user_mad_filter filter;
	cl_thread_t receiver;
} osmv_TOPSPIN_transport_mgr_t;

static void
__osmv_TOPSPIN_mad_addr_to_osm_addr(IN osm_vendor_t const *p_vend,
				    IN struct ib_mad *p_mad,
				    IN uint8_t is_smi,
				    OUT osm_mad_addr_t * p_mad_addr);

static void
__osmv_TOPSPIN_osm_addr_to_mad_addr(IN const osm_mad_addr_t * p_mad_addr,
				    IN uint8_t is_smi,
				    OUT struct ib_mad *p_mad);

void __osmv_TOPSPIN_receiver_thr(void *p_ctx)
{
	int ts_ret_code;
	struct ib_mad mad;
	osm_mad_addr_t mad_addr;
	osmv_bind_obj_t *const p_bo = (osmv_bind_obj_t *) p_ctx;
	ib_api_status_t status = IB_SUCCESS;

	OSM_LOG_ENTER(p_bo->p_vendor->p_log);

	/* Make sure the p_bo object is still relevant */
	if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
		return;

	/* we set the type of cancelation for this thread */
	/* pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); */

	while (1) {
		/* Make sure the p_bo object is still relevant */
		if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
			return;

		/* we read one mad at a time and pass it to the read callback function */
		ts_ret_code =
		    read(((osmv_TOPSPIN_transport_mgr_t *) (p_bo->
							    p_transp_mgr))->
			 device_fd, &mad, sizeof(mad));
		/* Make sure the p_bo object is still relevant */
		if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
			return;

		if (ts_ret_code != sizeof(mad)) {
			osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR,
				"__osmv_TOPSPIN_receiver_thr: ERR 6803: "
				"error with read, bytes = %d, errno = %d\n",
				ts_ret_code, errno);
			break;
		} else {
			osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
				"__osmv_TOPSPIN_receiver_thr: "
				"MAD QPN:%d SLID:0x%04x class:0x%02x "
				"method:0x%02x attr:0x%04x status:0x%04x "
				"tid:0x%016" PRIx64 "\n",
				mad.dqpn,
				cl_ntoh16(mad.slid),
				mad.mgmt_class,
				mad.r_method,
				cl_ntoh16(mad.attribute_id),
				cl_ntoh16(mad.status),
				cl_ntoh64(mad.transaction_id));

			/* first arrange an address */
			__osmv_TOPSPIN_mad_addr_to_osm_addr(p_bo->p_vendor,
							    &mad,
							    (((ib_mad_t *) &
							      mad)->
							     mgmt_class ==
							     IB_MCLASS_SUBN_LID)
							    ||
							    (((ib_mad_t *) &
							      mad)->
							     mgmt_class ==
							     IB_MCLASS_SUBN_DIR),
							    &mad_addr);

			/* call the receiver callback */

			status =
			    osmv_dispatch_mad((osm_bind_handle_t) p_bo,
					      (void *)&mad, &mad_addr);

			/* Make sure the p_bo object is still relevant */
			if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
				return;

			if (IB_INTERRUPTED == status) {

				osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
					"__osmv_TOPSPIN_receiver_thr: "
					"The bind handle %p is being closed. "
					"Breaking the loop.\n", p_bo);
				break;
			}
		}
	}

	OSM_LOG_EXIT(p_bo->p_vendor->p_log);
}

/*
 * NAME
 *   osmv_transport_init
 *
 * DESCRIPTION
 *   Setup the MAD transport infrastructure (filters, callbacks etc).
 */

ib_api_status_t
osmv_transport_init(IN osm_bind_info_t * p_info,
		    IN char hca_id[VENDOR_HCA_MAXNAMES],
		    IN uint8_t hca_idx, IN osmv_bind_obj_t * p_bo)
{
	cl_status_t cl_st;
	char device_file[16];
	int device_fd;
	int ts_ioctl_ret;
	osmv_TOPSPIN_transport_mgr_t *p_mgr =
	    malloc(sizeof(osmv_TOPSPIN_transport_mgr_t));
	int qpn;

	if (!p_mgr) {
		return IB_INSUFFICIENT_MEMORY;
	}

	memset(p_mgr, 0, sizeof(osmv_TOPSPIN_transport_mgr_t));

	/* open TopSpin file device */
	/* HACK: assume last char in hostid is the HCA index */
	sprintf(device_file, "/dev/ts_ua%u", hca_idx);
	device_fd = open(device_file, O_RDWR);
	if (device_fd < 0) {
		fprintf(stderr, "Fatal: Fail to open the file:%s err:%d\n",
			device_file, errno);
		return IB_ERROR;
	}

	/*
	 * Create the MAD filter on this file handle.
	 */

	p_mgr->filter.port = p_bo->port_num;
	p_mgr->filter.direction = TS_IB_MAD_DIRECTION_IN;
	p_mgr->filter.mask =
	    TS_IB_MAD_FILTER_DIRECTION |
	    TS_IB_MAD_FILTER_PORT |
	    TS_IB_MAD_FILTER_QPN | TS_IB_MAD_FILTER_MGMT_CLASS;

	switch (p_info->mad_class) {
	case IB_MCLASS_SUBN_LID:
	case IB_MCLASS_SUBN_DIR:
		qpn = 0;
		p_mgr->filter.qpn = qpn;
		p_mgr->filter.mgmt_class = IB_MCLASS_SUBN_LID;
		ts_ioctl_ret =
		    ioctl(device_fd, TS_IB_IOCSMADFILTADD, &p_mgr->filter);
		if (ts_ioctl_ret < 0) {
			return IB_ERROR;
		}

		p_mgr->filter.mgmt_class = IB_MCLASS_SUBN_DIR;
		ts_ioctl_ret =
		    ioctl(device_fd, TS_IB_IOCSMADFILTADD, &p_mgr->filter);
		if (ts_ioctl_ret < 0) {
			return IB_ERROR;
		}

		break;

	case IB_MCLASS_SUBN_ADM:
	default:
		qpn = 1;
		p_mgr->filter.qpn = qpn;
		p_mgr->filter.mgmt_class = p_info->mad_class;
		ts_ioctl_ret =
		    ioctl(device_fd, TS_IB_IOCSMADFILTADD, &p_mgr->filter);
		if (ts_ioctl_ret < 0) {
			return IB_ERROR;
		}
		break;
	}

	p_mgr->device_fd = device_fd;

	p_bo->p_transp_mgr = p_mgr;

	/* Initialize the magic_ptr to the pointer of the p_bo info.
	   This will be used to signal when the object is being destroyed, so no
	   real action will be done then. */
	p_bo->magic_ptr = p_bo;

	/* init receiver thread */
	cl_st =
	    cl_thread_init(&p_mgr->receiver, __osmv_TOPSPIN_receiver_thr,
			   (void *)p_bo, "osmv TOPSPIN rcv thr");

	return (ib_api_status_t) cl_st;
}

/*
 * NAME
 *   osmv_transport_send_mad
 *
 * DESCRIPTION
 *   Send a single MAD (256 byte)
 */

ib_api_status_t
osmv_transport_mad_send(IN const osm_bind_handle_t h_bind,
			IN void *p_mad, IN const osm_mad_addr_t * p_mad_addr)
{

	osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
	osm_vendor_t const *p_vend = p_bo->p_vendor;
	struct ib_mad ts_mad;
	int ret;
	ib_api_status_t status;

	const ib_mad_t *p_mad_hdr = p_mad;

	OSM_LOG_ENTER(p_vend->p_log);

	memset(&ts_mad, 0, sizeof(ts_mad));

	/* Make sure the p_bo object is still relevant */
	if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
		return IB_INVALID_CALLBACK;

	/*
	 * Copy the MAD over to the sent mad
	 */
	memcpy(&ts_mad, p_mad_hdr, MAD_BLOCK_SIZE);

	/*
	 * For all sends other than directed route SM MADs,
	 * acquire an address vector for the destination.
	 */
	if (p_mad_hdr->mgmt_class != IB_MCLASS_SUBN_DIR) {

		__osmv_TOPSPIN_osm_addr_to_mad_addr(p_mad_addr,
						    p_mad_hdr->mgmt_class ==
						    IB_MCLASS_SUBN_LID,
						    &ts_mad);
	} else {
		/* is a directed route - we need to construct a permissive address */
		/* we do not need port number since it is part of the mad_hndl */
		ts_mad.dlid = IB_LID_PERMISSIVE;
		ts_mad.slid = IB_LID_PERMISSIVE;
		ts_mad.sqpn = 0;
		ts_mad.dqpn = 0;
	}

	ts_mad.port = p_bo->port_num;

	osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
		"osmv_transport_mad_send: "
		"Sending QPN:%d DLID:0x%04x class:0x%02x "
		"method:0x%02x attr:0x%04x status:0x%04x "
		"tid:0x%016" PRIx64 "\n",
		ts_mad.dqpn,
		cl_ntoh16(ts_mad.dlid),
		ts_mad.mgmt_class,
		ts_mad.r_method,
		cl_ntoh16(ts_mad.attribute_id),
		cl_ntoh16(ts_mad.status), cl_ntoh64(ts_mad.transaction_id)
	    );

	/* send it */
	ret =
	    write(((osmv_TOPSPIN_transport_mgr_t *) (p_bo->p_transp_mgr))->
		  device_fd, &ts_mad, sizeof(ts_mad));

	if (ret != sizeof(ts_mad)) {
		osm_log(p_vend->p_log, OSM_LOG_ERROR,
			"osmv_transport_mad_send: ERR 6804: "
			"Error sending mad (%d).\n", ret);
		status = IB_ERROR;
		goto Exit;
	}

	status = IB_SUCCESS;

Exit:
	OSM_LOG_EXIT(p_vend->p_log);
	return (status);
}

/*
   register a new mad type to the opened device file
   and send a mad through - the main idea is to make
   the filter catch it such that the read unblocks
*/
void __osm_transport_gen_dummy_mad(osmv_bind_obj_t * p_bo)
{
	struct ib_mad ts_mad;
	osmv_TOPSPIN_transport_mgr_t *p_mgr =
	    (osmv_TOPSPIN_transport_mgr_t *) (p_bo->p_transp_mgr);
	struct ib_get_port_info_ioctl port_data;
	int ts_ioctl_ret;

	/* prepare the mad fields following the stored filter on the bind */
	memset(&ts_mad, 0, sizeof(ts_mad));
	ts_mad.format_version = 1;
	ts_mad.mgmt_class = p_mgr->filter.mgmt_class;
	ts_mad.attribute_id = 0x2;
	ts_mad.class_version = 1;
	ts_mad.r_method = cl_ntoh16(0x2);
	ts_mad.port = p_bo->port_num;
	ts_mad.sqpn = p_mgr->filter.qpn;
	ts_mad.dqpn = p_mgr->filter.qpn;
	ts_mad.slid = 0xffff;
	/* we must send to our local lid ... */
	port_data.port = p_bo->port_num;
	ts_ioctl_ret = ioctl(p_mgr->device_fd, TS_IB_IOCGPORTINFO, &port_data);
	ts_mad.dlid = port_data.port_info.lid;
	ts_mad.transaction_id = 0x9999;
	write(p_mgr->device_fd, &ts_mad, sizeof(ts_mad));
}

void osmv_transport_done(IN const osm_bind_handle_t h_bind)
{
	osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
	osmv_TOPSPIN_transport_mgr_t *p_tpot_mgr =
	    (osmv_TOPSPIN_transport_mgr_t *) (p_bo->p_transp_mgr);

	CL_ASSERT(p_bo);

	/* First of all - zero out the magic_ptr, so if a callback is called -
	   it'll know that we are currently closing down, and will not handle the
	   mad. */
	p_bo->magic_ptr = 0;
	/* usleep(3000000); */

	/* seems the only way to abort a blocking read is to make it read something */
	__osm_transport_gen_dummy_mad(p_bo);
	cl_thread_destroy(&(p_tpot_mgr->receiver));
	free(p_tpot_mgr);
}

static void
__osmv_TOPSPIN_osm_addr_to_mad_addr(IN const osm_mad_addr_t * p_mad_addr,
				    IN uint8_t is_smi, OUT struct ib_mad *p_mad)
{

	/* For global destination or Multicast address: */
	p_mad->dlid = cl_ntoh16(p_mad_addr->dest_lid);
	p_mad->sl = p_mad_addr->addr_type.gsi.service_level;
	if (is_smi) {
		p_mad->sqpn = 0;
		p_mad->dqpn = 0;
	} else {
		p_mad->sqpn = 1;
		p_mad->dqpn = cl_ntoh32(p_mad_addr->addr_type.gsi.remote_qp);
	}
	/*
	   HACK we limit to the first PKey Index assuming it will
	   always be the default PKey
	 */
	p_mad->pkey_index = 0;
}

static void
__osmv_TOPSPIN_mad_addr_to_osm_addr(IN osm_vendor_t const *p_vend,
				    IN struct ib_mad *p_mad,
				    IN uint8_t is_smi,
				    OUT osm_mad_addr_t * p_mad_addr)
{
	p_mad_addr->dest_lid = cl_hton16(p_mad->slid);
	p_mad_addr->static_rate = 0;
	p_mad_addr->path_bits = 0;
	if (is_smi) {
		/* SMI */
		p_mad_addr->addr_type.smi.source_lid = cl_hton16(p_mad->slid);
		p_mad_addr->addr_type.smi.port_num = p_mad->port;
	} else {
		/* GSI */
		p_mad_addr->addr_type.gsi.remote_qp = cl_ntoh32(p_mad->sqpn);
		p_mad_addr->addr_type.gsi.remote_qkey = IB_QP1_WELL_KNOWN_Q_KEY;
		/*  There is a TAVOR limitation that only one P_KEY is supported per */
		/*  QP - so QP1 must use IB_DEFAULT_PKEY */
		p_mad_addr->addr_type.gsi.pkey_ix = p_mad->pkey_index;
		p_mad_addr->addr_type.gsi.service_level = p_mad->sl;

		p_mad_addr->addr_type.gsi.global_route = FALSE;
		/* copy the GRH data if relevant - TopSpin imp doesnt relate to GRH!!! */
		/*
		   if (p_mad_addr->addr_type.gsi.global_route)
		   {
		   p_mad_addr->addr_type.gsi.grh_info.ver_class_flow =
		   ib_grh_set_ver_class_flow(p_rcv_desc->grh.IP_version,
		   p_rcv_desc->grh.traffic_class,
		   p_rcv_desc->grh.flow_label);
		   p_mad_addr->addr_type.gsi.grh_info.hop_limit =  p_rcv_desc->grh.hop_limit;
		   memcpy(&p_mad_addr->addr_type.gsi.grh_info.src_gid.raw,
		   &p_rcv_desc->grh.sgid, sizeof(ib_net64_t));
		   memcpy(&p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw,
		   p_rcv_desc->grh.dgid,  sizeof(ib_net64_t));
		   }
		 */
	}
}

/*
 *  NAME            osm_vendor_set_sm
 *
 *  DESCRIPTION     Modifies the port info for the bound port to set the "IS_SM" bit
 *                  according to the value given (TRUE or FALSE).
 */
#if (defined(OSM_VENDOR_INTF_TS_NO_VAPI) || defined(OSM_VENDOR_INTF_TS))

void osm_vendor_set_sm(IN osm_bind_handle_t h_bind, IN boolean_t is_sm_val)
{
	osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
	osm_vendor_t const *p_vend = p_bo->p_vendor;
	int ts_ioctl_ret;
	int device_fd =
	    ((osmv_TOPSPIN_transport_mgr_t *) (p_bo->p_transp_mgr))->device_fd;
	struct ib_set_port_info_ioctl set_port_data;

	OSM_LOG_ENTER(p_vend->p_log);

	memset(&set_port_data, 0, sizeof(set_port_data));

	set_port_data.port = p_bo->port_num;
	set_port_data.port_info.valid_fields = IB_PORT_IS_SM;
	set_port_data.port_info.is_sm = is_sm_val;
	ts_ioctl_ret = ioctl(device_fd, TS_IB_IOCSPORTINFO, &set_port_data);
	if (ts_ioctl_ret < 0) {
		osm_log(p_vend->p_log, OSM_LOG_ERROR,
			"osm_vendor_set_sm: ERR 6805: "
			"Unable set 'IS_SM' bit to:%u in port attributes (%d).\n",
			is_sm_val, ts_ioctl_ret);
	}

	OSM_LOG_EXIT(p_vend->p_log);
}

#endif