1 /* SHARED.H (c) Copyright Greg Smith, 2002-2009 */ 2 /* Shared Device Server header file */ 3 4 /*------------------------------------------------------------------- 5 * Shared device support (c)Copyright Greg Smith, 2002-2009 6 * 7 * Shared device support allows multiple Hercules instances to share 8 * devices. The device will be `local' to one instance and `remote' 9 * to all other instances. The local instance is the * `server' for 10 * that device and the remote instance is the `client'. You do not 11 * have to IPL an operating system on the device server. Any number 12 * of Hercules instances can act as a server in a Hercplex ;-) 13 * 14 * To use a device on a remote system, instead of specifying a file 15 * name on the device config statement, you specify 16 * 17 * ip_address_or_name:port:devnum 18 * 19 * For example: 20 * 21 * 0100 3350 localhost:3990:0100 22 * 23 * which says there is a device server on the local host listening 24 * on port 3990 and we want to use its 0100 device as 0100. The 25 * default port is 3990 and the default remote device number is the 26 * local device number. So we could say 27 * 28 * 0100 3350 localhost 29 * 30 * instead, providing we don't actually have a file `localhost'. 31 * Interestingly, the instance on the local host listening on 3990 32 * could have a statement 33 * 34 * 0100 3350 192.168.200.1::0200 35 * 36 * which means that instance in turn will use device 0200 on the 37 * server at 192.168.200.1 listening on port 3990. The original 38 * instance will have to `hop' thru the second instance to get 39 * to the real device. 40 * 41 * Device sharing can be `split' between multiple instances. 42 * For example, suppose instance A has 43 * 44 * SHRDPORT 3990 45 * 0100 3350 localhost:3991 46 * 0101 3350 mvscat 47 * 48 * and instance B has 49 * 50 * SHRDPORT 3991 51 * 0100 3350 mvsres 52 * 0101 3350 localhost 53 * 54 * Then each instance acts as both a client and as a server. 55 * 56 * When `SHRDPORT' is specified, thread `shared_server' is started 57 * at the end of Hercules initialization. In the example above, 58 * neither Hercules instance can initialize their devices until the 59 * server is started on each system. In this case, the device trying 60 * to access a server gets the `connecting' bit set on in the DEVBLK 61 * and the device still needs to initialize. After the shared server 62 * is started, a thread is attached for each device that is connecting 63 * to complete the connection (which is the device init handler). 64 * 65 * TECHNICAL BS: 66 * 67 * There are (at least) two approaches to sharing devices. One is to 68 * execute the channel program on the server system. The server will 69 * need to request from the client system information such as the ccw 70 * and the data to be written, and will need to send to the client 71 * data that has been read and status information. The second is to 72 * execute the channel program on the client system. Here the client 73 * system makes requests to the server system to read and write data. 74 * 75 * The second approach is currently implemented. The first approach 76 * arguably emulates `more correctly'. However, an advantage of the 77 * implemented approach is that it is easier because only the 78 * client sends requests and only the server sends responses. 79 * 80 * Both client and server have a DEVBLK structure for the device. 81 * Absurdly, perhaps, in originally designing an implementation for 82 * shared devices it was not clear what type of process should be the 83 * server. It was a quantum leap forward to realize that it could 84 * just be another hercules instance. 85 * 86 * PROTOCOL: 87 * (If this section is as boring for you to read as it was for me 88 * to write then please skip to the next section ;-) 89 * 90 * The client sends an 8 byte request header and maybe some data: 91 * 92 * +-----+-----+-----+-----+-----+-----+-----+-----+ 93 * | cmd |flag | devnum | id | length | 94 * +-----+-----+-----+-----+-----+-----+-----+-----+ 95 * 96 * <-------- length ---------> 97 * +----- . . . . . -----+ 98 * | data | 99 * +----- . . . . . -----+ 100 * 101 * `cmd' identifies the client request. The requests are: 102 * 103 * 0xe0 CONNECT Connect to the server. This requires 104 * the server to allocate resources to 105 * support the connection. Typically issued 106 * during device initialization or after being 107 * disconnected after a network error or timeout. 108 * 0xe1 DISCONNECT Disconnect from the server. The server 109 * can now release the allocated resources 110 * for the connection. Typically issued during 111 * device close or detach. 112 * 0xe2 START Start a channel program on the device. 113 * If the device is busy or reserved by 114 * another system then wait until the device 115 * is available unless the NOWAIT flag bit 116 * is set, then return a BUSY code. Once 117 * START succeeds then the device is unavailable 118 * until the END request. 119 * 0xe3 END Channel program has ended. Any waiters 120 * for the device can now retry. 121 * 0xe4 RESUME Similar to START except a suspended 122 * channel program has resumed. 123 * 0xe5 SUSPEND Similar to END except a channel program 124 * has suspended itself. If the channel 125 * program is not resumed then the END 126 * request is *not* issued. 127 * 0xe6 RESERVE Makes the device unavailable to any other 128 * system until a RELEASE request is issued. 129 * *Must* be issued within the scope of START/END. 130 * 0xe7 RELEASE Makes the device available to other systems 131 * after the next END request. 132 * *Must* be issued within the scope of 133 * START/END. 134 * 0xe8 READ Read from a device. A 4-byte `record' 135 * identifier is specified in the request 136 * data to identify what data to read in the 137 * device context. 138 * *Must* be issued within the scope of START/END. 139 * 0xe9 WRITE Write to a device. A 2-byte `offset' and 140 * a 4-byte `record' is specified in the request 141 * data, followed by the data to be written. 142 * `record' identifies what data is to be written 143 * in the device context and `offset' and `length' 144 * identify what to update in `record'. 145 * *Must* be issued within the scope of START/END. 146 * 0xea SENSE Retrieves the sense information after an i/o 147 * error has occurred on the server side. This 148 * is typically issued within the scope of the 149 * channel program having the error. Client side 150 * sense or concurrent sense will then pick up the 151 * sense data relevant to the i/o error. 152 * *Must* be issued within the scope of START/END. 153 * 0xeb QUERY Obtain device information, typically during 154 * device initialization. 155 * 0xec COMPRESS Negotiate compression parameters. Notifies the 156 * server what compression algorithms are supported 157 * by the client and whether or not data sent back 158 * and forth from the client or server should be 159 * compressed or not. Typically issued after CONNECT. 160 * *NOTE* This action should actually be SETOPT or 161 * some such; it was just easier to code a COMPRESS 162 * specific SETOPT (less code). 163 * 164 * `flag' qualifies the client request and varies by the request. 165 * 166 * 0x80 NOWAIT For START, if the device is unavailable then 167 * return BUSY instead of waiting for the device. 168 * 0x40 QUERY Identifies the QUERY request: 169 * 0x41 DEVCHAR Device characteristics data 170 * 0x42 DEVID Device identifier data 171 * 0x43 DEVUSED Hi used track/block (for dasdcopy) 172 * 0x48 CKDCYLS Number cylinders for CKD device 173 * 0x4c FBAORIGIN Origin block for FBA 174 * 0x4d FBANUMBLK Number of FBA blocks 175 * 0x4e FBABLKSIZ Size of an FBA block 176 * 0x3x COMP For WRITE, data is compressed at offset `x': 177 * 0x2x BZIP2 using bzip2 178 * 0x1x LIBZ using zlib 179 * 0xxy For COMPRESS, identifies the compression 180 * algorithms supported by the client (0x2y for bzip2, 181 * 0x1y for zlib, 0x3y for both) and the zlib compression 182 * parameter `y' for sending otherwise uncompressed data 183 * back and forth. If `y' is zero (default) then no 184 * uncompressed data is compressed between client & server. 185 * 186 * `devnum' identifies the device by number on the server instance. 187 * The device number may be different than the 188 * device number on the client instance. 189 * `id' identifies the client to the server. Each client has a unique 190 * positive (non-zero) identifier. For the initial 191 * CONNECT request `id' is zero. After a successful 192 * CONNECT, the server returns in the response header 193 * the identifier to be used for all other requests 194 * (including subsequent CONNECT requests). This is 195 * saved in dev->rmtid. 196 * `length' specifies the length of the data following the request header. 197 * Currently length is non-zero for READ/WRITE requests. 198 * 199 * The server sends an 8 byte response header and maybe some data: 200 * 201 * +-----+-----+-----+-----+-----+-----+-----+-----+ 202 * |code |stat | devnum | id | length | 203 * +-----+-----+-----+-----+-----+-----+-----+-----+ 204 * 205 * <-------- length ---------> 206 * +----- . . . . . -----+ 207 * | data | 208 * +----- . . . . . -----+ 209 * 210 * `code' indicates the response to the request. OK (0x00) indicates 211 * success however other codes also indicate success 212 * but qualified in some manner: 213 * 0x80 ERROR An error occurred. The server provides an error 214 * message in the data section. 215 * 0x40 IOERR An i/o error occurred during a READ/WRITE 216 * request. The status byte has the `unitstat' 217 * data. This should signal the client to issue the 218 * SENSE request to obtain the current sense data. 219 * 0x20 BUSY Device was not available for a START request and 220 * the NOWAIT flag bit was turned on. 221 * 0x10 COMP Data returned is compressed. The status byte 222 * indicates how the data is compressed (zlib or 223 * bzip2) and at what offset the compressed data 224 * starts (0 .. 15). This bit is only turned on 225 * when both the `code' and `status' bytes would 226 * otherwise be zero. 227 * 0x08 PURGE START request was issued by the client. A list 228 * of `records' to be purged from local cache is 229 * returned. These are `records' that have been 230 * updated since the last START/END request from 231 * the client by other systems. Each record identifier 232 * is a 4-byte field in the data segment. The number 233 * of records then is `length'/4. If the number of 234 * records exceeds a threshold (16) then `length' 235 * will be zero indicating that the client should 236 * purge all locally cached records for the device. 237 * 238 * `stat' contains status information as a result of the request. 239 * For READ/WRITE requests this contains the `unitstat' 240 * information if an IOERR occurred. 241 * 242 * `devnum' specifies the server device number 243 * 244 * `id' specifies the system identifier for the request. 245 * 246 * `length' is the size of the data returned. 247 * 248 * 249 * CACHING 250 * 251 * Cached records (eg CKD tracks or FBA blocks) are kept independently on 252 * both the client and server sides. Whenever the client issues a START 253 * request to initiate a channel program the server will return a list 254 * of records to purge from the client's cache that have been updated by 255 * other clients since the last START request. If the list is too large 256 * the server will indicate that the client should purge all records for 257 * the device. 258 * 259 * COMPRESSION 260 * 261 * Data that would normally be transferred uncompressed between client 262 * and host can optionally be compressed by specifying the `comp=' 263 * keyword on the device configuration statement or attach command. 264 * For example 265 * 266 * 0100 3350 192.168.2.12 comp=3 267 * 268 * The value of the `comp=' keyword is the zlib compression parameter 269 * which should be a number between 1 .. 9. A value closer to 1 means 270 * less compression but less processor time to perform the compression. 271 * A value closer to 9 means the data is compressed more but more processor 272 * time is required. 273 * 274 * If the server is on `localhost' then you should not specify `comp='. 275 * Otherwise you are just stealing processor time to do compression/ 276 * uncompression from hercules. If the server is on a local network 277 * then I would recommend specifying a low value such as 1, 2 or 3. 278 * We are on a curve here, trying to trade cpu cycles for network traffic 279 * to derive an optimal throughput. 280 * 281 * If the devices on the server are compressed devices (eg CCKD or CFBA) 282 * then the `records' (eg. track images or block groups) may be transferred 283 * compressed regardless of the `comp=' setting. This depends on whether 284 * the client supports the compression type (zlib or bzip2) of the record 285 * on the server and whether the record is actually compressed in the 286 * server cache. 287 * 288 * For example: 289 * 290 * Suppose on the client that you execute one or more channel programs 291 * to read a record on a ckd track, update a record on the same track, 292 * and then read another (or the same) record on the track. 293 * 294 * For the first read the server will read the track image and 295 * pass it to the client as it was originally compressed in the file. 296 * To update a portion of the track image the server must uncompress 297 * the track image so data in it can be updated. When the client next 298 * reads from the track image, the track image is uncompressed. 299 * 300 * Specifying `comp=' means that uncompressed data sent to the client 301 * will be compressed. If the data to be sent to the client is already 302 * compressed then the data is sent as is, unless the client has indicated 303 * that it does not support that compression algorithm. 304 * 305 * 306 * TODO 307 * 308 * 1. More doc (sorry, I got winded) 309 * 2. Delays observed during short transfers (redrive select ?) 310 * 3. Better server side behaviour due to disconnect 311 * 3. etc. 312 * 313 * 314 *-------------------------------------------------------------------*/ 315 316 #ifndef _HERCULES_SHARED_H 317 #define _HERCULES_SHARED_H 1 318 319 #include "hercules.h" 320 321 322 #ifndef _SHARED_C_ 323 #ifndef _HDASD_DLL_ 324 #define SHR_DLL_IMPORT DLL_IMPORT 325 #else /* _HDASD_DLL_ */ 326 #define SHR_DLL_IMPORT extern 327 #endif /* _HDASD_DLL_ */ 328 #else 329 #define SHR_DLL_IMPORT DLL_EXPORT 330 #endif 331 332 333 #define OPTION_SHARED_DEVICES 334 #undef FBA_SHARED 335 336 /* 337 * Differing version levels are not compatible 338 * Differing release levels are compatible 339 */ 340 341 #define SHARED_VERSION 0 /* Version level (0 .. 15) */ 342 #define SHARED_RELEASE 1 /* Release level (0 .. 15) */ 343 344 #define SHARED_MAX_SYS 8 /* Max number connections */ 345 typedef char SHRD_TRACE[128]; /* Trace entry */ 346 347 #include "hercules.h" 348 349 /* Requests */ 350 #define SHRD_CONNECT 0xe0 /* Connect */ 351 #define SHRD_DISCONNECT 0xe1 /* Disconnect */ 352 #define SHRD_START 0xe2 /* Start channel program */ 353 #define SHRD_END 0xe3 /* End channel program */ 354 #define SHRD_RESUME 0xe4 /* Resume channel program */ 355 #define SHRD_SUSPEND 0xe5 /* Suspend channel program */ 356 #define SHRD_RESERVE 0xe6 /* Reserve */ 357 #define SHRD_RELEASE 0xe7 /* Release */ 358 #define SHRD_READ 0xe8 /* Read data */ 359 #define SHRD_WRITE 0xe9 /* Write data */ 360 #define SHRD_SENSE 0xea /* Sense */ 361 #define SHRD_QUERY 0xeb /* Query */ 362 #define SHRD_COMPRESS 0xec /* Compress request */ 363 364 /* Response codes */ 365 #define SHRD_OK 0x00 /* Success */ 366 #define SHRD_ERROR 0x80 /* Failure */ 367 #define SHRD_IOERR 0x40 /* I/O error */ 368 #define SHRD_BUSY 0x20 /* Resource is busy */ 369 #define SHRD_COMP 0x10 /* Data is compressed */ 370 #define SHRD_PURGE 0x08 /* Purge list provided */ 371 372 #define SHRD_ERROR_INVALID 0xf0 /* Invalid request */ 373 #define SHRD_ERROR_BADVERS 0xf1 /* Version mismatch */ 374 #define SHRD_ERROR_NOTINIT 0xf2 /* Device not initialized */ 375 #define SHRD_ERROR_NOTCONN 0xf3 /* Not connected to device */ 376 #define SHRD_ERROR_NOTAVAIL 0xf4 /* No available SHRD */ 377 #define SHRD_ERROR_NOMEM 0xf5 /* Out of memory */ 378 #define SHRD_ERROR_NOTACTIVE 0xf6 /* Not device owner */ 379 #define SHRD_ERROR_NODEVICE 0xf7 /* No such device */ 380 #define SHRD_ERROR_CONNECTED 0xf8 /* Already connected */ 381 382 /* Flags */ 383 #define SHRD_NOWAIT 0x80 /* Don't wait if busy */ 384 #define SHRD_QUERY_REQUEST 0x40 /* Query request */ 385 #define SHRD_COMP_MASK 0x30 /* Mask to detect compression*/ 386 #define SHRD_COMP_OFF 0x0f /* Offset to compressed data */ 387 #define SHRD_COMP_MAX_OFF 15 /* Max offset allowed */ 388 #define SHRD_LIBZ 0x01 /* Compressed using zlib */ 389 #define SHRD_BZIP2 0x02 /* Compressed using bzip2 */ 390 391 /* Query Types */ 392 #define SHRD_DEVCHAR 0x41 /* Device characteristics */ 393 #define SHRD_DEVID 0x42 /* Device identifier */ 394 #define SHRD_USED 0x43 /* Device usage */ 395 #define SHRD_CKDCYLS 0x48 /* CKD number cylinders */ 396 #define SHRD_FBAORIGIN 0x4c /* FBA origin */ 397 #define SHRD_FBANUMBLK 0x4d /* FBA number blocks */ 398 #define SHRD_FBABLKSIZ 0x4e /* FBA block size */ 399 400 /* Constraints */ 401 #define SHARED_DEFAULT_PORT 3990 /* Default shared port */ 402 #define SHARED_PURGE_MAX 16 /* Max size of purge list */ 403 #define SHARED_MAX_MSGLEN 255 /* Max message length */ 404 #define SHARED_TIMEOUT 120 /* Disconnect timeout (sec) */ 405 #define SHARED_FORCE_TIMEOUT 300 /* Force disconnect (sec) */ 406 #define SHARED_SELECT_WAIT 10 /* Select timeout (sec) */ 407 #define SHARED_COMPRESS_MINLEN 512 /* Min length for compression*/ 408 409 struct SHRD { 410 int id; /* Identifier */ 411 int fd; /* Socket */ 412 char *ipaddr; /* IP addr of connected peer */ 413 time_t time; /* Time last request */ 414 int release; /* Client release level */ 415 int comp; /* Compression parameter */ 416 int comps; /* Compression supported */ 417 int pending:1, /* 1=Request pending */ 418 waiting:1, /* 1=Waiting for device */ 419 havehdr:1, /* 1=Header already read */ 420 disconnect:1; /* 1=Disconnect device */ 421 DBLWRD hdr; /* Header */ 422 int purgen; /* Number purge entries */ 423 FWORD purge[SHARED_PURGE_MAX];/* Purge list */ 424 }; 425 426 typedef struct _SHRD_HDR { 427 BYTE cmd; /* 0 Command */ 428 BYTE code; /* 1 Flags and Codes */ 429 U16 devnum; /* 2 Device number */ 430 U16 id; /* 4 Identifier */ 431 U16 len; /* 6 Data length */ 432 } SHRD_HDR; 433 /* Size must be 8 bytes */ 434 #define SHRD_HDR_SIZE sizeof(DBLWRD) 435 436 #define SHRD_SET_HDR(_buf, _cmd, _code, _devnum, _len, _id) \ 437 do { \ 438 SHRD_HDR *shdr = (SHRD_HDR *)(_buf); \ 439 shdr->cmd = (_cmd); \ 440 shdr->code = (_code); \ 441 store_hw (&shdr->devnum, (_devnum)); \ 442 store_hw (&shdr->len, (_len)); \ 443 store_hw (&shdr->id, (_id)); \ 444 } while (0) 445 446 #define SHRD_GET_HDR(_buf, _cmd, _code, _devnum, _len, _id) \ 447 do { \ 448 SHRD_HDR *shdr = (SHRD_HDR *)(_buf); \ 449 (_cmd) = shdr->cmd; \ 450 (_code) = shdr->code; \ 451 (_devnum) = (U16)fetch_hw (&shdr->devnum); \ 452 (_len) = (int)fetch_hw (&shdr->len); \ 453 (_id) = (int)fetch_hw (&shdr->id); \ 454 } while (0) 455 456 int shared_update_notify (DEVBLK *dev, int block); 457 int shared_ckd_init (DEVBLK *dev, int argc, char *argv[] ); 458 int shared_fba_init (DEVBLK *dev, int argc, char *argv[] ); 459 SHR_DLL_IMPORT void *shared_server (void *arg); 460 SHR_DLL_IMPORT int shared_cmd(int argc, char *argv[], char *cmdline); 461 462 #ifdef _SHARED_C_ 463 static int shared_ckd_close ( DEVBLK *dev ); 464 static int shared_fba_close (DEVBLK *dev); 465 static void shared_start(DEVBLK *dev); 466 static void shared_end (DEVBLK *dev); 467 static int shared_ckd_read (DEVBLK *dev, int trk, BYTE *unitstat); 468 static int shared_ckd_write (DEVBLK *dev, int trk, int off, 469 BYTE *buf, int len, BYTE *unitstat); 470 static int shared_ckd_trklen (DEVBLK *dev, BYTE *buf); 471 472 #if defined(FBA_SHARED) 473 static int shared_fba_read (DEVBLK *dev, int blkgrp, BYTE *unitstat); 474 static int shared_fba_write (DEVBLK *dev, int blkgrp, int off, 475 BYTE *buf, int len, BYTE *unitstat); 476 static int shared_fba_blkgrp_len (DEVBLK *dev, int blkgrp); 477 #endif 478 479 static int shared_used (DEVBLK *dev); 480 static void shared_reserve (DEVBLK *dev); 481 static void shared_release (DEVBLK *dev); 482 static int clientWrite (DEVBLK *dev, int block); 483 static void clientPurge (DEVBLK *dev, int n, void *buf); 484 static int clientPurgescan (int *answer, int ix, int i, void *data); 485 static int clientConnect (DEVBLK *dev, int retry); 486 static int clientRequest (DEVBLK *dev, BYTE *buf, int len, int cmd, 487 int flags, int *code, int *status); 488 static int clientSend (DEVBLK *dev, BYTE *hdr, BYTE *buf, int buflen); 489 static int clientRecv (DEVBLK *dev, BYTE *hdr, BYTE *buf, int buflen); 490 static int recvData(int sock, BYTE *hdr, BYTE *buf, int buflen, int server); 491 static void serverRequest (DEVBLK *dev, int ix, BYTE *hdr, BYTE *buf); 492 static int serverLocate (DEVBLK *dev, int id, int *avail); 493 static int serverId (DEVBLK *dev); 494 static int serverError (DEVBLK *dev, int ix, int code, int status, 495 char *msg); 496 static int serverSend (DEVBLK *dev, int ix, BYTE *hdr, BYTE *buf, 497 int buflen); 498 static int serverDisconnectable (DEVBLK *dev, int ix); 499 static void serverDisconnect (DEVBLK *dev, int ix); 500 static char *clientip (int sock); 501 static DEVBLK *findDevice (U16 devnum); 502 static void *serverConnect (int *psock); 503 static void shrdtrc (DEVBLK *dev, char *msg, ...); 504 #endif /* _SHARED_C_ */ 505 506 #endif /* _HERCULES_SHARED_H */ 507