1 /* $NetBSD: rf_dagffrd.c,v 1.8 2002/09/21 00:40:18 oster Exp $ */ 2 /* 3 * Copyright (c) 1995 Carnegie-Mellon University. 4 * All rights reserved. 5 * 6 * Author: Mark Holland, Daniel Stodolsky, William V. Courtright II 7 * 8 * Permission to use, copy, modify and distribute this software and 9 * its documentation is hereby granted, provided that both the copyright 10 * notice and this permission notice appear in all copies of the 11 * software, derivative works or modified versions, and any portions 12 * thereof, and that both notices appear in supporting documentation. 13 * 14 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 15 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 16 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 17 * 18 * Carnegie Mellon requests users of this software to return to 19 * 20 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 21 * School of Computer Science 22 * Carnegie Mellon University 23 * Pittsburgh PA 15213-3890 24 * 25 * any improvements or extensions that they make and grant Carnegie the 26 * rights to redistribute these changes. 27 */ 28 29 /* 30 * rf_dagffrd.c 31 * 32 * code for creating fault-free read DAGs 33 * 34 */ 35 36 #include <sys/cdefs.h> 37 __KERNEL_RCSID(0, "$NetBSD: rf_dagffrd.c,v 1.8 2002/09/21 00:40:18 oster Exp $"); 38 39 #include <dev/raidframe/raidframevar.h> 40 41 #include "rf_raid.h" 42 #include "rf_dag.h" 43 #include "rf_dagutils.h" 44 #include "rf_dagfuncs.h" 45 #include "rf_debugMem.h" 46 #include "rf_general.h" 47 #include "rf_dagffrd.h" 48 49 /****************************************************************************** 50 * 51 * General comments on DAG creation: 52 * 53 * All DAGs in this file use roll-away error recovery. Each DAG has a single 54 * commit node, usually called "Cmt." If an error occurs before the Cmt node 55 * is reached, the execution engine will halt forward execution and work 56 * backward through the graph, executing the undo functions. Assuming that 57 * each node in the graph prior to the Cmt node are undoable and atomic - or - 58 * does not make changes to permanent state, the graph will fail atomically. 59 * If an error occurs after the Cmt node executes, the engine will roll-forward 60 * through the graph, blindly executing nodes until it reaches the end. 61 * If a graph reaches the end, it is assumed to have completed successfully. 62 * 63 * A graph has only 1 Cmt node. 64 * 65 */ 66 67 68 /****************************************************************************** 69 * 70 * The following wrappers map the standard DAG creation interface to the 71 * DAG creation routines. Additionally, these wrappers enable experimentation 72 * with new DAG structures by providing an extra level of indirection, allowing 73 * the DAG creation routines to be replaced at this single point. 74 */ 75 76 void 77 rf_CreateFaultFreeReadDAG( 78 RF_Raid_t * raidPtr, 79 RF_AccessStripeMap_t * asmap, 80 RF_DagHeader_t * dag_h, 81 void *bp, 82 RF_RaidAccessFlags_t flags, 83 RF_AllocListElem_t * allocList) 84 { 85 rf_CreateNonredundantDAG(raidPtr, asmap, dag_h, bp, flags, allocList, 86 RF_IO_TYPE_READ); 87 } 88 89 90 /****************************************************************************** 91 * 92 * DAG creation code begins here 93 */ 94 95 /****************************************************************************** 96 * 97 * creates a DAG to perform a nonredundant read or write of data within one 98 * stripe. 99 * For reads, this DAG is as follows: 100 * 101 * /---- read ----\ 102 * Header -- Block ---- read ---- Commit -- Terminate 103 * \---- read ----/ 104 * 105 * For writes, this DAG is as follows: 106 * 107 * /---- write ----\ 108 * Header -- Commit ---- write ---- Block -- Terminate 109 * \---- write ----/ 110 * 111 * There is one disk node per stripe unit accessed, and all disk nodes are in 112 * parallel. 113 * 114 * Tricky point here: The first disk node (read or write) is created 115 * normally. Subsequent disk nodes are created by copying the first one, 116 * and modifying a few params. The "succedents" and "antecedents" fields are 117 * _not_ re-created in each node, but rather left pointing to the same array 118 * that was malloc'd when the first node was created. Thus, it's essential 119 * that when this DAG is freed, the succedents and antecedents fields be freed 120 * in ONLY ONE of the read nodes. This does not apply to the "params" field 121 * because it is recreated for each READ node. 122 * 123 * Note that normal-priority accesses do not need to be tagged with their 124 * parity stripe ID, because they will never be promoted. Hence, I've 125 * commented-out the code to do this, and marked it with UNNEEDED. 126 * 127 *****************************************************************************/ 128 129 void 130 rf_CreateNonredundantDAG( 131 RF_Raid_t * raidPtr, 132 RF_AccessStripeMap_t * asmap, 133 RF_DagHeader_t * dag_h, 134 void *bp, 135 RF_RaidAccessFlags_t flags, 136 RF_AllocListElem_t * allocList, 137 RF_IoType_t type) 138 { 139 RF_DagNode_t *nodes, *diskNodes, *blockNode, *commitNode, *termNode; 140 RF_PhysDiskAddr_t *pda = asmap->physInfo; 141 int (*doFunc) (RF_DagNode_t *), (*undoFunc) (RF_DagNode_t *); 142 int i, n, totalNumNodes; 143 char *name; 144 145 n = asmap->numStripeUnitsAccessed; 146 dag_h->creator = "NonredundantDAG"; 147 148 RF_ASSERT(RF_IO_IS_R_OR_W(type)); 149 switch (type) { 150 case RF_IO_TYPE_READ: 151 doFunc = rf_DiskReadFunc; 152 undoFunc = rf_DiskReadUndoFunc; 153 name = "R "; 154 if (rf_dagDebug) 155 printf("[Creating non-redundant read DAG]\n"); 156 break; 157 case RF_IO_TYPE_WRITE: 158 doFunc = rf_DiskWriteFunc; 159 undoFunc = rf_DiskWriteUndoFunc; 160 name = "W "; 161 if (rf_dagDebug) 162 printf("[Creating non-redundant write DAG]\n"); 163 break; 164 default: 165 RF_PANIC(); 166 } 167 168 /* 169 * For reads, the dag can not commit until the block node is reached. 170 * for writes, the dag commits immediately. 171 */ 172 dag_h->numCommitNodes = 1; 173 dag_h->numCommits = 0; 174 dag_h->numSuccedents = 1; 175 176 /* 177 * Node count: 178 * 1 block node 179 * n data reads (or writes) 180 * 1 commit node 181 * 1 terminator node 182 */ 183 RF_ASSERT(n > 0); 184 totalNumNodes = n + 3; 185 RF_CallocAndAdd(nodes, totalNumNodes, sizeof(RF_DagNode_t), 186 (RF_DagNode_t *), allocList); 187 i = 0; 188 diskNodes = &nodes[i]; 189 i += n; 190 blockNode = &nodes[i]; 191 i += 1; 192 commitNode = &nodes[i]; 193 i += 1; 194 termNode = &nodes[i]; 195 i += 1; 196 RF_ASSERT(i == totalNumNodes); 197 198 /* initialize nodes */ 199 switch (type) { 200 case RF_IO_TYPE_READ: 201 rf_InitNode(blockNode, rf_wait, RF_FALSE, rf_NullNodeFunc, rf_NullNodeUndoFunc, 202 NULL, n, 0, 0, 0, dag_h, "Nil", allocList); 203 rf_InitNode(commitNode, rf_wait, RF_TRUE, rf_NullNodeFunc, rf_NullNodeUndoFunc, 204 NULL, 1, n, 0, 0, dag_h, "Cmt", allocList); 205 rf_InitNode(termNode, rf_wait, RF_FALSE, rf_TerminateFunc, rf_TerminateUndoFunc, 206 NULL, 0, 1, 0, 0, dag_h, "Trm", allocList); 207 break; 208 case RF_IO_TYPE_WRITE: 209 rf_InitNode(blockNode, rf_wait, RF_FALSE, rf_NullNodeFunc, rf_NullNodeUndoFunc, 210 NULL, 1, 0, 0, 0, dag_h, "Nil", allocList); 211 rf_InitNode(commitNode, rf_wait, RF_TRUE, rf_NullNodeFunc, rf_NullNodeUndoFunc, 212 NULL, n, 1, 0, 0, dag_h, "Cmt", allocList); 213 rf_InitNode(termNode, rf_wait, RF_FALSE, rf_TerminateFunc, rf_TerminateUndoFunc, 214 NULL, 0, n, 0, 0, dag_h, "Trm", allocList); 215 break; 216 default: 217 RF_PANIC(); 218 } 219 220 for (i = 0; i < n; i++) { 221 RF_ASSERT(pda != NULL); 222 rf_InitNode(&diskNodes[i], rf_wait, RF_FALSE, doFunc, undoFunc, rf_GenericWakeupFunc, 223 1, 1, 4, 0, dag_h, name, allocList); 224 diskNodes[i].params[0].p = pda; 225 diskNodes[i].params[1].p = pda->bufPtr; 226 /* parity stripe id is not necessary */ 227 diskNodes[i].params[2].v = 0; 228 diskNodes[i].params[3].v = RF_CREATE_PARAM3(RF_IO_NORMAL_PRIORITY, 0, 0, 0); 229 pda = pda->next; 230 } 231 232 /* 233 * Connect nodes. 234 */ 235 236 /* connect hdr to block node */ 237 RF_ASSERT(blockNode->numAntecedents == 0); 238 dag_h->succedents[0] = blockNode; 239 240 if (type == RF_IO_TYPE_READ) { 241 /* connecting a nonredundant read DAG */ 242 RF_ASSERT(blockNode->numSuccedents == n); 243 RF_ASSERT(commitNode->numAntecedents == n); 244 for (i = 0; i < n; i++) { 245 /* connect block node to each read node */ 246 RF_ASSERT(diskNodes[i].numAntecedents == 1); 247 blockNode->succedents[i] = &diskNodes[i]; 248 diskNodes[i].antecedents[0] = blockNode; 249 diskNodes[i].antType[0] = rf_control; 250 251 /* connect each read node to the commit node */ 252 RF_ASSERT(diskNodes[i].numSuccedents == 1); 253 diskNodes[i].succedents[0] = commitNode; 254 commitNode->antecedents[i] = &diskNodes[i]; 255 commitNode->antType[i] = rf_control; 256 } 257 /* connect the commit node to the term node */ 258 RF_ASSERT(commitNode->numSuccedents == 1); 259 RF_ASSERT(termNode->numAntecedents == 1); 260 RF_ASSERT(termNode->numSuccedents == 0); 261 commitNode->succedents[0] = termNode; 262 termNode->antecedents[0] = commitNode; 263 termNode->antType[0] = rf_control; 264 } else { 265 /* connecting a nonredundant write DAG */ 266 /* connect the block node to the commit node */ 267 RF_ASSERT(blockNode->numSuccedents == 1); 268 RF_ASSERT(commitNode->numAntecedents == 1); 269 blockNode->succedents[0] = commitNode; 270 commitNode->antecedents[0] = blockNode; 271 commitNode->antType[0] = rf_control; 272 273 RF_ASSERT(commitNode->numSuccedents == n); 274 RF_ASSERT(termNode->numAntecedents == n); 275 RF_ASSERT(termNode->numSuccedents == 0); 276 for (i = 0; i < n; i++) { 277 /* connect the commit node to each write node */ 278 RF_ASSERT(diskNodes[i].numAntecedents == 1); 279 commitNode->succedents[i] = &diskNodes[i]; 280 diskNodes[i].antecedents[0] = commitNode; 281 diskNodes[i].antType[0] = rf_control; 282 283 /* connect each write node to the term node */ 284 RF_ASSERT(diskNodes[i].numSuccedents == 1); 285 diskNodes[i].succedents[0] = termNode; 286 termNode->antecedents[i] = &diskNodes[i]; 287 termNode->antType[i] = rf_control; 288 } 289 } 290 } 291 /****************************************************************************** 292 * Create a fault-free read DAG for RAID level 1 293 * 294 * Hdr -> Nil -> Rmir -> Cmt -> Trm 295 * 296 * The "Rmir" node schedules a read from the disk in the mirror pair with the 297 * shortest disk queue. the proper queue is selected at Rmir execution. this 298 * deferred mapping is unlike other archs in RAIDframe which generally fix 299 * mapping at DAG creation time. 300 * 301 * Parameters: raidPtr - description of the physical array 302 * asmap - logical & physical addresses for this access 303 * bp - buffer ptr (for holding read data) 304 * flags - general flags (e.g. disk locking) 305 * allocList - list of memory allocated in DAG creation 306 *****************************************************************************/ 307 308 static void 309 CreateMirrorReadDAG( 310 RF_Raid_t * raidPtr, 311 RF_AccessStripeMap_t * asmap, 312 RF_DagHeader_t * dag_h, 313 void *bp, 314 RF_RaidAccessFlags_t flags, 315 RF_AllocListElem_t * allocList, 316 int (*readfunc) (RF_DagNode_t * node)) 317 { 318 RF_DagNode_t *readNodes, *nodes, *blockNode, *commitNode, *termNode; 319 RF_PhysDiskAddr_t *data_pda = asmap->physInfo; 320 RF_PhysDiskAddr_t *parity_pda = asmap->parityInfo; 321 int i, n, totalNumNodes; 322 323 n = asmap->numStripeUnitsAccessed; 324 dag_h->creator = "RaidOneReadDAG"; 325 if (rf_dagDebug) { 326 printf("[Creating RAID level 1 read DAG]\n"); 327 } 328 /* 329 * This dag can not commit until the commit node is reached 330 * errors prior to the commit point imply the dag has failed. 331 */ 332 dag_h->numCommitNodes = 1; 333 dag_h->numCommits = 0; 334 dag_h->numSuccedents = 1; 335 336 /* 337 * Node count: 338 * n data reads 339 * 1 block node 340 * 1 commit node 341 * 1 terminator node 342 */ 343 RF_ASSERT(n > 0); 344 totalNumNodes = n + 3; 345 RF_CallocAndAdd(nodes, totalNumNodes, sizeof(RF_DagNode_t), 346 (RF_DagNode_t *), allocList); 347 i = 0; 348 readNodes = &nodes[i]; 349 i += n; 350 blockNode = &nodes[i]; 351 i += 1; 352 commitNode = &nodes[i]; 353 i += 1; 354 termNode = &nodes[i]; 355 i += 1; 356 RF_ASSERT(i == totalNumNodes); 357 358 /* initialize nodes */ 359 rf_InitNode(blockNode, rf_wait, RF_FALSE, rf_NullNodeFunc, 360 rf_NullNodeUndoFunc, NULL, n, 0, 0, 0, dag_h, "Nil", allocList); 361 rf_InitNode(commitNode, rf_wait, RF_TRUE, rf_NullNodeFunc, 362 rf_NullNodeUndoFunc, NULL, 1, n, 0, 0, dag_h, "Cmt", allocList); 363 rf_InitNode(termNode, rf_wait, RF_FALSE, rf_TerminateFunc, 364 rf_TerminateUndoFunc, NULL, 0, 1, 0, 0, dag_h, "Trm", allocList); 365 366 for (i = 0; i < n; i++) { 367 RF_ASSERT(data_pda != NULL); 368 RF_ASSERT(parity_pda != NULL); 369 rf_InitNode(&readNodes[i], rf_wait, RF_FALSE, readfunc, 370 rf_DiskReadMirrorUndoFunc, rf_GenericWakeupFunc, 1, 1, 5, 0, dag_h, 371 "Rmir", allocList); 372 readNodes[i].params[0].p = data_pda; 373 readNodes[i].params[1].p = data_pda->bufPtr; 374 /* parity stripe id is not necessary */ 375 readNodes[i].params[2].p = 0; 376 readNodes[i].params[3].v = RF_CREATE_PARAM3(RF_IO_NORMAL_PRIORITY, 0, 0, 0); 377 readNodes[i].params[4].p = parity_pda; 378 data_pda = data_pda->next; 379 parity_pda = parity_pda->next; 380 } 381 382 /* 383 * Connect nodes 384 */ 385 386 /* connect hdr to block node */ 387 RF_ASSERT(blockNode->numAntecedents == 0); 388 dag_h->succedents[0] = blockNode; 389 390 /* connect block node to read nodes */ 391 RF_ASSERT(blockNode->numSuccedents == n); 392 for (i = 0; i < n; i++) { 393 RF_ASSERT(readNodes[i].numAntecedents == 1); 394 blockNode->succedents[i] = &readNodes[i]; 395 readNodes[i].antecedents[0] = blockNode; 396 readNodes[i].antType[0] = rf_control; 397 } 398 399 /* connect read nodes to commit node */ 400 RF_ASSERT(commitNode->numAntecedents == n); 401 for (i = 0; i < n; i++) { 402 RF_ASSERT(readNodes[i].numSuccedents == 1); 403 readNodes[i].succedents[0] = commitNode; 404 commitNode->antecedents[i] = &readNodes[i]; 405 commitNode->antType[i] = rf_control; 406 } 407 408 /* connect commit node to term node */ 409 RF_ASSERT(commitNode->numSuccedents == 1); 410 RF_ASSERT(termNode->numAntecedents == 1); 411 RF_ASSERT(termNode->numSuccedents == 0); 412 commitNode->succedents[0] = termNode; 413 termNode->antecedents[0] = commitNode; 414 termNode->antType[0] = rf_control; 415 } 416 417 void 418 rf_CreateMirrorIdleReadDAG( 419 RF_Raid_t * raidPtr, 420 RF_AccessStripeMap_t * asmap, 421 RF_DagHeader_t * dag_h, 422 void *bp, 423 RF_RaidAccessFlags_t flags, 424 RF_AllocListElem_t * allocList) 425 { 426 CreateMirrorReadDAG(raidPtr, asmap, dag_h, bp, flags, allocList, 427 rf_DiskReadMirrorIdleFunc); 428 } 429 430 #if (RF_INCLUDE_CHAINDECLUSTER > 0) || (RF_INCLUDE_INTERDECLUSTER > 0) 431 432 void 433 rf_CreateMirrorPartitionReadDAG( 434 RF_Raid_t * raidPtr, 435 RF_AccessStripeMap_t * asmap, 436 RF_DagHeader_t * dag_h, 437 void *bp, 438 RF_RaidAccessFlags_t flags, 439 RF_AllocListElem_t * allocList) 440 { 441 CreateMirrorReadDAG(raidPtr, asmap, dag_h, bp, flags, allocList, 442 rf_DiskReadMirrorPartitionFunc); 443 } 444 #endif 445