1 /* $NetBSD: rf_dagffrd.c,v 1.6 2001/11/13 07:11:13 lukem Exp $ */ 2 /* 3 * Copyright (c) 1995 Carnegie-Mellon University. 4 * All rights reserved. 5 * 6 * Author: Mark Holland, Daniel Stodolsky, William V. Courtright II 7 * 8 * Permission to use, copy, modify and distribute this software and 9 * its documentation is hereby granted, provided that both the copyright 10 * notice and this permission notice appear in all copies of the 11 * software, derivative works or modified versions, and any portions 12 * thereof, and that both notices appear in supporting documentation. 13 * 14 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 15 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 16 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 17 * 18 * Carnegie Mellon requests users of this software to return to 19 * 20 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 21 * School of Computer Science 22 * Carnegie Mellon University 23 * Pittsburgh PA 15213-3890 24 * 25 * any improvements or extensions that they make and grant Carnegie the 26 * rights to redistribute these changes. 27 */ 28 29 /* 30 * rf_dagffrd.c 31 * 32 * code for creating fault-free read DAGs 33 * 34 */ 35 36 #include <sys/cdefs.h> 37 __KERNEL_RCSID(0, "$NetBSD: rf_dagffrd.c,v 1.6 2001/11/13 07:11:13 lukem Exp $"); 38 39 #include <dev/raidframe/raidframevar.h> 40 41 #include "rf_raid.h" 42 #include "rf_dag.h" 43 #include "rf_dagutils.h" 44 #include "rf_dagfuncs.h" 45 #include "rf_debugMem.h" 46 #include "rf_memchunk.h" 47 #include "rf_general.h" 48 #include "rf_dagffrd.h" 49 50 /****************************************************************************** 51 * 52 * General comments on DAG creation: 53 * 54 * All DAGs in this file use roll-away error recovery. Each DAG has a single 55 * commit node, usually called "Cmt." If an error occurs before the Cmt node 56 * is reached, the execution engine will halt forward execution and work 57 * backward through the graph, executing the undo functions. Assuming that 58 * each node in the graph prior to the Cmt node are undoable and atomic - or - 59 * does not make changes to permanent state, the graph will fail atomically. 60 * If an error occurs after the Cmt node executes, the engine will roll-forward 61 * through the graph, blindly executing nodes until it reaches the end. 62 * If a graph reaches the end, it is assumed to have completed successfully. 63 * 64 * A graph has only 1 Cmt node. 65 * 66 */ 67 68 69 /****************************************************************************** 70 * 71 * The following wrappers map the standard DAG creation interface to the 72 * DAG creation routines. Additionally, these wrappers enable experimentation 73 * with new DAG structures by providing an extra level of indirection, allowing 74 * the DAG creation routines to be replaced at this single point. 75 */ 76 77 void 78 rf_CreateFaultFreeReadDAG( 79 RF_Raid_t * raidPtr, 80 RF_AccessStripeMap_t * asmap, 81 RF_DagHeader_t * dag_h, 82 void *bp, 83 RF_RaidAccessFlags_t flags, 84 RF_AllocListElem_t * allocList) 85 { 86 rf_CreateNonredundantDAG(raidPtr, asmap, dag_h, bp, flags, allocList, 87 RF_IO_TYPE_READ); 88 } 89 90 91 /****************************************************************************** 92 * 93 * DAG creation code begins here 94 */ 95 96 /****************************************************************************** 97 * 98 * creates a DAG to perform a nonredundant read or write of data within one 99 * stripe. 100 * For reads, this DAG is as follows: 101 * 102 * /---- read ----\ 103 * Header -- Block ---- read ---- Commit -- Terminate 104 * \---- read ----/ 105 * 106 * For writes, this DAG is as follows: 107 * 108 * /---- write ----\ 109 * Header -- Commit ---- write ---- Block -- Terminate 110 * \---- write ----/ 111 * 112 * There is one disk node per stripe unit accessed, and all disk nodes are in 113 * parallel. 114 * 115 * Tricky point here: The first disk node (read or write) is created 116 * normally. Subsequent disk nodes are created by copying the first one, 117 * and modifying a few params. The "succedents" and "antecedents" fields are 118 * _not_ re-created in each node, but rather left pointing to the same array 119 * that was malloc'd when the first node was created. Thus, it's essential 120 * that when this DAG is freed, the succedents and antecedents fields be freed 121 * in ONLY ONE of the read nodes. This does not apply to the "params" field 122 * because it is recreated for each READ node. 123 * 124 * Note that normal-priority accesses do not need to be tagged with their 125 * parity stripe ID, because they will never be promoted. Hence, I've 126 * commented-out the code to do this, and marked it with UNNEEDED. 127 * 128 *****************************************************************************/ 129 130 void 131 rf_CreateNonredundantDAG( 132 RF_Raid_t * raidPtr, 133 RF_AccessStripeMap_t * asmap, 134 RF_DagHeader_t * dag_h, 135 void *bp, 136 RF_RaidAccessFlags_t flags, 137 RF_AllocListElem_t * allocList, 138 RF_IoType_t type) 139 { 140 RF_DagNode_t *nodes, *diskNodes, *blockNode, *commitNode, *termNode; 141 RF_PhysDiskAddr_t *pda = asmap->physInfo; 142 int (*doFunc) (RF_DagNode_t *), (*undoFunc) (RF_DagNode_t *); 143 int i, n, totalNumNodes; 144 char *name; 145 146 n = asmap->numStripeUnitsAccessed; 147 dag_h->creator = "NonredundantDAG"; 148 149 RF_ASSERT(RF_IO_IS_R_OR_W(type)); 150 switch (type) { 151 case RF_IO_TYPE_READ: 152 doFunc = rf_DiskReadFunc; 153 undoFunc = rf_DiskReadUndoFunc; 154 name = "R "; 155 if (rf_dagDebug) 156 printf("[Creating non-redundant read DAG]\n"); 157 break; 158 case RF_IO_TYPE_WRITE: 159 doFunc = rf_DiskWriteFunc; 160 undoFunc = rf_DiskWriteUndoFunc; 161 name = "W "; 162 if (rf_dagDebug) 163 printf("[Creating non-redundant write DAG]\n"); 164 break; 165 default: 166 RF_PANIC(); 167 } 168 169 /* 170 * For reads, the dag can not commit until the block node is reached. 171 * for writes, the dag commits immediately. 172 */ 173 dag_h->numCommitNodes = 1; 174 dag_h->numCommits = 0; 175 dag_h->numSuccedents = 1; 176 177 /* 178 * Node count: 179 * 1 block node 180 * n data reads (or writes) 181 * 1 commit node 182 * 1 terminator node 183 */ 184 RF_ASSERT(n > 0); 185 totalNumNodes = n + 3; 186 RF_CallocAndAdd(nodes, totalNumNodes, sizeof(RF_DagNode_t), 187 (RF_DagNode_t *), allocList); 188 i = 0; 189 diskNodes = &nodes[i]; 190 i += n; 191 blockNode = &nodes[i]; 192 i += 1; 193 commitNode = &nodes[i]; 194 i += 1; 195 termNode = &nodes[i]; 196 i += 1; 197 RF_ASSERT(i == totalNumNodes); 198 199 /* initialize nodes */ 200 switch (type) { 201 case RF_IO_TYPE_READ: 202 rf_InitNode(blockNode, rf_wait, RF_FALSE, rf_NullNodeFunc, rf_NullNodeUndoFunc, 203 NULL, n, 0, 0, 0, dag_h, "Nil", allocList); 204 rf_InitNode(commitNode, rf_wait, RF_TRUE, rf_NullNodeFunc, rf_NullNodeUndoFunc, 205 NULL, 1, n, 0, 0, dag_h, "Cmt", allocList); 206 rf_InitNode(termNode, rf_wait, RF_FALSE, rf_TerminateFunc, rf_TerminateUndoFunc, 207 NULL, 0, 1, 0, 0, dag_h, "Trm", allocList); 208 break; 209 case RF_IO_TYPE_WRITE: 210 rf_InitNode(blockNode, rf_wait, RF_FALSE, rf_NullNodeFunc, rf_NullNodeUndoFunc, 211 NULL, 1, 0, 0, 0, dag_h, "Nil", allocList); 212 rf_InitNode(commitNode, rf_wait, RF_TRUE, rf_NullNodeFunc, rf_NullNodeUndoFunc, 213 NULL, n, 1, 0, 0, dag_h, "Cmt", allocList); 214 rf_InitNode(termNode, rf_wait, RF_FALSE, rf_TerminateFunc, rf_TerminateUndoFunc, 215 NULL, 0, n, 0, 0, dag_h, "Trm", allocList); 216 break; 217 default: 218 RF_PANIC(); 219 } 220 221 for (i = 0; i < n; i++) { 222 RF_ASSERT(pda != NULL); 223 rf_InitNode(&diskNodes[i], rf_wait, RF_FALSE, doFunc, undoFunc, rf_GenericWakeupFunc, 224 1, 1, 4, 0, dag_h, name, allocList); 225 diskNodes[i].params[0].p = pda; 226 diskNodes[i].params[1].p = pda->bufPtr; 227 /* parity stripe id is not necessary */ 228 diskNodes[i].params[2].v = 0; 229 diskNodes[i].params[3].v = RF_CREATE_PARAM3(RF_IO_NORMAL_PRIORITY, 0, 0, 0); 230 pda = pda->next; 231 } 232 233 /* 234 * Connect nodes. 235 */ 236 237 /* connect hdr to block node */ 238 RF_ASSERT(blockNode->numAntecedents == 0); 239 dag_h->succedents[0] = blockNode; 240 241 if (type == RF_IO_TYPE_READ) { 242 /* connecting a nonredundant read DAG */ 243 RF_ASSERT(blockNode->numSuccedents == n); 244 RF_ASSERT(commitNode->numAntecedents == n); 245 for (i = 0; i < n; i++) { 246 /* connect block node to each read node */ 247 RF_ASSERT(diskNodes[i].numAntecedents == 1); 248 blockNode->succedents[i] = &diskNodes[i]; 249 diskNodes[i].antecedents[0] = blockNode; 250 diskNodes[i].antType[0] = rf_control; 251 252 /* connect each read node to the commit node */ 253 RF_ASSERT(diskNodes[i].numSuccedents == 1); 254 diskNodes[i].succedents[0] = commitNode; 255 commitNode->antecedents[i] = &diskNodes[i]; 256 commitNode->antType[i] = rf_control; 257 } 258 /* connect the commit node to the term node */ 259 RF_ASSERT(commitNode->numSuccedents == 1); 260 RF_ASSERT(termNode->numAntecedents == 1); 261 RF_ASSERT(termNode->numSuccedents == 0); 262 commitNode->succedents[0] = termNode; 263 termNode->antecedents[0] = commitNode; 264 termNode->antType[0] = rf_control; 265 } else { 266 /* connecting a nonredundant write DAG */ 267 /* connect the block node to the commit node */ 268 RF_ASSERT(blockNode->numSuccedents == 1); 269 RF_ASSERT(commitNode->numAntecedents == 1); 270 blockNode->succedents[0] = commitNode; 271 commitNode->antecedents[0] = blockNode; 272 commitNode->antType[0] = rf_control; 273 274 RF_ASSERT(commitNode->numSuccedents == n); 275 RF_ASSERT(termNode->numAntecedents == n); 276 RF_ASSERT(termNode->numSuccedents == 0); 277 for (i = 0; i < n; i++) { 278 /* connect the commit node to each write node */ 279 RF_ASSERT(diskNodes[i].numAntecedents == 1); 280 commitNode->succedents[i] = &diskNodes[i]; 281 diskNodes[i].antecedents[0] = commitNode; 282 diskNodes[i].antType[0] = rf_control; 283 284 /* connect each write node to the term node */ 285 RF_ASSERT(diskNodes[i].numSuccedents == 1); 286 diskNodes[i].succedents[0] = termNode; 287 termNode->antecedents[i] = &diskNodes[i]; 288 termNode->antType[i] = rf_control; 289 } 290 } 291 } 292 /****************************************************************************** 293 * Create a fault-free read DAG for RAID level 1 294 * 295 * Hdr -> Nil -> Rmir -> Cmt -> Trm 296 * 297 * The "Rmir" node schedules a read from the disk in the mirror pair with the 298 * shortest disk queue. the proper queue is selected at Rmir execution. this 299 * deferred mapping is unlike other archs in RAIDframe which generally fix 300 * mapping at DAG creation time. 301 * 302 * Parameters: raidPtr - description of the physical array 303 * asmap - logical & physical addresses for this access 304 * bp - buffer ptr (for holding read data) 305 * flags - general flags (e.g. disk locking) 306 * allocList - list of memory allocated in DAG creation 307 *****************************************************************************/ 308 309 static void 310 CreateMirrorReadDAG( 311 RF_Raid_t * raidPtr, 312 RF_AccessStripeMap_t * asmap, 313 RF_DagHeader_t * dag_h, 314 void *bp, 315 RF_RaidAccessFlags_t flags, 316 RF_AllocListElem_t * allocList, 317 int (*readfunc) (RF_DagNode_t * node)) 318 { 319 RF_DagNode_t *readNodes, *nodes, *blockNode, *commitNode, *termNode; 320 RF_PhysDiskAddr_t *data_pda = asmap->physInfo; 321 RF_PhysDiskAddr_t *parity_pda = asmap->parityInfo; 322 int i, n, totalNumNodes; 323 324 n = asmap->numStripeUnitsAccessed; 325 dag_h->creator = "RaidOneReadDAG"; 326 if (rf_dagDebug) { 327 printf("[Creating RAID level 1 read DAG]\n"); 328 } 329 /* 330 * This dag can not commit until the commit node is reached 331 * errors prior to the commit point imply the dag has failed. 332 */ 333 dag_h->numCommitNodes = 1; 334 dag_h->numCommits = 0; 335 dag_h->numSuccedents = 1; 336 337 /* 338 * Node count: 339 * n data reads 340 * 1 block node 341 * 1 commit node 342 * 1 terminator node 343 */ 344 RF_ASSERT(n > 0); 345 totalNumNodes = n + 3; 346 RF_CallocAndAdd(nodes, totalNumNodes, sizeof(RF_DagNode_t), 347 (RF_DagNode_t *), allocList); 348 i = 0; 349 readNodes = &nodes[i]; 350 i += n; 351 blockNode = &nodes[i]; 352 i += 1; 353 commitNode = &nodes[i]; 354 i += 1; 355 termNode = &nodes[i]; 356 i += 1; 357 RF_ASSERT(i == totalNumNodes); 358 359 /* initialize nodes */ 360 rf_InitNode(blockNode, rf_wait, RF_FALSE, rf_NullNodeFunc, 361 rf_NullNodeUndoFunc, NULL, n, 0, 0, 0, dag_h, "Nil", allocList); 362 rf_InitNode(commitNode, rf_wait, RF_TRUE, rf_NullNodeFunc, 363 rf_NullNodeUndoFunc, NULL, 1, n, 0, 0, dag_h, "Cmt", allocList); 364 rf_InitNode(termNode, rf_wait, RF_FALSE, rf_TerminateFunc, 365 rf_TerminateUndoFunc, NULL, 0, 1, 0, 0, dag_h, "Trm", allocList); 366 367 for (i = 0; i < n; i++) { 368 RF_ASSERT(data_pda != NULL); 369 RF_ASSERT(parity_pda != NULL); 370 rf_InitNode(&readNodes[i], rf_wait, RF_FALSE, readfunc, 371 rf_DiskReadMirrorUndoFunc, rf_GenericWakeupFunc, 1, 1, 5, 0, dag_h, 372 "Rmir", allocList); 373 readNodes[i].params[0].p = data_pda; 374 readNodes[i].params[1].p = data_pda->bufPtr; 375 /* parity stripe id is not necessary */ 376 readNodes[i].params[2].p = 0; 377 readNodes[i].params[3].v = RF_CREATE_PARAM3(RF_IO_NORMAL_PRIORITY, 0, 0, 0); 378 readNodes[i].params[4].p = parity_pda; 379 data_pda = data_pda->next; 380 parity_pda = parity_pda->next; 381 } 382 383 /* 384 * Connect nodes 385 */ 386 387 /* connect hdr to block node */ 388 RF_ASSERT(blockNode->numAntecedents == 0); 389 dag_h->succedents[0] = blockNode; 390 391 /* connect block node to read nodes */ 392 RF_ASSERT(blockNode->numSuccedents == n); 393 for (i = 0; i < n; i++) { 394 RF_ASSERT(readNodes[i].numAntecedents == 1); 395 blockNode->succedents[i] = &readNodes[i]; 396 readNodes[i].antecedents[0] = blockNode; 397 readNodes[i].antType[0] = rf_control; 398 } 399 400 /* connect read nodes to commit node */ 401 RF_ASSERT(commitNode->numAntecedents == n); 402 for (i = 0; i < n; i++) { 403 RF_ASSERT(readNodes[i].numSuccedents == 1); 404 readNodes[i].succedents[0] = commitNode; 405 commitNode->antecedents[i] = &readNodes[i]; 406 commitNode->antType[i] = rf_control; 407 } 408 409 /* connect commit node to term node */ 410 RF_ASSERT(commitNode->numSuccedents == 1); 411 RF_ASSERT(termNode->numAntecedents == 1); 412 RF_ASSERT(termNode->numSuccedents == 0); 413 commitNode->succedents[0] = termNode; 414 termNode->antecedents[0] = commitNode; 415 termNode->antType[0] = rf_control; 416 } 417 418 void 419 rf_CreateMirrorIdleReadDAG( 420 RF_Raid_t * raidPtr, 421 RF_AccessStripeMap_t * asmap, 422 RF_DagHeader_t * dag_h, 423 void *bp, 424 RF_RaidAccessFlags_t flags, 425 RF_AllocListElem_t * allocList) 426 { 427 CreateMirrorReadDAG(raidPtr, asmap, dag_h, bp, flags, allocList, 428 rf_DiskReadMirrorIdleFunc); 429 } 430 431 void 432 rf_CreateMirrorPartitionReadDAG( 433 RF_Raid_t * raidPtr, 434 RF_AccessStripeMap_t * asmap, 435 RF_DagHeader_t * dag_h, 436 void *bp, 437 RF_RaidAccessFlags_t flags, 438 RF_AllocListElem_t * allocList) 439 { 440 CreateMirrorReadDAG(raidPtr, asmap, dag_h, bp, flags, allocList, 441 rf_DiskReadMirrorPartitionFunc); 442 } 443