1<?php
2/**
3 * Helper class for representing batch file operations.
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
19 *
20 * @file
21 * @ingroup FileBackend
22 */
23
24/**
25 * Helper class for representing batch file operations.
26 * Do not use this class from places outside FileBackend.
27 *
28 * Methods should avoid throwing exceptions at all costs.
29 *
30 * @ingroup FileBackend
31 * @since 1.20
32 */
33class FileOpBatch {
34	/* Timeout related parameters */
35	private const MAX_BATCH_SIZE = 1000; // integer
36
37	/**
38	 * Attempt to perform a series of file operations.
39	 * Callers are responsible for handling file locking.
40	 *
41	 * $opts is an array of options, including:
42	 *   - force        : Errors that would normally cause a rollback do not.
43	 *                    The remaining operations are still attempted if any fail.
44	 *   - nonJournaled : Don't log this operation batch in the file journal.
45	 *   - concurrency  : Try to do this many operations in parallel when possible.
46	 *
47	 * The resulting StatusValue will be "OK" unless:
48	 *   - a) unexpected operation errors occurred (network partitions, disk full...)
49	 *   - b) predicted operation errors occurred and 'force' was not set
50	 *
51	 * @param FileOp[] $performOps List of FileOp operations
52	 * @param array $opts Batch operation options
53	 * @param FileJournal $journal Journal to log operations to
54	 * @return StatusValue
55	 */
56	public static function attempt( array $performOps, array $opts, FileJournal $journal ) {
57		$status = StatusValue::newGood();
58
59		$n = count( $performOps );
60		if ( $n > self::MAX_BATCH_SIZE ) {
61			$status->fatal( 'backend-fail-batchsize', $n, self::MAX_BATCH_SIZE );
62
63			return $status;
64		}
65
66		$batchId = $journal->getTimestampedUUID();
67		$ignoreErrors = !empty( $opts['force'] );
68		$journaled = empty( $opts['nonJournaled'] );
69		$maxConcurrency = $opts['concurrency'] ?? 1;
70
71		$entries = []; // file journal entry list
72		$predicates = FileOp::newPredicates(); // account for previous ops in prechecks
73		$curBatch = []; // concurrent FileOp sub-batch accumulation
74		$curBatchDeps = FileOp::newDependencies(); // paths used in FileOp sub-batch
75		$pPerformOps = []; // ordered list of concurrent FileOp sub-batches
76		$lastBackend = null; // last op backend name
77		// Do pre-checks for each operation; abort on failure...
78		foreach ( $performOps as $index => $fileOp ) {
79			$backendName = $fileOp->getBackend()->getName();
80			$fileOp->setBatchId( $batchId ); // transaction ID
81			// Decide if this op can be done concurrently within this sub-batch
82			// or if a new concurrent sub-batch must be started after this one...
83			if ( $fileOp->dependsOn( $curBatchDeps )
84				|| count( $curBatch ) >= $maxConcurrency
85				|| ( $backendName !== $lastBackend && count( $curBatch ) )
86			) {
87				$pPerformOps[] = $curBatch; // push this batch
88				$curBatch = []; // start a new sub-batch
89				$curBatchDeps = FileOp::newDependencies();
90			}
91			$lastBackend = $backendName;
92			$curBatch[$index] = $fileOp; // keep index
93			// Update list of affected paths in this batch
94			$curBatchDeps = $fileOp->applyDependencies( $curBatchDeps );
95			// Simulate performing the operation...
96			$oldPredicates = $predicates;
97			$subStatus = $fileOp->precheck( $predicates ); // updates $predicates
98			$status->merge( $subStatus );
99			if ( $subStatus->isOK() ) {
100				if ( $journaled ) { // journal log entries
101					$entries = array_merge( $entries,
102						$fileOp->getJournalEntries( $oldPredicates, $predicates ) );
103				}
104			} else { // operation failed?
105				$status->success[$index] = false;
106				++$status->failCount;
107				if ( !$ignoreErrors ) {
108					return $status; // abort
109				}
110			}
111		}
112		// Push the last sub-batch
113		if ( count( $curBatch ) ) {
114			$pPerformOps[] = $curBatch;
115		}
116
117		// Log the operations in the file journal...
118		if ( count( $entries ) ) {
119			$subStatus = $journal->logChangeBatch( $entries, $batchId );
120			if ( !$subStatus->isOK() ) {
121				$status->merge( $subStatus );
122
123				return $status; // abort
124			}
125		}
126
127		if ( $ignoreErrors ) { // treat precheck() fatals as mere warnings
128			$status->setResult( true, $status->value );
129		}
130
131		// Attempt each operation (in parallel if allowed and possible)...
132		self::runParallelBatches( $pPerformOps, $status );
133
134		return $status;
135	}
136
137	/**
138	 * Attempt a list of file operations sub-batches in series.
139	 *
140	 * The operations *in* each sub-batch will be done in parallel.
141	 * The caller is responsible for making sure the operations
142	 * within any given sub-batch do not depend on each other.
143	 * This will abort remaining ops on failure.
144	 *
145	 * @param FileOp[][] $pPerformOps Batches of file ops (batches use original indexes)
146	 * @param StatusValue $status
147	 */
148	protected static function runParallelBatches( array $pPerformOps, StatusValue $status ) {
149		$aborted = false; // set to true on unexpected errors
150		foreach ( $pPerformOps as $performOpsBatch ) {
151			if ( $aborted ) { // check batch op abort flag...
152				// We can't continue (even with $ignoreErrors) as $predicates is wrong.
153				// Log the remaining ops as failed for recovery...
154				foreach ( $performOpsBatch as $i => $fileOp ) {
155					$status->success[$i] = false;
156					++$status->failCount;
157					$fileOp->logFailure( 'attempt_aborted' );
158				}
159				continue;
160			}
161			/** @var StatusValue[] $statuses */
162			$statuses = [];
163			$opHandles = [];
164			// Get the backend; all sub-batch ops belong to a single backend
165			/** @var FileBackendStore $backend */
166			$backend = reset( $performOpsBatch )->getBackend();
167			// Get the operation handles or actually do it if there is just one.
168			// If attemptAsync() returns a StatusValue, it was either due to an error
169			// or the backend does not support async ops and did it synchronously.
170			foreach ( $performOpsBatch as $i => $fileOp ) {
171				if ( !isset( $status->success[$i] ) ) { // didn't already fail in precheck()
172					// Parallel ops may be disabled in config due to missing dependencies,
173					// (e.g. needing popen()). When they are, $performOpsBatch has size 1.
174					$subStatus = ( count( $performOpsBatch ) > 1 )
175						? $fileOp->attemptAsync()
176						: $fileOp->attempt();
177					if ( $subStatus->value instanceof FileBackendStoreOpHandle ) {
178						$opHandles[$i] = $subStatus->value; // deferred
179					} else {
180						$statuses[$i] = $subStatus; // done already
181					}
182				}
183			}
184			// Try to do all the operations concurrently...
185			$statuses += $backend->executeOpHandlesInternal( $opHandles );
186			// Marshall and merge all the responses (blocking)...
187			foreach ( $performOpsBatch as $i => $fileOp ) {
188				if ( !isset( $status->success[$i] ) ) { // didn't already fail in precheck()
189					$subStatus = $statuses[$i];
190					$status->merge( $subStatus );
191					if ( $subStatus->isOK() ) {
192						$status->success[$i] = true;
193						++$status->successCount;
194					} else {
195						$status->success[$i] = false;
196						++$status->failCount;
197						$aborted = true; // set abort flag; we can't continue
198					}
199				}
200			}
201		}
202	}
203}
204