1 //===-- NinjaBuildCommand.cpp ---------------------------------------------===//
2 //
3 // This source file is part of the Swift.org open source project
4 //
5 // Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors
6 // Licensed under Apache License v2.0 with Runtime Library Exception
7 //
8 // See http://swift.org/LICENSE.txt for license information
9 // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "NinjaBuildCommand.h"
14
15 #include "llbuild/Basic/Compiler.h"
16 #include "llbuild/Basic/FileInfo.h"
17 #include "llbuild/Basic/Hashing.h"
18 #include "llbuild/Basic/PlatformUtility.h"
19 #include "llbuild/Basic/SerialQueue.h"
20 #include "llbuild/Basic/Version.h"
21 #include "llbuild/Commands/Commands.h"
22 #include "llbuild/Core/BuildDB.h"
23 #include "llbuild/Core/BuildEngine.h"
24 #include "llbuild/Core/MakefileDepsParser.h"
25 #include "llbuild/Ninja/ManifestLoader.h"
26
27 #include "llvm/ADT/SmallString.h"
28 #include "llvm/Support/TimeValue.h"
29
30 #include "CommandLineStatusOutput.h"
31 #include "CommandUtil.h"
32
33 #include <atomic>
34 #include <cerrno>
35 #include <chrono>
36 #include <condition_variable>
37 #include <cstdarg>
38 #include <cstdlib>
39 #include <deque>
40 #include <mutex>
41 #include <sstream>
42 #include <thread>
43 #include <unordered_set>
44
45 #include <fcntl.h>
46 #include <signal.h>
47 #include <spawn.h>
48 #include <unistd.h>
49 #include <sys/stat.h>
50 #include <sys/wait.h>
51
52 using namespace llbuild;
53 using namespace llbuild::basic;
54 using namespace llbuild::commands;
55
56 #if !defined(_WIN32)
57 extern "C" {
58 extern char **environ;
59 }
60 #endif
61
getTimeInMicroseconds()62 static uint64_t getTimeInMicroseconds() {
63 llvm::sys::TimeValue now = llvm::sys::TimeValue::now();
64 return now.msec();
65 }
66
getFormattedString(const char * fmt,va_list ap1)67 static std::string getFormattedString(const char* fmt, va_list ap1) {
68 va_list ap2;
69 va_copy(ap2, ap1);
70 int count = vsnprintf(NULL, 0, fmt, ap1);
71 if (count <= 0) {
72 return "unable to format message";
73 }
74
75 std::string result = std::string(count, '\0');
76 if (vsnprintf(const_cast<char *>(result.c_str()), count + 1, fmt, ap2) < 0) {
77 return "unable to format message";
78 }
79
80 return result;
81 }
82
getFormattedString(const char * fmt,...)83 static std::string getFormattedString(const char* fmt, ...) {
84 va_list ap;
85 va_start(ap, fmt);
86 std::string result = getFormattedString(fmt, ap);
87 va_end(ap);
88 return result;
89 }
90
usage(int exitCode=1)91 static void usage(int exitCode=1) {
92 int optionWidth = 20;
93 fprintf(stderr, "Usage: %s ninja build [options] [<targets>...]\n",
94 getProgramName());
95 fprintf(stderr, "\nOptions:\n");
96 fprintf(stderr, " %-*s %s\n", optionWidth, "--help",
97 "show this help message and exit");
98 fprintf(stderr, " %-*s %s\n", optionWidth, "--version",
99 "print the Ninja compatible version number");
100 fprintf(stderr, " %-*s %s\n", optionWidth, "--simulate",
101 "simulate the build, assuming commands succeed");
102 fprintf(stderr, " %-*s %s\n", optionWidth, "-C, --chdir <PATH>",
103 "change directory to PATH before anything else");
104 fprintf(stderr, " %-*s %s\n", optionWidth, "--no-db",
105 "do not persist build results");
106 fprintf(stderr, " %-*s %s\n", optionWidth, "--db <PATH>",
107 "persist build results at PATH [default='build.db']");
108 fprintf(stderr, " %-*s %s\n", optionWidth, "-f <PATH>",
109 "load the manifest at PATH [default='build.ninja']");
110 fprintf(stderr, " %-*s %s\n", optionWidth, "-k <N>",
111 "keep building until N commands fail [default=1]");
112 fprintf(stderr, " %-*s %s\n", optionWidth, "-t, --tool <TOOL>",
113 "run a ninja tool. use 'list' to list available tools.");
114 fprintf(stderr, " %-*s %s\n", optionWidth, "-j, --jobs <JOBS>",
115 "number of jobs to build in parallel [default=cpu dependent]");
116 fprintf(stderr, " %-*s %s\n", optionWidth, "--no-regenerate",
117 "disable manifest auto-regeneration");
118 fprintf(stderr, " %-*s %s\n", optionWidth, "--dump-graph <PATH>",
119 "dump build graph to PATH in Graphviz DOT format");
120 fprintf(stderr, " %-*s %s\n", optionWidth, "--profile <PATH>",
121 "write a build profile trace event file to PATH");
122 fprintf(stderr, " %-*s %s\n", optionWidth, "--strict",
123 "use strict mode (no bug compatibility)");
124 fprintf(stderr, " %-*s %s\n", optionWidth, "--trace <PATH>",
125 "trace build engine operation to PATH");
126 fprintf(stderr, " %-*s %s\n", optionWidth, "--quiet",
127 "don't show information on executed commands");
128 fprintf(stderr, " %-*s %s\n", optionWidth, "-v, --verbose",
129 "show full invocation for executed commands");
130 fprintf(stderr, " %-*s %s\n", optionWidth, "-l <N>",
131 "start jobs only when load average is below N [not implemented]");
132 fprintf(stderr, " %-*s %s\n", optionWidth, "-d <TOOL>",
133 "enable debugging tool TOOL. 'list' for available [not implemented]");
134 ::exit(exitCode);
135 }
136
137 namespace {
138
139 /// The build execution queue manages the actual parallel execution of jobs
140 /// which have been enqueued as a result of command orocessing.
141 ///
142 /// This queue guarantees serial execution when only configured with a single
143 /// lane.
144 struct BuildExecutionQueue {
145 /// The number of lanes the queue was configured with.
146 unsigned numLanes;
147
148 /// A thread for each lane.
149 std::vector<std::unique_ptr<std::thread>> lanes;
150
151 /// The ready queue of jobs to execute.
152 std::deque<std::function<void(unsigned)>> readyJobs;
153 std::mutex readyJobsMutex;
154 std::condition_variable readyJobsCondition;
155
156 /// Use LIFO execution.
157 bool useLIFO;
158
159 public:
BuildExecutionQueue__anond6d98e720111::BuildExecutionQueue160 BuildExecutionQueue(unsigned numLanes, bool useLIFO)
161 : numLanes(numLanes), useLIFO(useLIFO)
162 {
163 for (unsigned i = 0; i != numLanes; ++i) {
164 lanes.push_back(std::unique_ptr<std::thread>(
165 new std::thread(
166 &BuildExecutionQueue::executeLane, this, i)));
167 }
168 }
~BuildExecutionQueue__anond6d98e720111::BuildExecutionQueue169 ~BuildExecutionQueue() {
170 // Shut down the lanes.
171 for (unsigned i = 0; i != numLanes; ++i) {
172 addJob({});
173 }
174 for (unsigned i = 0; i != numLanes; ++i) {
175 lanes[i]->join();
176 }
177 }
178
executeLane__anond6d98e720111::BuildExecutionQueue179 void executeLane(unsigned laneNumber) {
180 // oust execute items from the queue until shutdown.
181 while (true) {
182 // Take a job from the ready queue.
183 std::function<void(unsigned)> job;
184 {
185 std::unique_lock<std::mutex> lock(readyJobsMutex);
186
187 // While the queue is empty, wait for an item.
188 while (readyJobs.empty()) {
189 readyJobsCondition.wait(lock);
190 }
191
192 // Take an item according to the chosen policy.
193 if (useLIFO) {
194 job = readyJobs.back();
195 readyJobs.pop_back();
196 } else {
197 job = readyJobs.front();
198 readyJobs.pop_front();
199 }
200 }
201
202 // If we got an empty job, the queue is shutting down.
203 if (!job)
204 break;
205
206 // Process the job.
207 job(laneNumber);
208 }
209 }
210
addJob__anond6d98e720111::BuildExecutionQueue211 void addJob(std::function<void(unsigned)> job) {
212 std::lock_guard<std::mutex> guard(readyJobsMutex);
213 readyJobs.push_back(job);
214 readyJobsCondition.notify_one();
215 }
216 };
217
218 /// Result value that is computed by the rules for input and command files.
219 class BuildValue {
220 private:
221 // Copying and move assignment are disabled.
222 BuildValue(const BuildValue&) LLBUILD_DELETED_FUNCTION;
223 void operator=(const BuildValue&) LLBUILD_DELETED_FUNCTION;
224 BuildValue &operator=(BuildValue&& rhs) LLBUILD_DELETED_FUNCTION;
225
226 public:
227 static const int currentSchemaVersion = 3;
228
229 private:
230 enum class BuildValueKind : uint32_t {
231 /// A value produced by a existing input file.
232 ExistingInput = 0,
233
234 /// A value produced by a missing input file.
235 MissingInput,
236
237 /// A value produced by a successful command.
238 SuccessfulCommand,
239
240 /// A value produced by a failing command.
241 FailedCommand,
242
243 /// A value produced by a command that was not run.
244 SkippedCommand,
245 };
246
247 /// The kind of value.
248 BuildValueKind kind;
249
250 /// The number of attached output infos.
251 const uint32_t numOutputInfos = 0;
252
253 union {
254 /// The file info for the rule output, for existing inputs and successful
255 /// commands with a single output.
256 FileInfo asOutputInfo;
257
258 /// The file info for successful commands with multiple outputs.
259 FileInfo* asOutputInfos;
260 } valueData;
261
262 /// The command hash, for successful commands.
263 uint64_t commandHash;
264
265 private:
BuildValue()266 BuildValue() {}
BuildValue(BuildValueKind kind)267 BuildValue(BuildValueKind kind)
268 : kind(kind), numOutputInfos(0), commandHash(0)
269 {
270 }
BuildValue(BuildValueKind kind,const FileInfo & outputInfo,uint64_t commandHash=0)271 BuildValue(BuildValueKind kind, const FileInfo& outputInfo,
272 uint64_t commandHash = 0)
273 : kind(kind), numOutputInfos(1), commandHash(commandHash)
274 {
275 valueData.asOutputInfo = outputInfo;
276 }
BuildValue(BuildValueKind kind,const FileInfo * outputInfos,uint32_t numOutputInfos,uint64_t commandHash=0)277 BuildValue(BuildValueKind kind, const FileInfo* outputInfos,
278 uint32_t numOutputInfos, uint64_t commandHash = 0)
279 : kind(kind), numOutputInfos(numOutputInfos), commandHash(commandHash)
280 {
281 valueData.asOutputInfos = new FileInfo[numOutputInfos];
282 for (uint32_t i = 0; i != numOutputInfos; ++i) {
283 valueData.asOutputInfos[i] = outputInfos[i];
284 }
285 }
286
287 public:
288 // Build values can only be moved via construction, not copied.
BuildValue(BuildValue && rhs)289 BuildValue(BuildValue&& rhs) {
290 memcpy(this, &rhs, sizeof(rhs));
291 memset(&rhs, 0, sizeof(rhs));
292 }
~BuildValue()293 ~BuildValue() {
294 if (hasMultipleOutputs()) {
295 delete[] valueData.asOutputInfos;
296 }
297 }
298
makeExistingInput(const FileInfo & outputInfo)299 static BuildValue makeExistingInput(const FileInfo& outputInfo) {
300 return BuildValue(BuildValueKind::ExistingInput, outputInfo);
301 }
makeMissingInput()302 static BuildValue makeMissingInput() {
303 return BuildValue(BuildValueKind::MissingInput);
304 }
makeSuccessfulCommand(const FileInfo & outputInfo,uint64_t commandHash)305 static BuildValue makeSuccessfulCommand(const FileInfo& outputInfo,
306 uint64_t commandHash) {
307 return BuildValue(BuildValueKind::SuccessfulCommand, outputInfo,
308 commandHash);
309 }
makeSuccessfulCommand(const FileInfo * outputInfos,uint32_t numOutputInfos,uint64_t commandHash)310 static BuildValue makeSuccessfulCommand(const FileInfo* outputInfos,
311 uint32_t numOutputInfos,
312 uint64_t commandHash) {
313 // This ctor function should only be used for multiple outputs.
314 assert(numOutputInfos > 1);
315 return BuildValue(BuildValueKind::SuccessfulCommand, outputInfos,
316 numOutputInfos, commandHash);
317 }
makeFailedCommand()318 static BuildValue makeFailedCommand() {
319 return BuildValue(BuildValueKind::FailedCommand);
320 }
makeSkippedCommand()321 static BuildValue makeSkippedCommand() {
322 return BuildValue(BuildValueKind::SkippedCommand);
323 }
324
isExistingInput() const325 bool isExistingInput() const { return kind == BuildValueKind::ExistingInput; }
isMissingInput() const326 bool isMissingInput() const { return kind == BuildValueKind::MissingInput; }
isSuccessfulCommand() const327 bool isSuccessfulCommand() const {
328 return kind == BuildValueKind::SuccessfulCommand;
329 }
isFailedCommand() const330 bool isFailedCommand() const { return kind == BuildValueKind::FailedCommand; }
isSkippedCommand() const331 bool isSkippedCommand() const {
332 return kind == BuildValueKind::SkippedCommand;
333 }
334
hasMultipleOutputs() const335 bool hasMultipleOutputs() const {
336 return numOutputInfos > 1;
337 }
338
getNumOutputs() const339 unsigned getNumOutputs() const {
340 assert((isExistingInput() || isSuccessfulCommand()) &&
341 "invalid call for value kind");
342 return numOutputInfos;
343 }
344
getOutputInfo() const345 const FileInfo& getOutputInfo() const {
346 assert((isExistingInput() || isSuccessfulCommand()) &&
347 "invalid call for value kind");
348 assert(!hasMultipleOutputs() &&
349 "invalid call on result with multiple outputs");
350 return valueData.asOutputInfo;
351 }
352
getNthOutputInfo(unsigned n) const353 const FileInfo& getNthOutputInfo(unsigned n) const {
354 assert((isExistingInput() || isSuccessfulCommand()) &&
355 "invalid call for value kind");
356 assert(n < getNumOutputs());
357 if (hasMultipleOutputs()) {
358 return valueData.asOutputInfos[n];
359 } else {
360 assert(n == 0);
361 return valueData.asOutputInfo;
362 }
363 }
364
getCommandHash() const365 uint64_t getCommandHash() const {
366 assert(isSuccessfulCommand() && "invalid call for value kind");
367 return commandHash;
368 }
369
fromValue(const core::ValueType & value)370 static BuildValue fromValue(const core::ValueType& value) {
371 BuildValue result;
372 assert(value.size() >= sizeof(result));
373 memcpy(&result, value.data(), sizeof(result));
374
375 // If this result has multiple output values, deserialize them properly.
376 if (result.numOutputInfos > 1) {
377 assert(value.size() == (sizeof(result) +
378 result.numOutputInfos * sizeof(FileInfo)));
379 result.valueData.asOutputInfos = new FileInfo[result.numOutputInfos];
380 memcpy(result.valueData.asOutputInfos,
381 value.data() + sizeof(result),
382 result.numOutputInfos * sizeof(FileInfo));
383 } else {
384 assert(value.size() == sizeof(result));
385 }
386
387 return result;
388 }
389
toValue()390 core::ValueType toValue() {
391 if (numOutputInfos > 1) {
392 // FIXME: This could be packed one entry tighter.
393 std::vector<uint8_t> result(sizeof(*this) +
394 numOutputInfos * sizeof(FileInfo));
395 memcpy(result.data(), this, sizeof(*this));
396 memcpy(result.data() + sizeof(*this), valueData.asOutputInfos,
397 numOutputInfos * sizeof(FileInfo));
398 return result;
399 } else {
400 std::vector<uint8_t> result(sizeof(*this));
401 memcpy(result.data(), this, sizeof(*this));
402 return result;
403 }
404 }
405 };
406
407 struct NinjaBuildEngineDelegate : public core::BuildEngineDelegate {
408 class BuildContext* context = nullptr;
409
410 virtual core::Rule lookupRule(const core::KeyType& key) override;
411
412 virtual void cycleDetected(const std::vector<core::Rule*>& items) override;
413
414 virtual void error(const Twine& message) override;
415 };
416
417 /// Wrapper for information used during a single build.
418 class BuildContext {
419 public:
420 /// The build engine delegate.
421 NinjaBuildEngineDelegate delegate;
422
423 /// The engine in use.
424 core::BuildEngine engine;
425
426 /// The Ninja manifest we are operating on.
427 std::unique_ptr<ninja::Manifest> manifest;
428
429 /// Whether commands should print status information.
430 bool quiet = false;
431 /// Whether the build is being "simulated", in which case commands won't be
432 /// run and inputs will be assumed to exist.
433 bool simulate = false;
434 /// Whether to use strict mode.
435 bool strict = false;
436 /// Whether output should use verbose mode.
437 bool verbose = false;
438 /// The number of failed commands to tolerate, or 0 if unlimited
439 unsigned numFailedCommandsToTolerate = 1;
440
441 /// The build profile output file.
442 FILE *profileFP = nullptr;
443
444 /// Whether the build has been cancelled or not.
445 std::atomic<bool> isCancelled{false};
446
447 /// Whether the build was cancelled by SIGINT.
448 std::atomic<bool> wasCancelledBySigint{false};
449
450 /// The number of generated errors.
451 std::atomic<unsigned> numErrors{0};
452 /// The number of commands executed during the build
453 unsigned numBuiltCommands{0};
454 /// The number of output commands written, for numbering purposes.
455 unsigned numOutputDescriptions{0};
456 /// The number of failed commands.
457 std::atomic<unsigned> numFailedCommands{0};
458
459 /// @name Status Reporting Command Counts
460 /// @{
461
462 /// The number of commands being scanned.
463 std::atomic<unsigned> numCommandsScanning{0};
464 /// The number of commands that were up-to-date.
465 std::atomic<unsigned> numCommandsUpToDate{0};
466 /// The number of commands that have been completed.
467 std::atomic<unsigned> numCommandsCompleted{0};
468 /// The number of commands that were updated (started, but didn't actually run
469 /// the command).
470 std::atomic<unsigned> numCommandsUpdated{0};
471
472 /// @}
473
474 /// The status output object.
475 CommandLineStatusOutput statusOutput;
476
477 /// The serial queue we used to order output consistently.
478 SerialQueue outputQueue;
479
480 /// The limited queue we use to execute parallel jobs.
481 std::unique_ptr<BuildExecutionQueue> jobQueue;
482
483 /// The previous SIGINT handler.
484 struct sigaction previousSigintHandler;
485
486 /// The set of spawned processes to cancel when interrupted.
487 std::unordered_set<pid_t> spawnedProcesses;
488 std::mutex spawnedProcessesMutex;
489
490 /// Low-level flag for when a SIGINT has been received.
491 static std::atomic<bool> wasInterrupted;
492
493 /// Pipe used to allow detection of signals.
494 static int signalWatchingPipe[2];
495
sigintHandler(int)496 static void sigintHandler(int) {
497 // Set the atomic interrupt flag.
498 BuildContext::wasInterrupted = true;
499
500 // Write to wake up the signal monitoring thread.
501 char byte{};
502 sys::write(signalWatchingPipe[1], &byte, 1);
503 }
504
sendSignalToProcesses(int signal)505 void sendSignalToProcesses(int signal) {
506 std::unique_lock<std::mutex> lock(spawnedProcessesMutex);
507
508 for (pid_t pid: spawnedProcesses) {
509 // We are killing the whole process group here, this depends on us
510 // spawning each process in its own group earlier.
511 ::kill(-pid, signal);
512 }
513 }
514
515 /// Cancel the build in response to an interrupt event.
cancelBuildOnInterrupt()516 void cancelBuildOnInterrupt() {
517 sendSignalToProcesses(SIGINT);
518
519 emitNote("cancelling build.");
520 isCancelled = true;
521 wasCancelledBySigint = true;
522
523 // FIXME: In our model, we still wait for everything to terminate, which
524 // means a process that refuses to respond to SIGINT will cause us to just
525 // hang here. We should probably detect and report that and be willing to do
526 // a hard kill at some point (for example, on the second interrupt).
527 }
528
529 /// Check if an interrupt has occurred.
checkForInterrupt()530 void checkForInterrupt() {
531 // Save and clear the interrupt flag, atomically.
532 bool wasInterrupted = BuildContext::wasInterrupted.exchange(false);
533
534 // Process the interrupt flag, if present.
535 if (wasInterrupted) {
536 // Otherwise, process the interrupt.
537 cancelBuildOnInterrupt();
538 }
539 }
540
541 /// Thread function to wait for indications that signals have arrived and to
542 /// process them.
signalWaitThread()543 void signalWaitThread() {
544 // Wait for signal arrival indications.
545 while (true) {
546 char byte;
547 int res = sys::read(signalWatchingPipe[0], &byte, 1);
548
549 // If nothing was read, the pipe has been closed and we should shut down.
550 if (res == 0)
551 break;
552
553 // Otherwise, check if we were awoke because of an interrupt.
554 checkForInterrupt();
555 }
556
557 // Shut down the pipe.
558 sys::close(signalWatchingPipe[0]);
559 signalWatchingPipe[0] = -1;
560 }
561
562 public:
BuildContext()563 BuildContext()
564 : engine(delegate),
565 isCancelled(false)
566 {
567 // Open the status output.
568 std::string error;
569 if (!statusOutput.open(&error)) {
570 fprintf(stderr, "error: %s: unable to open output: %s\n",
571 getProgramName(), error.c_str());
572 exit(1);
573 }
574
575 // Register the context with the delegate.
576 delegate.context = this;
577
578 // Register an interrupt handler.
579 struct sigaction action{};
580 action.sa_handler = &BuildContext::sigintHandler;
581 sigaction(SIGINT, &action, &previousSigintHandler);
582
583 // Create a pipe and thread to watch for signals.
584 assert(BuildContext::signalWatchingPipe[0] == -1 &&
585 BuildContext::signalWatchingPipe[1] == -1);
586 if (basic::sys::pipe(BuildContext::signalWatchingPipe) < 0) {
587 perror("pipe");
588 }
589 new std::thread(&BuildContext::signalWaitThread, this);
590 }
591
~BuildContext()592 ~BuildContext() {
593 // Ensure the output queue is done.
594 outputQueue.sync([] {});
595
596 // Restore any previous SIGINT handler.
597 sigaction(SIGINT, &previousSigintHandler, NULL);
598
599 // Close the status output.
600 std::string error;
601 statusOutput.close(&error);
602
603 // Close the signal watching pipe.
604 sys::close(BuildContext::signalWatchingPipe[1]);
605 signalWatchingPipe[1] = -1;
606 }
607
608 /// @name Diagnostics Output
609 /// @{
610
611 /// Emit a status line, which can be updated.
612 ///
613 /// This method should only be called from the outputQueue.
emitStatus(const char * fmt,...)614 void emitStatus(const char* fmt, ...) {
615 va_list ap;
616 va_start(ap, fmt);
617 std::string message = getFormattedString(fmt, ap);
618 va_end(ap);
619
620 if (verbose) {
621 statusOutput.writeText(message + "\n");
622 } else {
623 statusOutput.setOrWriteLine(message);
624 }
625 }
626
627 /// Emit a diagnostic to the error stream.
emitDiagnostic(std::string kind,std::string message)628 void emitDiagnostic(std::string kind, std::string message) {
629 outputQueue.async([this, kind=std::move(kind), message=std::move(message)] {
630 statusOutput.finishLine();
631 fprintf(stderr, "%s: %s: %s\n", getProgramName(), kind.c_str(),
632 message.c_str());
633 });
634 }
635
636 /// Emit a diagnostic followed by a block of text, ensuring the text
637 /// immediately follows the diagnostic.
emitDiagnosticAndText(std::string kind,std::string message,std::string text)638 void emitDiagnosticAndText(std::string kind, std::string message,
639 std::string text) {
640 outputQueue.async([this, kind=std::move(kind), message=std::move(message),
641 text=std::move(text)] {
642 statusOutput.finishLine();
643 fprintf(stderr, "%s: %s: %s\n", getProgramName(), kind.c_str(),
644 message.c_str());
645 fflush(stderr);
646 fwrite(text.data(), text.size(), 1, stdout);
647 fflush(stdout);
648 });
649 }
650
651 /// Emit a block of text to the output.
emitText(std::string text)652 void emitText(std::string text) {
653 outputQueue.async([this, text=std::move(text)] {
654 statusOutput.finishLine();
655 fwrite(text.data(), text.size(), 1, stdout);
656 fflush(stdout);
657 });
658 }
659
emitText(const char * fmt,...)660 void emitText(const char* fmt, ...) {
661 va_list ap;
662 va_start(ap, fmt);
663 emitText(getFormattedString(fmt, ap));
664 va_end(ap);
665 }
666
emitError(std::string && message)667 void emitError(std::string&& message) {
668 emitDiagnostic("error", std::move(message));
669 ++numErrors;
670 }
671
emitErrorAndText(std::string && message,std::string && text)672 void emitErrorAndText(std::string&& message, std::string&& text) {
673 emitDiagnosticAndText("error", std::move(message), text);
674 ++numErrors;
675 }
676
emitError(const char * fmt,...)677 void emitError(const char* fmt, ...) {
678 va_list ap;
679 va_start(ap, fmt);
680 emitError(getFormattedString(fmt, ap));
681 va_end(ap);
682 }
683
emitNote(std::string && message)684 void emitNote(std::string&& message) {
685 emitDiagnostic("note", std::move(message));
686 }
687
emitNote(const char * fmt,...)688 void emitNote(const char* fmt, ...) {
689 va_list ap;
690 va_start(ap, fmt);
691 emitNote(getFormattedString(fmt, ap));
692 va_end(ap);
693 }
694
695 /// @}
696
reportMissingInput(const ninja::Node * node)697 void reportMissingInput(const ninja::Node* node) {
698 // We simply report the missing input here, the build will be cancelled when
699 // a rule sees it missing.
700 emitError("missing input '%s' and no rule to build it",
701 node->getPath().c_str());
702 }
703
incrementFailedCommands()704 void incrementFailedCommands() {
705 // Update our count of the number of failed commands.
706 unsigned numFailedCommands = ++this->numFailedCommands;
707
708 // Cancel the build, if the number of command failures exceeds the
709 // number to continue past.
710 if (numFailedCommandsToTolerate != 0 &&
711 numFailedCommands == numFailedCommandsToTolerate) {
712 emitError("stopping build due to command failures");
713 isCancelled = true;
714 }
715 }
716 };
717
718 std::atomic<bool> BuildContext::wasInterrupted{false};
719 int BuildContext::signalWatchingPipe[2]{-1, -1};
720
721 class BuildManifestActions : public ninja::ManifestLoaderActions {
722 BuildContext& context;
723 ninja::ManifestLoader* loader = 0;
724 unsigned numErrors = 0;
725 unsigned maxErrors = 20;
726
727 private:
initialize(ninja::ManifestLoader * loader)728 virtual void initialize(ninja::ManifestLoader* loader) override {
729 this->loader = loader;
730 }
731
error(std::string filename,std::string message,const ninja::Token & at)732 virtual void error(std::string filename, std::string message,
733 const ninja::Token& at) override {
734 if (numErrors++ >= maxErrors)
735 return;
736
737 util::emitError(filename, message, at, loader->getCurrentParser());
738 }
739
readFileContents(const std::string & fromFilename,const std::string & filename,const ninja::Token * forToken,std::unique_ptr<char[]> * data_out,uint64_t * length_out)740 virtual bool readFileContents(const std::string& fromFilename,
741 const std::string& filename,
742 const ninja::Token* forToken,
743 std::unique_ptr<char[]>* data_out,
744 uint64_t* length_out) override {
745 // Load the file contents and return if successful.
746 std::string error;
747 if (util::readFileContents(filename, data_out, length_out, &error))
748 return true;
749
750 // Otherwise, emit the error.
751 ++numErrors;
752 if (forToken) {
753 util::emitError(fromFilename, error, *forToken,
754 loader->getCurrentParser());
755 } else {
756 context.emitError(std::move(error));
757 }
758
759 return false;
760 };
761
762 public:
BuildManifestActions(BuildContext & context)763 BuildManifestActions(BuildContext& context) : context(context) {}
764
getNumErrors() const765 unsigned getNumErrors() const { return numErrors; }
766 };
767
768 static core::Task*
buildCommand(BuildContext & context,ninja::Command * command)769 buildCommand(BuildContext& context, ninja::Command* command) {
770 struct NinjaCommandTask : core::Task {
771 BuildContext& context;
772 ninja::Command* command;
773
774 /// If true, the command should be skipped (because of an error in an
775 /// input).
776 bool shouldSkip = false;
777
778 /// If true, the command had a missing input (this implies ShouldSkip is
779 /// true).
780 bool hasMissingInput = false;
781
782 /// If true, the command can be updated if the output is newer than all of
783 /// the inputs.
784 bool canUpdateIfNewer = true;
785
786 /// Information on the prior command result, if present.
787 bool hasPriorResult = false;
788 uint64_t priorCommandHash;
789
790 /// The timestamp of the most recently rebuilt input.
791 FileTimestamp newestModTime{ 0, 0 };
792
793 NinjaCommandTask(BuildContext& context, ninja::Command* command)
794 : context(context), command(command) {
795 // If this command uses discovered dependencies, we can never skip it (we
796 // don't yet have a way to account for the discovered dependencies, or
797 // preserve them if skipped).
798 //
799 // FIXME: We should support update-if-newer for commands with deps.
800 if (command->getDepsStyle() != ninja::Command::DepsStyleKind::None)
801 canUpdateIfNewer = false;
802 }
803
804 virtual void provideValue(core::BuildEngine& engine, uintptr_t inputID,
805 const core::ValueType& valueData) override {
806 // Process the input value to see if we should skip this command.
807 BuildValue value = BuildValue::fromValue(valueData);
808
809 // All direct inputs to NinjaCommandTask objects should be singleton
810 // values.
811 assert(!value.hasMultipleOutputs());
812
813 // If the value is not an existing input or a successful command, then we
814 // shouldn't run this command.
815 if (!value.isExistingInput() && !value.isSuccessfulCommand()) {
816 shouldSkip = true;
817 if (value.isMissingInput()) {
818 hasMissingInput = true;
819
820 context.reportMissingInput(command->getInputs()[inputID]);
821 }
822 } else {
823 // Otherwise, track the information used to determine if we can just
824 // update the command instead of running it.
825 const FileInfo& outputInfo = value.getOutputInfo();
826
827 // If there is a missing input file (from a successful command), we
828 // always need to run the command.
829 if (outputInfo.isMissing()) {
830 canUpdateIfNewer = false;
831 } else {
832 // Otherwise, keep track of the newest input.
833 if (outputInfo.modTime > newestModTime) {
834 newestModTime = outputInfo.modTime;
835 }
836 }
837 }
838 }
839
840 bool isImmediatelyCyclicInput(const ninja::Node* node) const {
841 for (const auto* output: command->getOutputs())
842 if (node == output)
843 return true;
844 return false;
845 }
846
847 void completeTask(BuildValue&& result, bool forceChange=false) {
848 context.engine.taskIsComplete(this, result.toValue(), forceChange);
849 }
850
851 virtual void start(core::BuildEngine& engine) override {
852 // If this is a phony rule, ignore any immediately cyclic dependencies in
853 // non-strict mode, which are generated frequently by CMake, but can be
854 // ignored by Ninja. See https://github.com/martine/ninja/issues/935.
855 //
856 // FIXME: Find a way to harden this more, or see if we can just get CMake
857 // to fix it.
858 bool isPhony = command->getRule() == context.manifest->getPhonyRule();
859
860 // Request all of the explicit and implicit inputs (the only difference
861 // between them is that implicit inputs do not appear in ${in} during
862 // variable expansion, but that has already been performed).
863 unsigned id = 0;
864 for (auto it = command->explicitInputs_begin(),
865 ie = command->explicitInputs_end(); it != ie; ++it, ++id) {
866 if (!context.strict && isPhony && isImmediatelyCyclicInput(*it))
867 continue;
868
869 engine.taskNeedsInput(this, (*it)->getPath(), id);
870 }
871 for (auto it = command->implicitInputs_begin(),
872 ie = command->implicitInputs_end(); it != ie; ++it, ++id) {
873 if (!context.strict && isPhony && isImmediatelyCyclicInput(*it))
874 continue;
875
876 engine.taskNeedsInput(this, (*it)->getPath(), id);
877 }
878
879 // Request all of the order-only inputs.
880 for (auto it = command->orderOnlyInputs_begin(),
881 ie = command->orderOnlyInputs_end(); it != ie; ++it) {
882 if (!context.strict && isPhony && isImmediatelyCyclicInput(*it))
883 continue;
884
885 engine.taskMustFollow(this, (*it)->getPath());
886 }
887 }
888
889 virtual void providePriorValue(core::BuildEngine& engine,
890 const core::ValueType& valueData) override {
891 BuildValue value = BuildValue::fromValue(valueData);
892
893 if (value.isSuccessfulCommand()) {
894 hasPriorResult = true;
895 priorCommandHash = value.getCommandHash();
896 }
897 }
898
899 /// Compute the output result for the command.
900 BuildValue computeCommandResult(uint64_t commandHash) const {
901 unsigned numOutputs = command->getOutputs().size();
902 if (numOutputs == 1) {
903 return BuildValue::makeSuccessfulCommand(
904 FileInfo::getInfoForPath(
905 command->getOutputs()[0]->getPath()),
906 commandHash);
907 } else {
908 std::vector<FileInfo> outputInfos(numOutputs);
909 for (unsigned i = 0; i != numOutputs; ++i) {
910 outputInfos[i] = FileInfo::getInfoForPath(
911 command->getOutputs()[i]->getPath());
912 }
913 return BuildValue::makeSuccessfulCommand(outputInfos.data(), numOutputs,
914 commandHash);
915 }
916 }
917
918 /// Check if it is legal to only update the result (versus rerunning)
919 /// because the outputs are newer than all of the inputs.
920 bool canUpdateIfNewerWithResult(const BuildValue& result) {
921 assert(result.isSuccessfulCommand());
922
923 // Check each output.
924 for (unsigned i = 0, e = result.getNumOutputs(); i != e; ++i) {
925 const FileInfo& outputInfo = result.getNthOutputInfo(i);
926
927 // If the output is missing, we need to rebuild.
928 if (outputInfo.isMissing())
929 return false;
930
931 // Check if the output is actually newer than the most recent input.
932 //
933 // In strict mode, we use a strict "newer-than" check here, to guarantee
934 // correctness in the face of equivalent timestamps. This is
935 // particularly important on OS X, which has a low resolution mtime.
936 //
937 // However, in non-strict mode, we need to be compatible with Ninja
938 // here, because there are some very important uses cases where this
939 // behavior is relied on. One major example is CMake's initial
940 // configuration checks using Ninja -- if this is not in place, those
941 // rules will try and rerun the generator of the "TRY_COMPILE" steps,
942 // and will enter an infinite reconfiguration loop. See also:
943 //
944 // See: http://www.cmake.org/Bug/view.php?id=15456
945 if (context.strict) {
946 if (outputInfo.modTime <= newestModTime)
947 return false;
948 } else {
949 if (outputInfo.modTime < newestModTime)
950 return false;
951 }
952 }
953
954 return true;
955 }
956
957 virtual void inputsAvailable(core::BuildEngine& engine) override {
958 // If the build is cancelled, skip everything.
959 if (context.isCancelled) {
960 return completeTask(BuildValue::makeSkippedCommand());
961 }
962
963 // Ignore phony commands.
964 //
965 // FIXME: Is it right to bring this up-to-date when one of the inputs
966 // indicated a failure? It probably doesn't matter.
967 uint64_t commandHash = basic::hashString(command->getCommandString());
968 if (command->getRule() == context.manifest->getPhonyRule()) {
969 // Get the result.
970 BuildValue result = computeCommandResult(commandHash);
971
972 // If any output is missing, then we always want to force the change to
973 // propagate.
974 bool forceChange = false;
975 for (unsigned i = 0, e = result.getNumOutputs(); i != e; ++i) {
976 if (result.getNthOutputInfo(i).isMissing()) {
977 forceChange = true;
978 break;
979 }
980 }
981
982 return completeTask(std::move(result), forceChange);
983 }
984
985 // If it is legal to simply update the command, then if the command output
986 // exists and is newer than all of the inputs, don't actually run the
987 // command (just bring it up-to-date).
988 if (canUpdateIfNewer) {
989 // If this isn't a generator command and its command hash differs, we
990 // can't update it.
991 if (!command->hasGeneratorFlag() &&
992 (!hasPriorResult || priorCommandHash != commandHash))
993 canUpdateIfNewer = false;
994
995 if (canUpdateIfNewer) {
996 BuildValue result = computeCommandResult(commandHash);
997
998 if (canUpdateIfNewerWithResult(result)) {
999 // Update the count of the number of commands which have been
1000 // updated without being rerun.
1001 ++context.numCommandsUpdated;
1002
1003 return completeTask(std::move(result));
1004 }
1005 }
1006 }
1007
1008 // Otherwise, actually run the command.
1009
1010 ++context.numBuiltCommands;
1011
1012 // If we are simulating the build, just print the description and
1013 // complete.
1014 if (context.simulate) {
1015 if (!context.quiet)
1016 writeDescription(context, command);
1017 return completeTask(BuildValue::makeSkippedCommand());
1018 }
1019
1020 // If not simulating, but this command should be skipped, then do nothing.
1021 if (shouldSkip) {
1022 // If this command had a failed input, treat it as having failed.
1023 if (hasMissingInput) {
1024 context.emitError("cannot build '%s' due to missing input",
1025 command->getOutputs()[0]->getPath().c_str());
1026
1027 // Update the count of failed commands.
1028 context.incrementFailedCommands();
1029 }
1030
1031 return completeTask(BuildValue::makeSkippedCommand());
1032 }
1033 assert(!hasMissingInput);
1034
1035 // Otherwise, enqueue the job to run later.
1036 context.jobQueue->addJob([&] (unsigned bucket) {
1037 // Suppress static analyzer false positive on generalized lambda capture
1038 // (rdar://problem/22165130).
1039 #ifndef __clang_analyzer__
1040 // Take care to not rely on the ``this`` object, which may disappear
1041 // before the queue executes this block.
1042 BuildContext& localContext(context);
1043 ninja::Command* localCommand(command);
1044
1045 if (localContext.profileFP) {
1046 localContext.outputQueue.sync(
1047 [&localContext=localContext, localCommand=localCommand, bucket] {
1048 uint64_t startTime = getTimeInMicroseconds();
1049 fprintf(localContext.profileFP,
1050 ("{ \"name\": \"%s\", \"ph\": \"B\", \"pid\": 0, "
1051 "\"tid\": %d, \"ts\": %llu},\n"),
1052 localCommand->getEffectiveDescription().c_str(), bucket,
1053 static_cast<unsigned long long>(startTime));
1054 });
1055 }
1056
1057 executeCommand();
1058
1059 if (localContext.profileFP) {
1060 localContext.outputQueue.sync(
1061 [&localContext=localContext, localCommand=localCommand, bucket] {
1062 uint64_t endTime = getTimeInMicroseconds();
1063 fprintf(localContext.profileFP,
1064 ("{ \"name\": \"%s\", \"ph\": \"E\", \"pid\": 0, "
1065 "\"tid\": %d, \"ts\": %llu},\n"),
1066 localCommand->getEffectiveDescription().c_str(), bucket,
1067 static_cast<unsigned long long>(endTime));
1068 });
1069 }
1070 #endif
1071 });
1072 }
1073
1074 static unsigned getNumPossibleMaxCommands(BuildContext& context) {
1075 // Compute the "possible" number of maximum commands that will be
1076 // run. This is only the "possible" max because we can start running
1077 // commands before dependency scanning is complete -- we include the
1078 // number of commands that are being scanned so that this number will
1079 // always be greater than the number of commands that have been executed
1080 // until the very last command is run.
1081 int totalPossibleMaxCommands =
1082 context.numCommandsCompleted + context.numCommandsScanning;
1083
1084 // Compute the number of max commands to show, subtracting out all the
1085 // commands that we avoided running.
1086 int possibleMaxCommands = totalPossibleMaxCommands -
1087 (context.numCommandsUpToDate + context.numCommandsUpdated);
1088
1089 return possibleMaxCommands;
1090 }
1091
1092 static void writeDescription(BuildContext& context,
1093 ninja::Command* command) {
1094 const std::string& description =
1095 context.verbose ? command->getCommandString() :
1096 command->getEffectiveDescription();
1097 context.emitStatus(
1098 "[%d/%d] %s", ++context.numOutputDescriptions,
1099 getNumPossibleMaxCommands(context), description.c_str());
1100
1101 // Whenever we write a description for a console job, make sure to finish
1102 // the output under the expectation that the console job might write to
1103 // the output. We don't make any attempt to lock this in case the console
1104 // job can run concurrently with anything else.
1105 if (command->getExecutionPool() == context.manifest->getConsolePool())
1106 context.statusOutput.finishLine();
1107 }
1108
1109 void executeCommand() {
1110 // If the build is cancelled, skip the job.
1111 if (context.isCancelled) {
1112 return completeTask(BuildValue::makeSkippedCommand());
1113 }
1114
1115 // Write the description on the output queue, taking care to not rely on
1116 // the ``this`` object, which may disappear before the queue executes this
1117 // block.
1118 if (!context.quiet) {
1119 // Suppress static analyzer false positive on generalized lambda capture
1120 // (rdar://problem/22165130).
1121 #ifndef __clang_analyzer__
1122 // If this is a console job, do the write synchronously to ensure it
1123 // appears before the task might start.
1124 if (command->getExecutionPool() == context.manifest->getConsolePool()) {
1125 context.outputQueue.sync([&context=context, command=command] {
1126 writeDescription(context, command);
1127 });
1128 } else {
1129 context.outputQueue.async([&context=context, command=command] {
1130 writeDescription(context, command);
1131 });
1132 }
1133 #endif
1134 }
1135
1136 // Actually run the command.
1137 if (!spawnAndWaitForCommand()) {
1138 // If the command failed, complete the task with the failed result and
1139 // always propagate.
1140 return completeTask(BuildValue::makeFailedCommand(),
1141 /*ForceChange=*/true);
1142 }
1143
1144 // Otherwise, the command succeeded so process the dependencies.
1145 if (!processDiscoveredDependencies()) {
1146 context.incrementFailedCommands();
1147 return completeTask(BuildValue::makeFailedCommand(),
1148 /*ForceChange=*/true);
1149 }
1150
1151 // Complete the task with a successful value.
1152 //
1153 // We always restat the output, but we honor Ninja's restat flag by
1154 // forcing downstream propagation if it isn't set.
1155 uint64_t commandHash = basic::hashString(command->getCommandString());
1156 BuildValue result = computeCommandResult(commandHash);
1157 return completeTask(std::move(result),
1158 /*ForceChange=*/!command->hasRestatFlag());
1159 }
1160
1161 /// Execute the command process and wait for it to complete.
1162 ///
1163 /// \returns True if the command succeeded.
1164 bool spawnAndWaitForCommand() const {
1165 bool isConsole = command->getExecutionPool() ==
1166 context.manifest->getConsolePool();
1167
1168 // Initialize the spawn attributes.
1169 posix_spawnattr_t attributes;
1170 posix_spawnattr_init(&attributes);
1171
1172 // Unmask all signals
1173 sigset_t noSignals;
1174 sigemptyset(&noSignals);
1175 posix_spawnattr_setsigmask(&attributes, &noSignals);
1176
1177 // Reset all signals to default behavior.
1178 //
1179 // On Linux, this can only be used to reset signals that are legal to
1180 // modify, so we have to take care about the set we use.
1181 #if defined(__linux__)
1182 sigset_t mostSignals;
1183 sigemptyset(&mostSignals);
1184 for (int i = 1; i < SIGSYS; ++i) {
1185 if (i == SIGKILL || i == SIGSTOP) continue;
1186 sigaddset(&mostSignals, i);
1187 }
1188 posix_spawnattr_setsigdefault(&attributes, &mostSignals);
1189 #else
1190 sigset_t mostSignals;
1191 sigfillset(&mostSignals);
1192 sigdelset(&mostSignals, SIGKILL);
1193 sigdelset(&mostSignals, SIGSTOP);
1194 posix_spawnattr_setsigdefault(&attributes, &mostSignals);
1195 #endif
1196
1197 // Establish a separate process group.
1198 posix_spawnattr_setpgroup(&attributes, 0);
1199
1200 // Set the attribute flags.
1201 unsigned flags = POSIX_SPAWN_SETSIGMASK | POSIX_SPAWN_SETSIGDEF;
1202 if (!isConsole)
1203 flags |= POSIX_SPAWN_SETPGROUP;
1204
1205 // Close all other files by default.
1206 //
1207 // FIXME: This is an Apple-specific extension, and we will have to do
1208 // something else on other platforms (and unfortunately, there isn't
1209 // really an easy answer other than using a stub executable).
1210 #ifdef __APPLE__
1211 flags |= POSIX_SPAWN_CLOEXEC_DEFAULT;
1212 #endif
1213
1214 posix_spawnattr_setflags(&attributes, flags);
1215
1216 // Setup the file actions.
1217 posix_spawn_file_actions_t fileActions;
1218 posix_spawn_file_actions_init(&fileActions);
1219
1220 // Open /dev/null as stdin.
1221 posix_spawn_file_actions_addopen(
1222 &fileActions, 0, "/dev/null", O_RDONLY, 0);
1223
1224 // Create a pipe to use to read the command output, if necessary.
1225 int pipe[2]{ -1, -1 };
1226 if (!isConsole) {
1227 if (basic::sys::pipe(pipe) < 0) {
1228 context.emitError("unable to create command pipe (%s)",
1229 strerror(errno));
1230 return false;
1231 }
1232
1233 // Open the write end of the pipe as stdout and stderr.
1234 posix_spawn_file_actions_adddup2(&fileActions, pipe[1], 1);
1235 posix_spawn_file_actions_adddup2(&fileActions, pipe[1], 2);
1236
1237 // Close the read and write ends of the pipe.
1238 posix_spawn_file_actions_addclose(&fileActions, pipe[0]);
1239 posix_spawn_file_actions_addclose(&fileActions, pipe[1]);
1240 } else {
1241 // Otherwise, propagate the current stdout/stderr.
1242 posix_spawn_file_actions_adddup2(&fileActions, 1, 1);
1243 posix_spawn_file_actions_adddup2(&fileActions, 2, 2);
1244 }
1245
1246 // Spawn the command.
1247 const char* args[4];
1248 args[0] = "/bin/sh";
1249 args[1] = "-c";
1250 args[2] = command->getCommandString().c_str();
1251 args[3] = nullptr;
1252
1253 // Spawn the command.
1254 pid_t pid;
1255 {
1256 // We need to hold the spawn processes lock when we spawn, to ensure that
1257 // we don't create a process in between when we are cancelled.
1258 std::lock_guard<std::mutex> guard(context.spawnedProcessesMutex);
1259
1260 if (posix_spawn(&pid, args[0], /*file_actions=*/&fileActions,
1261 /*attrp=*/&attributes, const_cast<char**>(args),
1262 environ) != 0) {
1263 context.emitError("unable to spawn process (%s)", strerror(errno));
1264 return false;
1265 }
1266
1267 // The console process will get interrupted automatically.
1268 if (!isConsole)
1269 context.spawnedProcesses.insert(pid);
1270 }
1271
1272 posix_spawn_file_actions_destroy(&fileActions);
1273 posix_spawnattr_destroy(&attributes);
1274
1275 // Read the command output, if buffering.
1276 SmallString<1024> outputData;
1277 if (!isConsole) {
1278 // Close the write end of the output pipe.
1279 ::close(pipe[1]);
1280
1281 // Read all the data from the output pipe.
1282 while (true) {
1283 char buf[4096];
1284 ssize_t numBytes = read(pipe[0], buf, sizeof(buf));
1285 if (numBytes < 0) {
1286 context.emitError("unable to read from output pipe (%s)",
1287 strerror(errno));
1288 break;
1289 }
1290
1291 if (numBytes == 0)
1292 break;
1293
1294 outputData.insert(outputData.end(), &buf[0], &buf[numBytes]);
1295 }
1296
1297 // Close the read end of the pipe.
1298 ::close(pipe[0]);
1299 }
1300
1301 // Wait for the command to complete.
1302 int status, result = waitpid(pid, &status, 0);
1303 while (result == -1 && errno == EINTR)
1304 result = waitpid(pid, &status, 0);
1305
1306 // Update the set of spawned processes.
1307 {
1308 std::lock_guard<std::mutex> guard(context.spawnedProcessesMutex);
1309 context.spawnedProcesses.erase(pid);
1310 }
1311
1312 if (result == -1) {
1313 context.emitError("unable to wait for process (%s)", strerror(errno));
1314 }
1315
1316 // If the build has been interrupted, return without writing any output or
1317 // command status (since they will also have been interrupted).
1318 if (context.isCancelled && context.wasCancelledBySigint) {
1319 // We still return an accurate status just in case the command actually
1320 // completed successfully.
1321 return status == 0;
1322 }
1323
1324 // If the child failed, show the full command and the output.
1325 if (status != 0) {
1326 // If the process was killed by SIGINT, assume it is because we were
1327 // interrupted.
1328 bool cancelled = WIFSIGNALED(status) && (WTERMSIG(status) == SIGINT || WTERMSIG(status) == SIGKILL);
1329 if (cancelled)
1330 return false;
1331
1332 // Otherwise, report the failure.
1333 context.emitErrorAndText(
1334 getFormattedString(
1335 "process failed: %s", command->getCommandString().c_str()),
1336 std::string(outputData.data(), outputData.size()));
1337
1338 // Update the count of failed commands.
1339 context.incrementFailedCommands();
1340
1341 return false;
1342 } else {
1343 // Write the output data, if buffered.
1344 if (!outputData.empty()) {
1345 context.emitText(std::string(outputData.data(), outputData.size()));
1346 }
1347 }
1348
1349 return true;
1350 }
1351
1352 bool processDiscoveredDependencies() {
1353 // Process the discovered dependencies, if used.
1354 switch (command->getDepsStyle()) {
1355 case ninja::Command::DepsStyleKind::None:
1356 return true;
1357 case ninja::Command::DepsStyleKind::MSVC: {
1358 context.emitError("MSVC style dependencies are unsupported");
1359 return false;
1360 }
1361 case ninja::Command::DepsStyleKind::GCC: {
1362 // Read the dependencies file.
1363 std::string error;
1364 std::unique_ptr<char[]> data;
1365 uint64_t length;
1366 if (!util::readFileContents(command->getDepsFile(), &data, &length,
1367 &error)) {
1368 // If the file is missing, just ignore it for consistency with Ninja
1369 // (when using stored deps) in non-strict mode.
1370 if (!context.strict)
1371 return true;
1372
1373 // FIXME: Error handling.
1374 context.emitError("unable to read dependency file: %s (%s)",
1375 command->getDepsFile().c_str(), error.c_str());
1376 return false;
1377 }
1378
1379 // Parse the output.
1380 //
1381 // We just ignore the rule, and add any dependency that we encounter in
1382 // the file.
1383 struct DepsActions : public core::MakefileDepsParser::ParseActions {
1384 BuildContext& context;
1385 NinjaCommandTask* task;
1386 const std::string& path;
1387 unsigned numErrors{0};
1388
1389 DepsActions(BuildContext& context, NinjaCommandTask* task,
1390 const std::string& path)
1391 : context(context), task(task), path(path) {}
1392
1393 virtual void error(const char* message, uint64_t position) override {
1394 context.emitError(
1395 "error reading dependency file: %s (%s) at offset %u",
1396 path.c_str(), message, unsigned(position));
1397 ++numErrors;
1398 }
1399
1400 virtual void actOnRuleDependency(const char* dependency,
1401 uint64_t length,
1402 const StringRef
1403 unescapedWord) override {
1404 context.engine.taskDiscoveredDependency(task, unescapedWord);
1405 }
1406
1407 virtual void actOnRuleStart(const char* name, uint64_t length,
1408 const StringRef unescapedWord) override {}
1409 virtual void actOnRuleEnd() override {}
1410 };
1411
1412 DepsActions actions(context, this, command->getDepsFile());
1413 core::MakefileDepsParser(data.get(), length, actions).parse();
1414 return actions.numErrors == 0;
1415 }
1416 }
1417
1418 assert(0 && "unexpected case");
1419 return false;
1420 }
1421 };
1422
1423 return context.engine.registerTask(new NinjaCommandTask(context, command));
1424 }
1425
buildInput(BuildContext & context,ninja::Node * input)1426 static core::Task* buildInput(BuildContext& context, ninja::Node* input) {
1427 struct NinjaInputTask : core::Task {
1428 BuildContext& context;
1429 ninja::Node* node;
1430
1431 NinjaInputTask(BuildContext& context, ninja::Node* node)
1432 : context(context), node(node) { }
1433
1434 virtual void provideValue(core::BuildEngine& engine, uintptr_t inputID,
1435 const core::ValueType& value) override { }
1436
1437 virtual void start(core::BuildEngine& engine) override { }
1438
1439 virtual void inputsAvailable(core::BuildEngine& engine) override {
1440 if (context.simulate) {
1441 engine.taskIsComplete(
1442 this, BuildValue::makeExistingInput({}).toValue());
1443 return;
1444 }
1445
1446 auto outputInfo = FileInfo::getInfoForPath(node->getPath());
1447 if (outputInfo.isMissing()) {
1448 engine.taskIsComplete(this, BuildValue::makeMissingInput().toValue());
1449 return;
1450 }
1451
1452 engine.taskIsComplete(
1453 this, BuildValue::makeExistingInput(outputInfo).toValue());
1454 }
1455 };
1456
1457 return context.engine.registerTask(new NinjaInputTask(context, input));
1458 }
1459
1460 static core::Task*
buildTargets(BuildContext & context,const std::vector<std::string> & targetsToBuild)1461 buildTargets(BuildContext& context,
1462 const std::vector<std::string>& targetsToBuild) {
1463 struct TargetsTask : core::Task {
1464 BuildContext& context;
1465 std::vector<std::string> targetsToBuild;
1466
1467 TargetsTask(BuildContext& context,
1468 const std::vector<std::string>& targetsToBuild)
1469 : context(context), targetsToBuild(targetsToBuild) { }
1470
1471 virtual void provideValue(core::BuildEngine& engine, uintptr_t inputID,
1472 const core::ValueType& valueData) override {
1473 BuildValue value = BuildValue::fromValue(valueData);
1474
1475 if (value.isMissingInput()) {
1476 context.emitError("unknown target '%s'",
1477 targetsToBuild[inputID].c_str());
1478 }
1479 }
1480
1481 virtual void start(core::BuildEngine& engine) override {
1482 // Request all of the targets.
1483 unsigned id = 0;
1484 for (const auto& target: targetsToBuild) {
1485 engine.taskNeedsInput(this, target, id++);
1486 }
1487 }
1488
1489 virtual void inputsAvailable(core::BuildEngine& engine) override {
1490 // Complete the job.
1491 engine.taskIsComplete(
1492 this, BuildValue::makeSuccessfulCommand({}, 0).toValue());
1493 return;
1494 }
1495 };
1496
1497 return context.engine.registerTask(new TargetsTask(context, targetsToBuild));
1498 }
1499
1500 static core::Task*
selectCompositeBuildResult(BuildContext & context,ninja::Command * command,unsigned inputIndex,const core::KeyType & compositeRuleName)1501 selectCompositeBuildResult(BuildContext& context, ninja::Command* command,
1502 unsigned inputIndex,
1503 const core::KeyType& compositeRuleName) {
1504 struct SelectResultTask : core::Task {
1505 const BuildContext& context;
1506 const ninja::Command* command;
1507 const unsigned inputIndex;
1508 const core::KeyType compositeRuleName;
1509 const core::ValueType *compositeValueData = nullptr;
1510
1511 SelectResultTask(BuildContext& context, ninja::Command* command,
1512 unsigned inputIndex,
1513 const core::KeyType& compositeRuleName)
1514 : context(context), command(command),
1515 inputIndex(inputIndex), compositeRuleName(compositeRuleName) { }
1516
1517 virtual void start(core::BuildEngine& engine) override {
1518 // Request the composite input.
1519 engine.taskNeedsInput(this, compositeRuleName, 0);
1520 }
1521
1522 virtual void provideValue(core::BuildEngine& engine, uintptr_t inputID,
1523 const core::ValueType& valueData) override {
1524 compositeValueData = &valueData;
1525 }
1526
1527 virtual void inputsAvailable(core::BuildEngine& engine) override {
1528 // Construct the appropriate build value from the result.
1529 assert(compositeValueData);
1530 BuildValue value(BuildValue::fromValue(*compositeValueData));
1531
1532 // If the input was a failed or skipped command, propagate that result.
1533 if (value.isFailedCommand() || value.isSkippedCommand()) {
1534 engine.taskIsComplete(this, value.toValue(), /*ForceChange=*/true);
1535 } else {
1536 // FIXME: We don't try and set this in response to the restat flag on
1537 // the incoming command, because it doesn't generally work -- the output
1538 // will just honor update-if-newer and still not run. We need to move to
1539 // a different model for handling restat = 0 to get this to work
1540 // properly.
1541 bool forceChange = false;
1542
1543 // Otherwise, the value should be a successful command with file info
1544 // for each output.
1545 assert(value.isSuccessfulCommand() && value.hasMultipleOutputs() &&
1546 inputIndex < value.getNumOutputs());
1547
1548 // The result is the InputIndex-th element, and the command hash is
1549 // propagated.
1550 engine.taskIsComplete(
1551 this, BuildValue::makeSuccessfulCommand(
1552 value.getNthOutputInfo(inputIndex),
1553 value.getCommandHash()).toValue(),
1554 forceChange);
1555 }
1556 }
1557 };
1558
1559 return context.engine.registerTask(
1560 new SelectResultTask(context, command, inputIndex, compositeRuleName));
1561 }
1562
buildInputIsResultValid(ninja::Node * node,const core::ValueType & valueData)1563 static bool buildInputIsResultValid(ninja::Node* node,
1564 const core::ValueType& valueData) {
1565 BuildValue value = BuildValue::fromValue(valueData);
1566
1567 // If the prior value wasn't for an existing input, recompute.
1568 if (!value.isExistingInput())
1569 return false;
1570
1571 // Otherwise, the result is valid if the path exists and the hash has not
1572 // changed.
1573 //
1574 // FIXME: This is inefficient, we will end up doing the stat twice, once when
1575 // we check the value for up to dateness, and once when we "build" the output.
1576 //
1577 // We can solve this by caching ourselves but I wonder if it is something the
1578 // engine should support more naturally.
1579 auto info = FileInfo::getInfoForPath(node->getPath());
1580 if (info.isMissing())
1581 return false;
1582
1583 return value.getOutputInfo() == info;
1584 }
1585
buildCommandIsResultValid(ninja::Command * command,const core::ValueType & valueData)1586 static bool buildCommandIsResultValid(ninja::Command* command,
1587 const core::ValueType& valueData) {
1588 BuildValue value = BuildValue::fromValue(valueData);
1589
1590 // If the prior value wasn't for a successful command, recompute.
1591 if (!value.isSuccessfulCommand())
1592 return false;
1593
1594 // For non-generator commands, if the command hash has changed, recompute.
1595 if (!command->hasGeneratorFlag()) {
1596 if (value.getCommandHash() != basic::hashString(
1597 command->getCommandString()))
1598 return false;
1599 }
1600
1601 // Check the timestamps on each of the outputs.
1602 for (unsigned i = 0, e = command->getOutputs().size(); i != e; ++i) {
1603 // Always rebuild if the output is missing.
1604 auto info = FileInfo::getInfoForPath(command->getOutputs()[i]->getPath());
1605 if (info.isMissing())
1606 return false;
1607
1608 // Otherwise, the result is valid if file information has not changed.
1609 //
1610 // Note that we may still decide not to actually run the command based on
1611 // the update-if-newer handling, but it does require running the task.
1612 if (value.getNthOutputInfo(i) != info)
1613 return false;
1614 }
1615
1616 return true;
1617 }
1618
selectCompositeIsResultValid(ninja::Command * command,const core::ValueType & valueData)1619 static bool selectCompositeIsResultValid(ninja::Command* command,
1620 const core::ValueType& valueData) {
1621 BuildValue value = BuildValue::fromValue(valueData);
1622
1623 // If the prior value wasn't for a successful command, recompute.
1624 if (!value.isSuccessfulCommand())
1625 return false;
1626
1627 // If the command's signature has changed since it was built, rebuild. This is
1628 // important for ensuring that we properly reevaluate the select rule when
1629 // it's incoming composite rule no longer exists.
1630 if (value.getCommandHash() != basic::hashString(command->getCommandString()))
1631 return false;
1632
1633 // Otherwise, this result is always valid.
1634 return true;
1635 }
1636
updateCommandStatus(BuildContext & context,ninja::Command * command,core::Rule::StatusKind status)1637 static void updateCommandStatus(BuildContext& context,
1638 ninja::Command* command,
1639 core::Rule::StatusKind status) {
1640 // Ignore phony rules.
1641 if (command->getRule() == context.manifest->getPhonyRule())
1642 return;
1643
1644 // Track the number of commands which are currently being scanned along with
1645 // the total number of completed commands.
1646 if (status == core::Rule::StatusKind::IsScanning) {
1647 ++context.numCommandsScanning;
1648 } else if (status == core::Rule::StatusKind::IsUpToDate) {
1649 --context.numCommandsScanning;
1650 ++context.numCommandsUpToDate;
1651 ++context.numCommandsCompleted;
1652 } else {
1653 assert(status == core::Rule::StatusKind::IsComplete);
1654 --context.numCommandsScanning;
1655 ++context.numCommandsCompleted;
1656 }
1657 }
1658
lookupRule(const core::KeyType & key)1659 core::Rule NinjaBuildEngineDelegate::lookupRule(const core::KeyType& key) {
1660 // We created rules for all of the commands up front, so if we are asked for a
1661 // rule here it is because we are looking for an input.
1662
1663 // Get the node for this input.
1664 //
1665 // FIXME: This is frequently a redundant lookup, given that the caller might
1666 // well have had the Node* available. This is something that would be nice
1667 // to avoid when we support generic key types.
1668 ninja::Node* node = context->manifest->getOrCreateNode(key);
1669
1670 return core::Rule{
1671 node->getPath(),
1672 [&, node] (core::BuildEngine&) {
1673 return buildInput(*context, node);
1674 },
1675 [&, node] (core::BuildEngine&, const core::Rule&,
1676 const core::ValueType& value) {
1677 // If simulating, assume cached results are valid.
1678 if (context->simulate)
1679 return true;
1680
1681 return buildInputIsResultValid(node, value);
1682 } };
1683 }
1684
cycleDetected(const std::vector<core::Rule * > & cycle)1685 void NinjaBuildEngineDelegate::cycleDetected(
1686 const std::vector<core::Rule*>& cycle) {
1687 // Report the cycle.
1688 std::stringstream message;
1689 message << "cycle detected among targets:";
1690 bool first = true;
1691 for (const auto* rule: cycle) {
1692 if (!first)
1693 message << " ->";
1694 message << " \"" << rule->key << '"';
1695 first = false;
1696 }
1697
1698 context->emitError(message.str());
1699
1700 // Cancel the build.
1701 context->isCancelled = true;
1702 }
1703
error(const Twine & message)1704 void NinjaBuildEngineDelegate::error(const Twine& message) {
1705 // Report the error.
1706 context->emitError("error: " + message.str());
1707
1708 // Cancel the build.
1709 context->isCancelled = true;
1710 }
1711
1712 }
1713
executeNinjaBuildCommand(std::vector<std::string> args)1714 int commands::executeNinjaBuildCommand(std::vector<std::string> args) {
1715 std::string chdirPath = "";
1716 std::string customTool = "";
1717 std::string dbFilename = "build.db";
1718 std::string dumpGraphPath, profileFilename, traceFilename;
1719 std::string manifestFilename = "build.ninja";
1720
1721 // Create a context for the build.
1722 bool autoRegenerateManifest = true;
1723 bool quiet = false;
1724 bool simulate = false;
1725 bool strict = false;
1726 bool verbose = false;
1727 unsigned numJobsInParallel = 0;
1728 unsigned numFailedCommandsToTolerate = 1;
1729 double maximumLoadAverage = 0.0;
1730 std::vector<std::string> debugTools;
1731
1732 while (!args.empty() && args[0][0] == '-') {
1733 const std::string option = args[0];
1734 args.erase(args.begin());
1735
1736 if (option == "--")
1737 break;
1738
1739 if (option == "--version") {
1740 // Report a fake version for tools (like CMake) that detect compatibility
1741 // based on the 'Ninja' version.
1742 printf("1.5 Ninja Compatible (%s)\n", getLLBuildFullVersion().c_str());
1743 return 0;
1744 } else if (option == "--help") {
1745 usage(/*exitCode=*/0);
1746 } else if (option == "--simulate") {
1747 simulate = true;
1748 } else if (option == "--quiet") {
1749 quiet = true;
1750 } else if (option == "-C" || option == "--chdir") {
1751 if (args.empty()) {
1752 fprintf(stderr, "%s: error: missing argument to '%s'\n\n",
1753 getProgramName(), option.c_str());
1754 usage();
1755 }
1756 chdirPath = args[0];
1757 args.erase(args.begin());
1758 } else if (option == "--no-db") {
1759 dbFilename = "";
1760 } else if (option == "--db") {
1761 if (args.empty()) {
1762 fprintf(stderr, "%s: error: missing argument to '%s'\n\n",
1763 getProgramName(), option.c_str());
1764 usage();
1765 }
1766 dbFilename = args[0];
1767 args.erase(args.begin());
1768 } else if (option == "--dump-graph") {
1769 if (args.empty()) {
1770 fprintf(stderr, "%s: error: missing argument to '%s'\n\n",
1771 getProgramName(), option.c_str());
1772 usage();
1773 }
1774 dumpGraphPath = args[0];
1775 args.erase(args.begin());
1776 } else if (option == "-f") {
1777 if (args.empty()) {
1778 fprintf(stderr, "%s: error: missing argument to '%s'\n\n",
1779 getProgramName(), option.c_str());
1780 usage();
1781 }
1782 manifestFilename = args[0];
1783 args.erase(args.begin());
1784 } else if (option == "-k") {
1785 if (args.empty()) {
1786 fprintf(stderr, "%s: error: missing argument to '%s'\n\n",
1787 getProgramName(), option.c_str());
1788 usage();
1789 }
1790 char *end;
1791 numFailedCommandsToTolerate = ::strtol(args[0].c_str(), &end, 10);
1792 if (*end != '\0') {
1793 fprintf(stderr, "%s: error: invalid argument '%s' to '%s'\n\n",
1794 getProgramName(), args[0].c_str(), option.c_str());
1795 usage();
1796 }
1797 args.erase(args.begin());
1798 } else if (option == "-l") {
1799 if (args.empty()) {
1800 fprintf(stderr, "%s: error: missing argument to '%s'\n\n",
1801 getProgramName(), option.c_str());
1802 usage();
1803 }
1804 char *end;
1805 maximumLoadAverage = ::strtod(args[0].c_str(), &end);
1806 if (*end != '\0') {
1807 fprintf(stderr, "%s: error: invalid argument '%s' to '%s'\n\n",
1808 getProgramName(), args[0].c_str(), option.c_str());
1809 usage();
1810 }
1811 args.erase(args.begin());
1812 } else if (option == "-j" || option == "--jobs") {
1813 if (args.empty()) {
1814 fprintf(stderr, "%s: error: missing argument to '%s'\n\n",
1815 getProgramName(), option.c_str());
1816 usage();
1817 }
1818 char *end;
1819 numJobsInParallel = ::strtol(args[0].c_str(), &end, 10);
1820 if (*end != '\0') {
1821 fprintf(stderr, "%s: error: invalid argument '%s' to '%s'\n\n",
1822 getProgramName(), args[0].c_str(), option.c_str());
1823 usage();
1824 }
1825 args.erase(args.begin());
1826 } else if (StringRef(option).startswith("-j")) {
1827 char *end;
1828 numJobsInParallel = ::strtol(&option[2], &end, 10);
1829 if (*end != '\0') {
1830 fprintf(stderr, "%s: error: invalid argument '%s' to '-j'\n\n",
1831 getProgramName(), &option[2]);
1832 usage();
1833 }
1834 } else if (option == "--no-regenerate") {
1835 autoRegenerateManifest = false;
1836 } else if (option == "--profile") {
1837 if (args.empty()) {
1838 fprintf(stderr, "%s: error: missing argument to '%s'\n\n",
1839 getProgramName(), option.c_str());
1840 usage();
1841 }
1842 profileFilename = args[0];
1843 args.erase(args.begin());
1844 } else if (option == "--strict") {
1845 strict = true;
1846 } else if (option == "-t" || option == "--tool") {
1847 if (args.empty()) {
1848 fprintf(stderr, "%s: error: missing argument to '%s'\n\n",
1849 getProgramName(), option.c_str());
1850 usage();
1851 }
1852 customTool = args[0];
1853 args.erase(args.begin());
1854 } else if (option == "-d") {
1855 if (args.empty()) {
1856 fprintf(stderr, "%s: error: missing argument to '%s'\n\n",
1857 getProgramName(), option.c_str());
1858 usage();
1859 }
1860 debugTools.push_back(args[0]);
1861 args.erase(args.begin());
1862 } else if (option == "--trace") {
1863 if (args.empty()) {
1864 fprintf(stderr, "%s: error: missing argument to '%s'\n\n",
1865 getProgramName(), option.c_str());
1866 usage();
1867 }
1868 traceFilename = args[0];
1869 args.erase(args.begin());
1870 } else if (option == "-v" || option == "--verbose") {
1871 verbose = true;
1872 } else {
1873 fprintf(stderr, "%s: error: invalid option: '%s'\n\n",
1874 getProgramName(), option.c_str());
1875 usage();
1876 }
1877 }
1878
1879 if (maximumLoadAverage > 0.0) {
1880 fprintf(stderr, "%s: warning: maximum load average %.8g not implemented\n",
1881 getProgramName(), maximumLoadAverage);
1882 }
1883
1884 if (!debugTools.empty()) {
1885 fprintf(stderr, "%s: warning: debug tools not implemented\n",
1886 getProgramName());
1887 }
1888
1889 // Honor the --chdir option, if used.
1890 if (!chdirPath.empty()) {
1891 if (!sys::chdir(chdirPath.c_str())) {
1892 fprintf(stderr, "%s: error: unable to honor --chdir: %s\n",
1893 getProgramName(), strerror(errno));
1894 return 1;
1895 }
1896
1897 // Print a message about the changed directory. The exact format here is
1898 // important, it is recognized by other tools (like Emacs).
1899 fprintf(stdout, "%s: Entering directory `%s'\n", getProgramName(),
1900 chdirPath.c_str());
1901 fflush(stdout);
1902 }
1903
1904 if (!customTool.empty()) {
1905 std::vector<std::string> availableTools = {
1906 "targets",
1907 "list",
1908 };
1909
1910 if (std::find(availableTools.begin(), availableTools.end(), customTool) ==
1911 availableTools.end()) {
1912 fprintf(stderr, "error: unknown tool '%s'\n", customTool.c_str());
1913 return 1;
1914 } else if (customTool == "list") {
1915 if (!args.empty()) {
1916 fprintf(stderr, "error: unsupported arguments to tool '%s'\n",
1917 customTool.c_str());
1918 return 1;
1919 }
1920
1921 fprintf(stdout, "available ninja tools:\n");
1922 for (const auto& tool: availableTools) {
1923 fprintf(stdout, " %s\n", tool.c_str());
1924 }
1925
1926 return 0;
1927 }
1928 }
1929
1930 // Run up to two iterations, the first one loads the manifest and rebuilds it
1931 // if necessary, the second only runs if the manifest needs to be reloaded.
1932 //
1933 // This is somewhat inefficient in the case where the manifest needs to be
1934 // reloaded (we reopen the database, for example), but we don't expect that to
1935 // be a common case spot in practice.
1936 for (int iteration = 0; iteration != 2; ++iteration) {
1937 BuildContext context;
1938
1939 context.numFailedCommandsToTolerate = numFailedCommandsToTolerate;
1940 context.quiet = quiet;
1941 context.simulate = simulate;
1942 context.strict = strict;
1943 context.verbose = verbose;
1944
1945 // Create the job queue to use.
1946 //
1947 // When running in parallel, we use a LIFO queue to work around the default
1948 // traversal of the BuildEngine tending to build in BFS order. This is
1949 // generally at avoiding clustering of links.
1950 //
1951 // FIXME: Do a serious analysis of scheduling, including ideally an active
1952 // scheduler in the execution queue.
1953 if (numJobsInParallel == 0) {
1954 unsigned numCPUs = std::thread::hardware_concurrency();
1955 if (numCPUs == 0) {
1956 context.emitError("unable to detect number of CPUs (%s)",
1957 strerror(errno));
1958 return 1;
1959 }
1960
1961 numJobsInParallel = numCPUs + 2;
1962 }
1963 bool useLIFO = (numJobsInParallel > 1);
1964 context.jobQueue.reset(new BuildExecutionQueue(numJobsInParallel, useLIFO));
1965
1966 // Load the manifest.
1967 BuildManifestActions actions(context);
1968 ninja::ManifestLoader loader(manifestFilename, actions);
1969 context.manifest = loader.load();
1970
1971 // If there were errors loading, we are done.
1972 if (unsigned numErrors = actions.getNumErrors()) {
1973 context.emitNote("%d errors generated.", numErrors);
1974 return 1;
1975 }
1976
1977 // Run the targets tool, if specified.
1978 if (!customTool.empty() && customTool == "targets") {
1979 if (args.size() != 1 || args[0] != "all") {
1980 if (args.empty()) {
1981 context.emitError("unsupported arguments to tool '%s'",
1982 customTool.c_str());
1983 } else {
1984 context.emitError("unsupported argument to tool '%s': '%s'",
1985 customTool.c_str(), args[0].c_str());
1986 }
1987 return 1;
1988 }
1989
1990 for (const auto command: context.manifest->getCommands()) {
1991 for (const auto& output: command->getOutputs()) {
1992 fprintf(stdout, "%s: %s\n", output->getPath().c_str(),
1993 command->getRule()->getName().c_str());
1994 }
1995 }
1996
1997 return 0;
1998 }
1999
2000 // Otherwise, run the build.
2001
2002 // Parse the positional arguments.
2003 std::vector<std::string> targetsToBuild(args);
2004
2005 // Attach the database, if requested.
2006 if (!dbFilename.empty()) {
2007 std::string error;
2008 std::unique_ptr<core::BuildDB> db(
2009 core::createSQLiteBuildDB(dbFilename,
2010 BuildValue::currentSchemaVersion,
2011 &error));
2012 if (!db || !context.engine.attachDB(std::move(db), &error)) {
2013 context.emitError("unable to open build database: %s", error.c_str());
2014 return 1;
2015 }
2016 }
2017
2018 // Enable tracing, if requested.
2019 if (!traceFilename.empty()) {
2020 std::string error;
2021 if (!context.engine.enableTracing(traceFilename, &error)) {
2022 context.emitError("unable to enable tracing: %s", error.c_str());
2023 return 1;
2024 }
2025 }
2026
2027 // Create rules for all of the build commands up front.
2028 //
2029 // FIXME: We should probably also move this to be dynamic.
2030 for (const auto command: context.manifest->getCommands()) {
2031 // If this command has a single output, create the trivial rule.
2032 if (command->getOutputs().size() == 1) {
2033 context.engine.addRule({
2034 command->getOutputs()[0]->getPath(),
2035 [=, &context](core::BuildEngine& engine) {
2036 return buildCommand(context, command);
2037 },
2038 [=, &context](core::BuildEngine&, const core::Rule& rule,
2039 const core::ValueType value) {
2040 // If simulating, assume cached results are valid.
2041 if (context.simulate)
2042 return true;
2043
2044 return buildCommandIsResultValid(command, value);
2045 },
2046 [=, &context](core::BuildEngine&, core::Rule::StatusKind status) {
2047 updateCommandStatus(context, command, status);
2048 } });
2049 continue;
2050 }
2051
2052 // Otherwise, create a composite rule group for the multiple outputs.
2053
2054 // Create a signature for the composite rule.
2055 //
2056 // FIXME: Make efficient.
2057 std::string compositeRuleName = "";
2058 for (auto& output: command->getOutputs()) {
2059 if (!compositeRuleName.empty())
2060 compositeRuleName += "&&";
2061 compositeRuleName += output->getPath();
2062 }
2063
2064 // Add the composite rule, which will run the command and build all
2065 // outputs.
2066 context.engine.addRule({
2067 compositeRuleName,
2068 [=, &context](core::BuildEngine& engine) {
2069 return buildCommand(context, command);
2070 },
2071 [=, &context](core::BuildEngine&, const core::Rule& rule,
2072 const core::ValueType value) {
2073 // If simulating, assume cached results are valid.
2074 if (context.simulate)
2075 return true;
2076
2077 return buildCommandIsResultValid(command, value);
2078 },
2079 [=, &context](core::BuildEngine&, core::Rule::StatusKind status) {
2080 updateCommandStatus(context, command, status);
2081 } });
2082
2083 // Create the per-output selection rules that select the individual output
2084 // result from the composite result.
2085 for (unsigned i = 0, e = command->getOutputs().size(); i != e; ++i) {
2086 context.engine.addRule({
2087 command->getOutputs()[i]->getPath(),
2088 [=, &context] (core::BuildEngine&) {
2089 return selectCompositeBuildResult(context, command, i,
2090 compositeRuleName);
2091 },
2092 [=, &context] (core::BuildEngine&, const core::Rule& rule,
2093 const core::ValueType value) {
2094 // If simulating, assume cached results are valid.
2095 if (context.simulate)
2096 return true;
2097
2098 return selectCompositeIsResultValid(command, value);
2099 } });
2100 }
2101 }
2102
2103 // If this is the first iteration, build the manifest, unless disabled.
2104 if (autoRegenerateManifest && iteration == 0) {
2105 context.engine.build(manifestFilename);
2106
2107 // If the manifest was rebuilt, then reload it and build again.
2108 if (context.numBuiltCommands) {
2109 continue;
2110 }
2111
2112 // Otherwise, perform the main build.
2113 //
2114 // FIXME: This is somewhat inefficient, as we will end up repeating any
2115 // dependency scanning that was required for checking the manifest. We can
2116 // fix this by building the manifest inline with the targets...
2117 }
2118
2119 // If using a build profile, open it.
2120 if (!profileFilename.empty()) {
2121 context.profileFP = ::fopen(profileFilename.c_str(), "w");
2122 if (!context.profileFP) {
2123 context.emitError("unable to open build profile '%s' (%s)\n",
2124 profileFilename.c_str(), strerror(errno));
2125 return 1;
2126 }
2127
2128 fprintf(context.profileFP, "[\n");
2129 }
2130
2131 // If no explicit targets were named, build the default targets.
2132 if (targetsToBuild.empty()) {
2133 for (auto& target: context.manifest->getDefaultTargets())
2134 targetsToBuild.push_back(target->getPath());
2135
2136 // If there are no default targets, then build all of the root targets.
2137 if (targetsToBuild.empty()) {
2138 std::unordered_set<const ninja::Node*> inputNodes;
2139
2140 // Collect all of the input nodes.
2141 for (const auto& command: context.manifest->getCommands()) {
2142 for (const auto* input: command->getInputs()) {
2143 inputNodes.emplace(input);
2144 }
2145 }
2146
2147 // Build all of the targets that are not an input.
2148 for (const auto& command: context.manifest->getCommands()) {
2149 for (const auto& output: command->getOutputs()) {
2150 if (!inputNodes.count(output)) {
2151 targetsToBuild.push_back(output->getPath());
2152 }
2153 }
2154 }
2155 }
2156 }
2157
2158 // Generate an error if there is nothing to build.
2159 if (targetsToBuild.empty()) {
2160 context.emitError("no targets to build");
2161 return 1;
2162 }
2163
2164 // If building multiple targets, do so via a dummy rule to allow them to
2165 // build concurrently (and without duplicates).
2166 //
2167 // FIXME: We should sort out eventually whether the engine itself should
2168 // support this. It seems like an obvious feature, but it is also trivial
2169 // for the client to implement on top of the existing API.
2170 if (targetsToBuild.size() > 1) {
2171 // Create a dummy rule to build all targets.
2172 context.engine.addRule({
2173 "<<build>>",
2174 [&](core::BuildEngine&) {
2175 return buildTargets(context, targetsToBuild);
2176 },
2177 [&](core::BuildEngine&, const core::Rule&, const core::ValueType&) {
2178 // Always rebuild the dummy rule.
2179 return false;
2180 } });
2181
2182 context.engine.build("<<build>>");
2183 } else {
2184 context.engine.build(targetsToBuild[0]);
2185 }
2186
2187 if (!dumpGraphPath.empty()) {
2188 context.engine.dumpGraphToFile(dumpGraphPath);
2189 }
2190
2191 // Close the build profile, if used.
2192 if (context.profileFP) {
2193 ::fclose(context.profileFP);
2194
2195 context.emitNote(
2196 "wrote build profile to '%s', use Chrome's about:tracing to view.",
2197 profileFilename.c_str());
2198 }
2199
2200 // If the build was cancelled by SIGINT, cause ourself to also die by SIGINT
2201 // to support proper shell behavior.
2202 if (context.wasCancelledBySigint) {
2203 // Ensure SIGINT action is default.
2204 struct sigaction action{};
2205 action.sa_handler = SIG_DFL;
2206 sigaction(SIGINT, &action, 0);
2207
2208 kill(getpid(), SIGINT);
2209 std::this_thread::sleep_for(std::chrono::microseconds(1000));
2210 return 2;
2211 }
2212
2213 // If there were command failures, report the count.
2214 if (context.numFailedCommands) {
2215 context.emitError("build had %d command failures",
2216 context.numFailedCommands.load());
2217 }
2218
2219 // If the build was stopped because of an error, return an error status.
2220 if (context.numErrors) {
2221 return 1;
2222 }
2223
2224 // Otherwise, if nothing was done, print a single message to let the user
2225 // know we completed successfully.
2226 if (!context.quiet && !context.numBuiltCommands) {
2227 context.emitNote("no work to do.");
2228 }
2229
2230 // If we reached here on the first iteration, then we don't need a second
2231 // and are done.
2232 if (iteration == 0)
2233 break;
2234 }
2235
2236 // Return an appropriate exit status.
2237 return 0;
2238 }
2239