1 // Copyright 2012 Google Inc. All Rights Reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 #include "deps_log.h"
16
17 #include <sys/stat.h>
18 #ifndef _WIN32
19 #include <unistd.h>
20 #endif
21
22 #include "graph.h"
23 #include "util.h"
24 #include "test.h"
25
26 using namespace std;
27
28 namespace {
29
30 const char kTestFilename[] = "DepsLogTest-tempfile";
31
32 struct DepsLogTest : public testing::Test {
SetUp__anonebf1ddab0111::DepsLogTest33 virtual void SetUp() {
34 // In case a crashing test left a stale file behind.
35 unlink(kTestFilename);
36 }
TearDown__anonebf1ddab0111::DepsLogTest37 virtual void TearDown() {
38 unlink(kTestFilename);
39 }
40 };
41
TEST_F(DepsLogTest,WriteRead)42 TEST_F(DepsLogTest, WriteRead) {
43 State state1;
44 DepsLog log1;
45 string err;
46 EXPECT_TRUE(log1.OpenForWrite(kTestFilename, &err));
47 ASSERT_EQ("", err);
48
49 {
50 vector<Node*> deps;
51 deps.push_back(state1.GetNode("foo.h", 0));
52 deps.push_back(state1.GetNode("bar.h", 0));
53 log1.RecordDeps(state1.GetNode("out.o", 0), 1, deps);
54
55 deps.clear();
56 deps.push_back(state1.GetNode("foo.h", 0));
57 deps.push_back(state1.GetNode("bar2.h", 0));
58 log1.RecordDeps(state1.GetNode("out2.o", 0), 2, deps);
59
60 DepsLog::Deps* log_deps = log1.GetDeps(state1.GetNode("out.o", 0));
61 ASSERT_TRUE(log_deps);
62 ASSERT_EQ(1, log_deps->mtime);
63 ASSERT_EQ(2, log_deps->node_count);
64 ASSERT_EQ("foo.h", log_deps->nodes[0]->path());
65 ASSERT_EQ("bar.h", log_deps->nodes[1]->path());
66 }
67
68 log1.Close();
69
70 State state2;
71 DepsLog log2;
72 EXPECT_TRUE(log2.Load(kTestFilename, &state2, &err));
73 ASSERT_EQ("", err);
74
75 ASSERT_EQ(log1.nodes().size(), log2.nodes().size());
76 for (int i = 0; i < (int)log1.nodes().size(); ++i) {
77 Node* node1 = log1.nodes()[i];
78 Node* node2 = log2.nodes()[i];
79 ASSERT_EQ(i, node1->id());
80 ASSERT_EQ(node1->id(), node2->id());
81 }
82
83 // Spot-check the entries in log2.
84 DepsLog::Deps* log_deps = log2.GetDeps(state2.GetNode("out2.o", 0));
85 ASSERT_TRUE(log_deps);
86 ASSERT_EQ(2, log_deps->mtime);
87 ASSERT_EQ(2, log_deps->node_count);
88 ASSERT_EQ("foo.h", log_deps->nodes[0]->path());
89 ASSERT_EQ("bar2.h", log_deps->nodes[1]->path());
90 }
91
TEST_F(DepsLogTest,LotsOfDeps)92 TEST_F(DepsLogTest, LotsOfDeps) {
93 const int kNumDeps = 100000; // More than 64k.
94
95 State state1;
96 DepsLog log1;
97 string err;
98 EXPECT_TRUE(log1.OpenForWrite(kTestFilename, &err));
99 ASSERT_EQ("", err);
100
101 {
102 vector<Node*> deps;
103 for (int i = 0; i < kNumDeps; ++i) {
104 char buf[32];
105 sprintf(buf, "file%d.h", i);
106 deps.push_back(state1.GetNode(buf, 0));
107 }
108 log1.RecordDeps(state1.GetNode("out.o", 0), 1, deps);
109
110 DepsLog::Deps* log_deps = log1.GetDeps(state1.GetNode("out.o", 0));
111 ASSERT_EQ(kNumDeps, log_deps->node_count);
112 }
113
114 log1.Close();
115
116 State state2;
117 DepsLog log2;
118 EXPECT_TRUE(log2.Load(kTestFilename, &state2, &err));
119 ASSERT_EQ("", err);
120
121 DepsLog::Deps* log_deps = log2.GetDeps(state2.GetNode("out.o", 0));
122 ASSERT_EQ(kNumDeps, log_deps->node_count);
123 }
124
125 // Verify that adding the same deps twice doesn't grow the file.
TEST_F(DepsLogTest,DoubleEntry)126 TEST_F(DepsLogTest, DoubleEntry) {
127 // Write some deps to the file and grab its size.
128 int file_size;
129 {
130 State state;
131 DepsLog log;
132 string err;
133 EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
134 ASSERT_EQ("", err);
135
136 vector<Node*> deps;
137 deps.push_back(state.GetNode("foo.h", 0));
138 deps.push_back(state.GetNode("bar.h", 0));
139 log.RecordDeps(state.GetNode("out.o", 0), 1, deps);
140 log.Close();
141
142 struct stat st;
143 ASSERT_EQ(0, stat(kTestFilename, &st));
144 file_size = (int)st.st_size;
145 ASSERT_GT(file_size, 0);
146 }
147
148 // Now reload the file, and read the same deps.
149 {
150 State state;
151 DepsLog log;
152 string err;
153 EXPECT_TRUE(log.Load(kTestFilename, &state, &err));
154
155 EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
156 ASSERT_EQ("", err);
157
158 vector<Node*> deps;
159 deps.push_back(state.GetNode("foo.h", 0));
160 deps.push_back(state.GetNode("bar.h", 0));
161 log.RecordDeps(state.GetNode("out.o", 0), 1, deps);
162 log.Close();
163
164 struct stat st;
165 ASSERT_EQ(0, stat(kTestFilename, &st));
166 int file_size_2 = (int)st.st_size;
167 ASSERT_EQ(file_size, file_size_2);
168 }
169 }
170
171 // Verify that adding the new deps works and can be compacted away.
TEST_F(DepsLogTest,Recompact)172 TEST_F(DepsLogTest, Recompact) {
173 const char kManifest[] =
174 "rule cc\n"
175 " command = cc\n"
176 " deps = gcc\n"
177 "build out.o: cc\n"
178 "build other_out.o: cc\n";
179
180 // Write some deps to the file and grab its size.
181 int file_size;
182 {
183 State state;
184 ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest));
185 DepsLog log;
186 string err;
187 ASSERT_TRUE(log.OpenForWrite(kTestFilename, &err));
188 ASSERT_EQ("", err);
189
190 vector<Node*> deps;
191 deps.push_back(state.GetNode("foo.h", 0));
192 deps.push_back(state.GetNode("bar.h", 0));
193 log.RecordDeps(state.GetNode("out.o", 0), 1, deps);
194
195 deps.clear();
196 deps.push_back(state.GetNode("foo.h", 0));
197 deps.push_back(state.GetNode("baz.h", 0));
198 log.RecordDeps(state.GetNode("other_out.o", 0), 1, deps);
199
200 log.Close();
201
202 struct stat st;
203 ASSERT_EQ(0, stat(kTestFilename, &st));
204 file_size = (int)st.st_size;
205 ASSERT_GT(file_size, 0);
206 }
207
208 // Now reload the file, and add slightly different deps.
209 int file_size_2;
210 {
211 State state;
212 ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest));
213 DepsLog log;
214 string err;
215 ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
216
217 ASSERT_TRUE(log.OpenForWrite(kTestFilename, &err));
218 ASSERT_EQ("", err);
219
220 vector<Node*> deps;
221 deps.push_back(state.GetNode("foo.h", 0));
222 log.RecordDeps(state.GetNode("out.o", 0), 1, deps);
223 log.Close();
224
225 struct stat st;
226 ASSERT_EQ(0, stat(kTestFilename, &st));
227 file_size_2 = (int)st.st_size;
228 // The file should grow to record the new deps.
229 ASSERT_GT(file_size_2, file_size);
230 }
231
232 // Now reload the file, verify the new deps have replaced the old, then
233 // recompact.
234 int file_size_3;
235 {
236 State state;
237 ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest));
238 DepsLog log;
239 string err;
240 ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
241
242 Node* out = state.GetNode("out.o", 0);
243 DepsLog::Deps* deps = log.GetDeps(out);
244 ASSERT_TRUE(deps);
245 ASSERT_EQ(1, deps->mtime);
246 ASSERT_EQ(1, deps->node_count);
247 ASSERT_EQ("foo.h", deps->nodes[0]->path());
248
249 Node* other_out = state.GetNode("other_out.o", 0);
250 deps = log.GetDeps(other_out);
251 ASSERT_TRUE(deps);
252 ASSERT_EQ(1, deps->mtime);
253 ASSERT_EQ(2, deps->node_count);
254 ASSERT_EQ("foo.h", deps->nodes[0]->path());
255 ASSERT_EQ("baz.h", deps->nodes[1]->path());
256
257 ASSERT_TRUE(log.Recompact(kTestFilename, &err));
258
259 // The in-memory deps graph should still be valid after recompaction.
260 deps = log.GetDeps(out);
261 ASSERT_TRUE(deps);
262 ASSERT_EQ(1, deps->mtime);
263 ASSERT_EQ(1, deps->node_count);
264 ASSERT_EQ("foo.h", deps->nodes[0]->path());
265 ASSERT_EQ(out, log.nodes()[out->id()]);
266
267 deps = log.GetDeps(other_out);
268 ASSERT_TRUE(deps);
269 ASSERT_EQ(1, deps->mtime);
270 ASSERT_EQ(2, deps->node_count);
271 ASSERT_EQ("foo.h", deps->nodes[0]->path());
272 ASSERT_EQ("baz.h", deps->nodes[1]->path());
273 ASSERT_EQ(other_out, log.nodes()[other_out->id()]);
274
275 // The file should have shrunk a bit for the smaller deps.
276 struct stat st;
277 ASSERT_EQ(0, stat(kTestFilename, &st));
278 file_size_3 = (int)st.st_size;
279 ASSERT_LT(file_size_3, file_size_2);
280 }
281
282 // Now reload the file and recompact with an empty manifest. The previous
283 // entries should be removed.
284 {
285 State state;
286 // Intentionally not parsing kManifest here.
287 DepsLog log;
288 string err;
289 ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
290
291 Node* out = state.GetNode("out.o", 0);
292 DepsLog::Deps* deps = log.GetDeps(out);
293 ASSERT_TRUE(deps);
294 ASSERT_EQ(1, deps->mtime);
295 ASSERT_EQ(1, deps->node_count);
296 ASSERT_EQ("foo.h", deps->nodes[0]->path());
297
298 Node* other_out = state.GetNode("other_out.o", 0);
299 deps = log.GetDeps(other_out);
300 ASSERT_TRUE(deps);
301 ASSERT_EQ(1, deps->mtime);
302 ASSERT_EQ(2, deps->node_count);
303 ASSERT_EQ("foo.h", deps->nodes[0]->path());
304 ASSERT_EQ("baz.h", deps->nodes[1]->path());
305
306 ASSERT_TRUE(log.Recompact(kTestFilename, &err));
307
308 // The previous entries should have been removed.
309 deps = log.GetDeps(out);
310 ASSERT_FALSE(deps);
311
312 deps = log.GetDeps(other_out);
313 ASSERT_FALSE(deps);
314
315 // The .h files pulled in via deps should no longer have ids either.
316 ASSERT_EQ(-1, state.LookupNode("foo.h")->id());
317 ASSERT_EQ(-1, state.LookupNode("baz.h")->id());
318
319 // The file should have shrunk more.
320 struct stat st;
321 ASSERT_EQ(0, stat(kTestFilename, &st));
322 int file_size_4 = (int)st.st_size;
323 ASSERT_LT(file_size_4, file_size_3);
324 }
325 }
326
327 // Verify that invalid file headers cause a new build.
TEST_F(DepsLogTest,InvalidHeader)328 TEST_F(DepsLogTest, InvalidHeader) {
329 const char *kInvalidHeaders[] = {
330 "", // Empty file.
331 "# ninjad", // Truncated first line.
332 "# ninjadeps\n", // No version int.
333 "# ninjadeps\n\001\002", // Truncated version int.
334 "# ninjadeps\n\001\002\003\004" // Invalid version int.
335 };
336 for (size_t i = 0; i < sizeof(kInvalidHeaders) / sizeof(kInvalidHeaders[0]);
337 ++i) {
338 FILE* deps_log = fopen(kTestFilename, "wb");
339 ASSERT_TRUE(deps_log != NULL);
340 ASSERT_EQ(
341 strlen(kInvalidHeaders[i]),
342 fwrite(kInvalidHeaders[i], 1, strlen(kInvalidHeaders[i]), deps_log));
343 ASSERT_EQ(0 ,fclose(deps_log));
344
345 string err;
346 DepsLog log;
347 State state;
348 ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
349 EXPECT_EQ("bad deps log signature or version; starting over", err);
350 }
351 }
352
353 // Simulate what happens when loading a truncated log file.
TEST_F(DepsLogTest,Truncated)354 TEST_F(DepsLogTest, Truncated) {
355 // Create a file with some entries.
356 {
357 State state;
358 DepsLog log;
359 string err;
360 EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
361 ASSERT_EQ("", err);
362
363 vector<Node*> deps;
364 deps.push_back(state.GetNode("foo.h", 0));
365 deps.push_back(state.GetNode("bar.h", 0));
366 log.RecordDeps(state.GetNode("out.o", 0), 1, deps);
367
368 deps.clear();
369 deps.push_back(state.GetNode("foo.h", 0));
370 deps.push_back(state.GetNode("bar2.h", 0));
371 log.RecordDeps(state.GetNode("out2.o", 0), 2, deps);
372
373 log.Close();
374 }
375
376 // Get the file size.
377 struct stat st;
378 ASSERT_EQ(0, stat(kTestFilename, &st));
379
380 // Try reloading at truncated sizes.
381 // Track how many nodes/deps were found; they should decrease with
382 // smaller sizes.
383 int node_count = 5;
384 int deps_count = 2;
385 for (int size = (int)st.st_size; size > 0; --size) {
386 string err;
387 ASSERT_TRUE(Truncate(kTestFilename, size, &err));
388
389 State state;
390 DepsLog log;
391 EXPECT_TRUE(log.Load(kTestFilename, &state, &err));
392 if (!err.empty()) {
393 // At some point the log will be so short as to be unparseable.
394 break;
395 }
396
397 ASSERT_GE(node_count, (int)log.nodes().size());
398 node_count = log.nodes().size();
399
400 // Count how many non-NULL deps entries there are.
401 int new_deps_count = 0;
402 for (vector<DepsLog::Deps*>::const_iterator i = log.deps().begin();
403 i != log.deps().end(); ++i) {
404 if (*i)
405 ++new_deps_count;
406 }
407 ASSERT_GE(deps_count, new_deps_count);
408 deps_count = new_deps_count;
409 }
410 }
411
412 // Run the truncation-recovery logic.
TEST_F(DepsLogTest,TruncatedRecovery)413 TEST_F(DepsLogTest, TruncatedRecovery) {
414 // Create a file with some entries.
415 {
416 State state;
417 DepsLog log;
418 string err;
419 EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
420 ASSERT_EQ("", err);
421
422 vector<Node*> deps;
423 deps.push_back(state.GetNode("foo.h", 0));
424 deps.push_back(state.GetNode("bar.h", 0));
425 log.RecordDeps(state.GetNode("out.o", 0), 1, deps);
426
427 deps.clear();
428 deps.push_back(state.GetNode("foo.h", 0));
429 deps.push_back(state.GetNode("bar2.h", 0));
430 log.RecordDeps(state.GetNode("out2.o", 0), 2, deps);
431
432 log.Close();
433 }
434
435 // Shorten the file, corrupting the last record.
436 {
437 struct stat st;
438 ASSERT_EQ(0, stat(kTestFilename, &st));
439 string err;
440 ASSERT_TRUE(Truncate(kTestFilename, st.st_size - 2, &err));
441 }
442
443 // Load the file again, add an entry.
444 {
445 State state;
446 DepsLog log;
447 string err;
448 EXPECT_TRUE(log.Load(kTestFilename, &state, &err));
449 ASSERT_EQ("premature end of file; recovering", err);
450 err.clear();
451
452 // The truncated entry should've been discarded.
453 EXPECT_EQ(NULL, log.GetDeps(state.GetNode("out2.o", 0)));
454
455 EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
456 ASSERT_EQ("", err);
457
458 // Add a new entry.
459 vector<Node*> deps;
460 deps.push_back(state.GetNode("foo.h", 0));
461 deps.push_back(state.GetNode("bar2.h", 0));
462 log.RecordDeps(state.GetNode("out2.o", 0), 3, deps);
463
464 log.Close();
465 }
466
467 // Load the file a third time to verify appending after a mangled
468 // entry doesn't break things.
469 {
470 State state;
471 DepsLog log;
472 string err;
473 EXPECT_TRUE(log.Load(kTestFilename, &state, &err));
474
475 // The truncated entry should exist.
476 DepsLog::Deps* deps = log.GetDeps(state.GetNode("out2.o", 0));
477 ASSERT_TRUE(deps);
478 }
479 }
480
481 } // anonymous namespace
482