1 // Copyright 2012 Google Inc. All Rights Reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 //     http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 
15 #include "deps_log.h"
16 
17 #include <sys/stat.h>
18 #ifndef _WIN32
19 #include <unistd.h>
20 #endif
21 
22 #include "graph.h"
23 #include "util.h"
24 #include "test.h"
25 
26 using namespace std;
27 
28 namespace {
29 
30 const char kTestFilename[] = "DepsLogTest-tempfile";
31 
32 struct DepsLogTest : public testing::Test {
SetUp__anonebf1ddab0111::DepsLogTest33   virtual void SetUp() {
34     // In case a crashing test left a stale file behind.
35     unlink(kTestFilename);
36   }
TearDown__anonebf1ddab0111::DepsLogTest37   virtual void TearDown() {
38     unlink(kTestFilename);
39   }
40 };
41 
TEST_F(DepsLogTest,WriteRead)42 TEST_F(DepsLogTest, WriteRead) {
43   State state1;
44   DepsLog log1;
45   string err;
46   EXPECT_TRUE(log1.OpenForWrite(kTestFilename, &err));
47   ASSERT_EQ("", err);
48 
49   {
50     vector<Node*> deps;
51     deps.push_back(state1.GetNode("foo.h", 0));
52     deps.push_back(state1.GetNode("bar.h", 0));
53     log1.RecordDeps(state1.GetNode("out.o", 0), 1, deps);
54 
55     deps.clear();
56     deps.push_back(state1.GetNode("foo.h", 0));
57     deps.push_back(state1.GetNode("bar2.h", 0));
58     log1.RecordDeps(state1.GetNode("out2.o", 0), 2, deps);
59 
60     DepsLog::Deps* log_deps = log1.GetDeps(state1.GetNode("out.o", 0));
61     ASSERT_TRUE(log_deps);
62     ASSERT_EQ(1, log_deps->mtime);
63     ASSERT_EQ(2, log_deps->node_count);
64     ASSERT_EQ("foo.h", log_deps->nodes[0]->path());
65     ASSERT_EQ("bar.h", log_deps->nodes[1]->path());
66   }
67 
68   log1.Close();
69 
70   State state2;
71   DepsLog log2;
72   EXPECT_TRUE(log2.Load(kTestFilename, &state2, &err));
73   ASSERT_EQ("", err);
74 
75   ASSERT_EQ(log1.nodes().size(), log2.nodes().size());
76   for (int i = 0; i < (int)log1.nodes().size(); ++i) {
77     Node* node1 = log1.nodes()[i];
78     Node* node2 = log2.nodes()[i];
79     ASSERT_EQ(i, node1->id());
80     ASSERT_EQ(node1->id(), node2->id());
81   }
82 
83   // Spot-check the entries in log2.
84   DepsLog::Deps* log_deps = log2.GetDeps(state2.GetNode("out2.o", 0));
85   ASSERT_TRUE(log_deps);
86   ASSERT_EQ(2, log_deps->mtime);
87   ASSERT_EQ(2, log_deps->node_count);
88   ASSERT_EQ("foo.h", log_deps->nodes[0]->path());
89   ASSERT_EQ("bar2.h", log_deps->nodes[1]->path());
90 }
91 
TEST_F(DepsLogTest,LotsOfDeps)92 TEST_F(DepsLogTest, LotsOfDeps) {
93   const int kNumDeps = 100000;  // More than 64k.
94 
95   State state1;
96   DepsLog log1;
97   string err;
98   EXPECT_TRUE(log1.OpenForWrite(kTestFilename, &err));
99   ASSERT_EQ("", err);
100 
101   {
102     vector<Node*> deps;
103     for (int i = 0; i < kNumDeps; ++i) {
104       char buf[32];
105       sprintf(buf, "file%d.h", i);
106       deps.push_back(state1.GetNode(buf, 0));
107     }
108     log1.RecordDeps(state1.GetNode("out.o", 0), 1, deps);
109 
110     DepsLog::Deps* log_deps = log1.GetDeps(state1.GetNode("out.o", 0));
111     ASSERT_EQ(kNumDeps, log_deps->node_count);
112   }
113 
114   log1.Close();
115 
116   State state2;
117   DepsLog log2;
118   EXPECT_TRUE(log2.Load(kTestFilename, &state2, &err));
119   ASSERT_EQ("", err);
120 
121   DepsLog::Deps* log_deps = log2.GetDeps(state2.GetNode("out.o", 0));
122   ASSERT_EQ(kNumDeps, log_deps->node_count);
123 }
124 
125 // Verify that adding the same deps twice doesn't grow the file.
TEST_F(DepsLogTest,DoubleEntry)126 TEST_F(DepsLogTest, DoubleEntry) {
127   // Write some deps to the file and grab its size.
128   int file_size;
129   {
130     State state;
131     DepsLog log;
132     string err;
133     EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
134     ASSERT_EQ("", err);
135 
136     vector<Node*> deps;
137     deps.push_back(state.GetNode("foo.h", 0));
138     deps.push_back(state.GetNode("bar.h", 0));
139     log.RecordDeps(state.GetNode("out.o", 0), 1, deps);
140     log.Close();
141 
142     struct stat st;
143     ASSERT_EQ(0, stat(kTestFilename, &st));
144     file_size = (int)st.st_size;
145     ASSERT_GT(file_size, 0);
146   }
147 
148   // Now reload the file, and read the same deps.
149   {
150     State state;
151     DepsLog log;
152     string err;
153     EXPECT_TRUE(log.Load(kTestFilename, &state, &err));
154 
155     EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
156     ASSERT_EQ("", err);
157 
158     vector<Node*> deps;
159     deps.push_back(state.GetNode("foo.h", 0));
160     deps.push_back(state.GetNode("bar.h", 0));
161     log.RecordDeps(state.GetNode("out.o", 0), 1, deps);
162     log.Close();
163 
164     struct stat st;
165     ASSERT_EQ(0, stat(kTestFilename, &st));
166     int file_size_2 = (int)st.st_size;
167     ASSERT_EQ(file_size, file_size_2);
168   }
169 }
170 
171 // Verify that adding the new deps works and can be compacted away.
TEST_F(DepsLogTest,Recompact)172 TEST_F(DepsLogTest, Recompact) {
173   const char kManifest[] =
174 "rule cc\n"
175 "  command = cc\n"
176 "  deps = gcc\n"
177 "build out.o: cc\n"
178 "build other_out.o: cc\n";
179 
180   // Write some deps to the file and grab its size.
181   int file_size;
182   {
183     State state;
184     ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest));
185     DepsLog log;
186     string err;
187     ASSERT_TRUE(log.OpenForWrite(kTestFilename, &err));
188     ASSERT_EQ("", err);
189 
190     vector<Node*> deps;
191     deps.push_back(state.GetNode("foo.h", 0));
192     deps.push_back(state.GetNode("bar.h", 0));
193     log.RecordDeps(state.GetNode("out.o", 0), 1, deps);
194 
195     deps.clear();
196     deps.push_back(state.GetNode("foo.h", 0));
197     deps.push_back(state.GetNode("baz.h", 0));
198     log.RecordDeps(state.GetNode("other_out.o", 0), 1, deps);
199 
200     log.Close();
201 
202     struct stat st;
203     ASSERT_EQ(0, stat(kTestFilename, &st));
204     file_size = (int)st.st_size;
205     ASSERT_GT(file_size, 0);
206   }
207 
208   // Now reload the file, and add slightly different deps.
209   int file_size_2;
210   {
211     State state;
212     ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest));
213     DepsLog log;
214     string err;
215     ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
216 
217     ASSERT_TRUE(log.OpenForWrite(kTestFilename, &err));
218     ASSERT_EQ("", err);
219 
220     vector<Node*> deps;
221     deps.push_back(state.GetNode("foo.h", 0));
222     log.RecordDeps(state.GetNode("out.o", 0), 1, deps);
223     log.Close();
224 
225     struct stat st;
226     ASSERT_EQ(0, stat(kTestFilename, &st));
227     file_size_2 = (int)st.st_size;
228     // The file should grow to record the new deps.
229     ASSERT_GT(file_size_2, file_size);
230   }
231 
232   // Now reload the file, verify the new deps have replaced the old, then
233   // recompact.
234   int file_size_3;
235   {
236     State state;
237     ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest));
238     DepsLog log;
239     string err;
240     ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
241 
242     Node* out = state.GetNode("out.o", 0);
243     DepsLog::Deps* deps = log.GetDeps(out);
244     ASSERT_TRUE(deps);
245     ASSERT_EQ(1, deps->mtime);
246     ASSERT_EQ(1, deps->node_count);
247     ASSERT_EQ("foo.h", deps->nodes[0]->path());
248 
249     Node* other_out = state.GetNode("other_out.o", 0);
250     deps = log.GetDeps(other_out);
251     ASSERT_TRUE(deps);
252     ASSERT_EQ(1, deps->mtime);
253     ASSERT_EQ(2, deps->node_count);
254     ASSERT_EQ("foo.h", deps->nodes[0]->path());
255     ASSERT_EQ("baz.h", deps->nodes[1]->path());
256 
257     ASSERT_TRUE(log.Recompact(kTestFilename, &err));
258 
259     // The in-memory deps graph should still be valid after recompaction.
260     deps = log.GetDeps(out);
261     ASSERT_TRUE(deps);
262     ASSERT_EQ(1, deps->mtime);
263     ASSERT_EQ(1, deps->node_count);
264     ASSERT_EQ("foo.h", deps->nodes[0]->path());
265     ASSERT_EQ(out, log.nodes()[out->id()]);
266 
267     deps = log.GetDeps(other_out);
268     ASSERT_TRUE(deps);
269     ASSERT_EQ(1, deps->mtime);
270     ASSERT_EQ(2, deps->node_count);
271     ASSERT_EQ("foo.h", deps->nodes[0]->path());
272     ASSERT_EQ("baz.h", deps->nodes[1]->path());
273     ASSERT_EQ(other_out, log.nodes()[other_out->id()]);
274 
275     // The file should have shrunk a bit for the smaller deps.
276     struct stat st;
277     ASSERT_EQ(0, stat(kTestFilename, &st));
278     file_size_3 = (int)st.st_size;
279     ASSERT_LT(file_size_3, file_size_2);
280   }
281 
282   // Now reload the file and recompact with an empty manifest. The previous
283   // entries should be removed.
284   {
285     State state;
286     // Intentionally not parsing kManifest here.
287     DepsLog log;
288     string err;
289     ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
290 
291     Node* out = state.GetNode("out.o", 0);
292     DepsLog::Deps* deps = log.GetDeps(out);
293     ASSERT_TRUE(deps);
294     ASSERT_EQ(1, deps->mtime);
295     ASSERT_EQ(1, deps->node_count);
296     ASSERT_EQ("foo.h", deps->nodes[0]->path());
297 
298     Node* other_out = state.GetNode("other_out.o", 0);
299     deps = log.GetDeps(other_out);
300     ASSERT_TRUE(deps);
301     ASSERT_EQ(1, deps->mtime);
302     ASSERT_EQ(2, deps->node_count);
303     ASSERT_EQ("foo.h", deps->nodes[0]->path());
304     ASSERT_EQ("baz.h", deps->nodes[1]->path());
305 
306     ASSERT_TRUE(log.Recompact(kTestFilename, &err));
307 
308     // The previous entries should have been removed.
309     deps = log.GetDeps(out);
310     ASSERT_FALSE(deps);
311 
312     deps = log.GetDeps(other_out);
313     ASSERT_FALSE(deps);
314 
315     // The .h files pulled in via deps should no longer have ids either.
316     ASSERT_EQ(-1, state.LookupNode("foo.h")->id());
317     ASSERT_EQ(-1, state.LookupNode("baz.h")->id());
318 
319     // The file should have shrunk more.
320     struct stat st;
321     ASSERT_EQ(0, stat(kTestFilename, &st));
322     int file_size_4 = (int)st.st_size;
323     ASSERT_LT(file_size_4, file_size_3);
324   }
325 }
326 
327 // Verify that invalid file headers cause a new build.
TEST_F(DepsLogTest,InvalidHeader)328 TEST_F(DepsLogTest, InvalidHeader) {
329   const char *kInvalidHeaders[] = {
330     "",                              // Empty file.
331     "# ninjad",                      // Truncated first line.
332     "# ninjadeps\n",                 // No version int.
333     "# ninjadeps\n\001\002",         // Truncated version int.
334     "# ninjadeps\n\001\002\003\004"  // Invalid version int.
335   };
336   for (size_t i = 0; i < sizeof(kInvalidHeaders) / sizeof(kInvalidHeaders[0]);
337        ++i) {
338     FILE* deps_log = fopen(kTestFilename, "wb");
339     ASSERT_TRUE(deps_log != NULL);
340     ASSERT_EQ(
341         strlen(kInvalidHeaders[i]),
342         fwrite(kInvalidHeaders[i], 1, strlen(kInvalidHeaders[i]), deps_log));
343     ASSERT_EQ(0 ,fclose(deps_log));
344 
345     string err;
346     DepsLog log;
347     State state;
348     ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
349     EXPECT_EQ("bad deps log signature or version; starting over", err);
350   }
351 }
352 
353 // Simulate what happens when loading a truncated log file.
TEST_F(DepsLogTest,Truncated)354 TEST_F(DepsLogTest, Truncated) {
355   // Create a file with some entries.
356   {
357     State state;
358     DepsLog log;
359     string err;
360     EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
361     ASSERT_EQ("", err);
362 
363     vector<Node*> deps;
364     deps.push_back(state.GetNode("foo.h", 0));
365     deps.push_back(state.GetNode("bar.h", 0));
366     log.RecordDeps(state.GetNode("out.o", 0), 1, deps);
367 
368     deps.clear();
369     deps.push_back(state.GetNode("foo.h", 0));
370     deps.push_back(state.GetNode("bar2.h", 0));
371     log.RecordDeps(state.GetNode("out2.o", 0), 2, deps);
372 
373     log.Close();
374   }
375 
376   // Get the file size.
377   struct stat st;
378   ASSERT_EQ(0, stat(kTestFilename, &st));
379 
380   // Try reloading at truncated sizes.
381   // Track how many nodes/deps were found; they should decrease with
382   // smaller sizes.
383   int node_count = 5;
384   int deps_count = 2;
385   for (int size = (int)st.st_size; size > 0; --size) {
386     string err;
387     ASSERT_TRUE(Truncate(kTestFilename, size, &err));
388 
389     State state;
390     DepsLog log;
391     EXPECT_TRUE(log.Load(kTestFilename, &state, &err));
392     if (!err.empty()) {
393       // At some point the log will be so short as to be unparseable.
394       break;
395     }
396 
397     ASSERT_GE(node_count, (int)log.nodes().size());
398     node_count = log.nodes().size();
399 
400     // Count how many non-NULL deps entries there are.
401     int new_deps_count = 0;
402     for (vector<DepsLog::Deps*>::const_iterator i = log.deps().begin();
403          i != log.deps().end(); ++i) {
404       if (*i)
405         ++new_deps_count;
406     }
407     ASSERT_GE(deps_count, new_deps_count);
408     deps_count = new_deps_count;
409   }
410 }
411 
412 // Run the truncation-recovery logic.
TEST_F(DepsLogTest,TruncatedRecovery)413 TEST_F(DepsLogTest, TruncatedRecovery) {
414   // Create a file with some entries.
415   {
416     State state;
417     DepsLog log;
418     string err;
419     EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
420     ASSERT_EQ("", err);
421 
422     vector<Node*> deps;
423     deps.push_back(state.GetNode("foo.h", 0));
424     deps.push_back(state.GetNode("bar.h", 0));
425     log.RecordDeps(state.GetNode("out.o", 0), 1, deps);
426 
427     deps.clear();
428     deps.push_back(state.GetNode("foo.h", 0));
429     deps.push_back(state.GetNode("bar2.h", 0));
430     log.RecordDeps(state.GetNode("out2.o", 0), 2, deps);
431 
432     log.Close();
433   }
434 
435   // Shorten the file, corrupting the last record.
436   {
437     struct stat st;
438     ASSERT_EQ(0, stat(kTestFilename, &st));
439     string err;
440     ASSERT_TRUE(Truncate(kTestFilename, st.st_size - 2, &err));
441   }
442 
443   // Load the file again, add an entry.
444   {
445     State state;
446     DepsLog log;
447     string err;
448     EXPECT_TRUE(log.Load(kTestFilename, &state, &err));
449     ASSERT_EQ("premature end of file; recovering", err);
450     err.clear();
451 
452     // The truncated entry should've been discarded.
453     EXPECT_EQ(NULL, log.GetDeps(state.GetNode("out2.o", 0)));
454 
455     EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
456     ASSERT_EQ("", err);
457 
458     // Add a new entry.
459     vector<Node*> deps;
460     deps.push_back(state.GetNode("foo.h", 0));
461     deps.push_back(state.GetNode("bar2.h", 0));
462     log.RecordDeps(state.GetNode("out2.o", 0), 3, deps);
463 
464     log.Close();
465   }
466 
467   // Load the file a third time to verify appending after a mangled
468   // entry doesn't break things.
469   {
470     State state;
471     DepsLog log;
472     string err;
473     EXPECT_TRUE(log.Load(kTestFilename, &state, &err));
474 
475     // The truncated entry should exist.
476     DepsLog::Deps* deps = log.GetDeps(state.GetNode("out2.o", 0));
477     ASSERT_TRUE(deps);
478   }
479 }
480 
481 }  // anonymous namespace
482