1 /*
2 * This file has been donated to Jam.
3 */
4
5 /*
6 * Craig W. McPheeters, Alias|Wavefront.
7 *
8 * hcache.c hcache.h - handle caching of #includes in source files.
9 *
10 * Create a cache of files scanned for headers. When starting jam, look for the
11 * cache file and load it if present. When finished the binding phase, create a
12 * new header cache. The cache contains files, their timestamps and the header
13 * files found in their scan. During the binding phase of jam, look in the
14 * header cache first for the headers contained in a file. If the cache is
15 * present and valid, use its contents. This results in dramatic speedups with
16 * large projects (e.g. 3min -> 1min startup for one project.)
17 *
18 * External routines:
19 * hcache_init() - read and parse the local .jamdeps file.
20 * hcache_done() - write a new .jamdeps file.
21 * hcache() - return list of headers on target. Use cache or do a scan.
22 *
23 * The dependency file format is an ASCII file with 1 line per target. Each line
24 * has the following fields:
25 * @boundname@ timestamp_sec timestamp_nsec @file@ @file@ @file@ ...
26 */
27
28 #include "config.h"
29
30 #ifdef OPT_HEADER_CACHE_EXT
31
32 #include "jam.h"
33 #include "hcache.h"
34
35 #include "hash.h"
36 #include "headers.h"
37 #include "lists.h"
38 #include "modules.h"
39 #include "object.h"
40 #include "parse.h"
41 #include "regexp.h"
42 #include "rules.h"
43 #include "search.h"
44 #include "timestamp.h"
45 #include "variable.h"
46 #include "output.h"
47
48 typedef struct hcachedata HCACHEDATA ;
49
50 struct hcachedata
51 {
52 OBJECT * boundname;
53 timestamp time;
54 LIST * includes;
55 LIST * hdrscan; /* the HDRSCAN value for this target */
56 int age; /* if too old, we will remove it from cache */
57 HCACHEDATA * next;
58 };
59
60
61 static struct hash * hcachehash = 0;
62 static HCACHEDATA * hcachelist = 0;
63
64 static int queries = 0;
65 static int hits = 0;
66
67 #define CACHE_FILE_VERSION "version 5"
68 #define CACHE_RECORD_HEADER "header"
69 #define CACHE_RECORD_END "end"
70
71
72 /*
73 * Return the name of the header cache file. May return NULL.
74 *
75 * The user sets this by setting the HCACHEFILE variable in a Jamfile. We cache
76 * the result so the user can not change the cache file during header scanning.
77 */
78
cache_name(void)79 static const char * cache_name( void )
80 {
81 static OBJECT * name = 0;
82 if ( !name )
83 {
84 LIST * const hcachevar = var_get( root_module(), constant_HCACHEFILE );
85
86 if ( !list_empty( hcachevar ) )
87 {
88 TARGET * const t = bindtarget( list_front( hcachevar ) );
89
90 pushsettings( root_module(), t->settings );
91 /* Do not expect the cache file to be generated, so pass 0 as the
92 * third argument to search. Expect the location to be specified via
93 * LOCATE, so pass 0 as the fourth argument.
94 */
95 object_free( t->boundname );
96 t->boundname = search( t->name, &t->time, 0, 0 );
97 popsettings( root_module(), t->settings );
98
99 name = object_copy( t->boundname );
100 }
101 }
102 return name ? object_str( name ) : 0;
103 }
104
105
106 /*
107 * Return the maximum age a cache entry can have before it is purged from the
108 * cache.
109 */
110
cache_maxage(void)111 static int cache_maxage( void )
112 {
113 int age = 100;
114 LIST * const var = var_get( root_module(), constant_HCACHEMAXAGE );
115 if ( !list_empty( var ) )
116 {
117 age = atoi( object_str( list_front( var ) ) );
118 if ( age < 0 )
119 age = 0;
120 }
121 return age;
122 }
123
124
125 /*
126 * Read a netstring. The caveat is that the string can not contain ASCII 0. The
127 * returned value is as returned by object_new().
128 */
129
read_netstring(FILE * f)130 OBJECT * read_netstring( FILE * f )
131 {
132 unsigned long len;
133 static char * buf = NULL;
134 static unsigned long buf_len = 0;
135
136 if ( fscanf( f, " %9lu", &len ) != 1 )
137 return NULL;
138 if ( fgetc( f ) != (int)'\t' )
139 return NULL;
140
141 if ( len > 1024 * 64 )
142 return NULL; /* sanity check */
143
144 if ( len > buf_len )
145 {
146 unsigned long new_len = buf_len * 2;
147 if ( new_len < len )
148 new_len = len;
149 buf = (char *)BJAM_REALLOC( buf, new_len + 1 );
150 if ( buf )
151 buf_len = new_len;
152 }
153
154 if ( !buf )
155 return NULL;
156
157 if ( fread( buf, 1, len, f ) != len )
158 return NULL;
159 if ( fgetc( f ) != (int)'\n' )
160 return NULL;
161
162 buf[ len ] = 0;
163 return object_new( buf );
164 }
165
166
167 /*
168 * Write a netstring.
169 */
170
write_netstring(FILE * f,char const * s)171 void write_netstring( FILE * f, char const * s )
172 {
173 if ( !s )
174 s = "";
175 fprintf( f, "%lu\t%s\n", (long unsigned)strlen( s ), s );
176 }
177
178
hcache_init()179 void hcache_init()
180 {
181 FILE * f;
182 OBJECT * version = 0;
183 int header_count = 0;
184 const char * hcachename;
185
186 if ( hcachehash )
187 return;
188
189 hcachehash = hashinit( sizeof( HCACHEDATA ), "hcache" );
190
191 if ( !( hcachename = cache_name() ) )
192 return;
193
194 if ( !( f = fopen( hcachename, "rb" ) ) )
195 return;
196
197 version = read_netstring( f );
198
199 if ( !version || strcmp( object_str( version ), CACHE_FILE_VERSION ) )
200 goto bail;
201
202 while ( 1 )
203 {
204 HCACHEDATA cachedata;
205 HCACHEDATA * c;
206 OBJECT * record_type = 0;
207 OBJECT * time_secs_str = 0;
208 OBJECT * time_nsecs_str = 0;
209 OBJECT * age_str = 0;
210 OBJECT * includes_count_str = 0;
211 OBJECT * hdrscan_count_str = 0;
212 int i;
213 int count;
214 LIST * l;
215 int found;
216
217 cachedata.boundname = 0;
218 cachedata.includes = 0;
219 cachedata.hdrscan = 0;
220
221 record_type = read_netstring( f );
222 if ( !record_type )
223 {
224 err_printf( "invalid %s\n", hcachename );
225 goto cleanup;
226 }
227 if ( !strcmp( object_str( record_type ), CACHE_RECORD_END ) )
228 {
229 object_free( record_type );
230 break;
231 }
232 if ( strcmp( object_str( record_type ), CACHE_RECORD_HEADER ) )
233 {
234 err_printf( "invalid %s with record separator <%s>\n",
235 hcachename, record_type ? object_str( record_type ) : "<null>" );
236 goto cleanup;
237 }
238
239 cachedata.boundname = read_netstring( f );
240 time_secs_str = read_netstring( f );
241 time_nsecs_str = read_netstring( f );
242 age_str = read_netstring( f );
243 includes_count_str = read_netstring( f );
244
245 if ( !cachedata.boundname || !time_secs_str || !time_nsecs_str ||
246 !age_str || !includes_count_str )
247 {
248 err_printf( "invalid %s\n", hcachename );
249 goto cleanup;
250 }
251
252 timestamp_init( &cachedata.time, atoi( object_str( time_secs_str ) ),
253 atoi( object_str( time_nsecs_str ) ) );
254 cachedata.age = atoi( object_str( age_str ) ) + 1;
255
256 count = atoi( object_str( includes_count_str ) );
257 for ( l = L0, i = 0; i < count; ++i )
258 {
259 OBJECT * const s = read_netstring( f );
260 if ( !s )
261 {
262 err_printf( "invalid %s\n", hcachename );
263 list_free( l );
264 goto cleanup;
265 }
266 l = list_push_back( l, s );
267 }
268 cachedata.includes = l;
269
270 hdrscan_count_str = read_netstring( f );
271 if ( !hdrscan_count_str )
272 {
273 err_printf( "invalid %s\n", hcachename );
274 goto cleanup;
275 }
276
277 count = atoi( object_str( hdrscan_count_str ) );
278 for ( l = L0, i = 0; i < count; ++i )
279 {
280 OBJECT * const s = read_netstring( f );
281 if ( !s )
282 {
283 err_printf( "invalid %s\n", hcachename );
284 list_free( l );
285 goto cleanup;
286 }
287 l = list_push_back( l, s );
288 }
289 cachedata.hdrscan = l;
290
291 c = (HCACHEDATA *)hash_insert( hcachehash, cachedata.boundname, &found )
292 ;
293 if ( !found )
294 {
295 c->boundname = cachedata.boundname;
296 c->includes = cachedata.includes;
297 c->hdrscan = cachedata.hdrscan;
298 c->age = cachedata.age;
299 timestamp_copy( &c->time, &cachedata.time );
300 }
301 else
302 {
303 err_printf( "can not insert header cache item, bailing on %s"
304 "\n", hcachename );
305 goto cleanup;
306 }
307
308 c->next = hcachelist;
309 hcachelist = c;
310
311 ++header_count;
312
313 object_free( record_type );
314 object_free( time_secs_str );
315 object_free( time_nsecs_str );
316 object_free( age_str );
317 object_free( includes_count_str );
318 object_free( hdrscan_count_str );
319 continue;
320
321 cleanup:
322
323 if ( record_type ) object_free( record_type );
324 if ( time_secs_str ) object_free( time_secs_str );
325 if ( time_nsecs_str ) object_free( time_nsecs_str );
326 if ( age_str ) object_free( age_str );
327 if ( includes_count_str ) object_free( includes_count_str );
328 if ( hdrscan_count_str ) object_free( hdrscan_count_str );
329
330 if ( cachedata.boundname ) object_free( cachedata.boundname );
331 if ( cachedata.includes ) list_free( cachedata.includes );
332 if ( cachedata.hdrscan ) list_free( cachedata.hdrscan );
333
334 goto bail;
335 }
336
337 if ( DEBUG_HEADER )
338 out_printf( "hcache read from file %s\n", hcachename );
339
340 bail:
341 if ( version )
342 object_free( version );
343 fclose( f );
344 }
345
346
hcache_done()347 void hcache_done()
348 {
349 FILE * f;
350 HCACHEDATA * c;
351 int header_count = 0;
352 const char * hcachename;
353 int maxage;
354
355 if ( !hcachehash )
356 return;
357
358 if ( !( hcachename = cache_name() ) )
359 goto cleanup;
360
361 if ( !( f = fopen( hcachename, "wb" ) ) )
362 goto cleanup;
363
364 maxage = cache_maxage();
365
366 /* Print out the version. */
367 write_netstring( f, CACHE_FILE_VERSION );
368
369 c = hcachelist;
370 for ( c = hcachelist; c; c = c->next )
371 {
372 LISTITER iter;
373 LISTITER end;
374 char time_secs_str[ 30 ];
375 char time_nsecs_str[ 30 ];
376 char age_str[ 30 ];
377 char includes_count_str[ 30 ];
378 char hdrscan_count_str[ 30 ];
379
380 if ( maxage == 0 )
381 c->age = 0;
382 else if ( c->age > maxage )
383 continue;
384
385 sprintf( includes_count_str, "%lu", (long unsigned)list_length(
386 c->includes ) );
387 sprintf( hdrscan_count_str, "%lu", (long unsigned)list_length(
388 c->hdrscan ) );
389 sprintf( time_secs_str, "%lu", (long unsigned)c->time.secs );
390 sprintf( time_nsecs_str, "%lu", (long unsigned)c->time.nsecs );
391 sprintf( age_str, "%lu", (long unsigned)c->age );
392
393 write_netstring( f, CACHE_RECORD_HEADER );
394 write_netstring( f, object_str( c->boundname ) );
395 write_netstring( f, time_secs_str );
396 write_netstring( f, time_nsecs_str );
397 write_netstring( f, age_str );
398 write_netstring( f, includes_count_str );
399 for ( iter = list_begin( c->includes ), end = list_end( c->includes );
400 iter != end; iter = list_next( iter ) )
401 write_netstring( f, object_str( list_item( iter ) ) );
402 write_netstring( f, hdrscan_count_str );
403 for ( iter = list_begin( c->hdrscan ), end = list_end( c->hdrscan );
404 iter != end; iter = list_next( iter ) )
405 write_netstring( f, object_str( list_item( iter ) ) );
406 fputs( "\n", f );
407 ++header_count;
408 }
409 write_netstring( f, CACHE_RECORD_END );
410
411 if ( DEBUG_HEADER )
412 out_printf( "hcache written to %s. %d dependencies, %.0f%% hit rate\n",
413 hcachename, header_count, queries ? 100.0 * hits / queries : 0 );
414
415 fclose ( f );
416
417 cleanup:
418 for ( c = hcachelist; c; c = c->next )
419 {
420 list_free( c->includes );
421 list_free( c->hdrscan );
422 object_free( c->boundname );
423 }
424
425 hcachelist = 0;
426 if ( hcachehash )
427 hashdone( hcachehash );
428 hcachehash = 0;
429 }
430
431
hcache(TARGET * t,int rec,regexp * re[],LIST * hdrscan)432 LIST * hcache( TARGET * t, int rec, regexp * re[], LIST * hdrscan )
433 {
434 HCACHEDATA * c;
435
436 ++queries;
437
438 if ( ( c = (HCACHEDATA *)hash_find( hcachehash, t->boundname ) ) )
439 {
440 if ( !timestamp_cmp( &c->time, &t->time ) )
441 {
442 LIST * const l1 = hdrscan;
443 LIST * const l2 = c->hdrscan;
444 LISTITER iter1 = list_begin( l1 );
445 LISTITER const end1 = list_end( l1 );
446 LISTITER iter2 = list_begin( l2 );
447 LISTITER const end2 = list_end( l2 );
448 while ( iter1 != end1 && iter2 != end2 )
449 {
450 if ( !object_equal( list_item( iter1 ), list_item( iter2 ) ) )
451 iter1 = end1;
452 else
453 {
454 iter1 = list_next( iter1 );
455 iter2 = list_next( iter2 );
456 }
457 }
458 if ( iter1 != end1 || iter2 != end2 )
459 {
460 if ( DEBUG_HEADER )
461 {
462 out_printf( "HDRSCAN out of date in cache for %s\n",
463 object_str( t->boundname ) );
464 out_printf(" real : ");
465 list_print( hdrscan );
466 out_printf( "\n cached: " );
467 list_print( c->hdrscan );
468 out_printf( "\n" );
469 }
470
471 list_free( c->includes );
472 list_free( c->hdrscan );
473 c->includes = L0;
474 c->hdrscan = L0;
475 }
476 else
477 {
478 if ( DEBUG_HEADER )
479 out_printf( "using header cache for %s\n", object_str(
480 t->boundname ) );
481 c->age = 0;
482 ++hits;
483 return list_copy( c->includes );
484 }
485 }
486 else
487 {
488 if ( DEBUG_HEADER )
489 out_printf ("header cache out of date for %s\n", object_str(
490 t->boundname ) );
491 list_free( c->includes );
492 list_free( c->hdrscan );
493 c->includes = L0;
494 c->hdrscan = L0;
495 }
496 }
497 else
498 {
499 int found;
500 c = (HCACHEDATA *)hash_insert( hcachehash, t->boundname, &found );
501 if ( !found )
502 {
503 c->boundname = object_copy( t->boundname );
504 c->next = hcachelist;
505 hcachelist = c;
506 }
507 }
508
509 /* 'c' points at the cache entry. Its out of date. */
510 {
511 LIST * const l = headers1( L0, t->boundname, rec, re );
512
513 timestamp_copy( &c->time, &t->time );
514 c->age = 0;
515 c->includes = list_copy( l );
516 c->hdrscan = list_copy( hdrscan );
517
518 return l;
519 }
520 }
521
522 #endif /* OPT_HEADER_CACHE_EXT */
523