1 /*
2 ** Copyright (c) 2014 D. Richard Hipp
3 **
4 ** This program is free software; you can redistribute it and/or
5 ** modify it under the terms of the Simplified BSD License (also
6 ** known as the "2-Clause License" or "FreeBSD License".)
7 **
8 ** This program is distributed in the hope that it will be useful,
9 ** but without any warranty; without even the implied warranty of
10 ** merchantability or fitness for a particular purpose.
11 **
12 ** Author contact information:
13 ** drh@sqlite.org
14 **
15 *******************************************************************************
16 **
17 ** This file implements a cache for expense operations such as
18 ** /zip and /tarball.
19 */
20 #include "config.h"
21 #include <sqlite3.h>
22 #include "cache.h"
23
24 /*
25 ** Construct the name of the repository cache file
26 */
cacheName(void)27 static char *cacheName(void){
28 int i;
29 int n;
30
31 if( g.zRepositoryName==0 ) return 0;
32 n = (int)strlen(g.zRepositoryName);
33 for(i=n-1; i>=0; i--){
34 if( g.zRepositoryName[i]=='/' ){ i = n; break; }
35 if( g.zRepositoryName[i]=='.' ) break;
36 }
37 if( i<0 ) i = n;
38 return mprintf("%.*s.cache", i, g.zRepositoryName);
39 }
40
41 /*
42 ** Attempt to open the cache database, if such a database exists.
43 ** Make sure the cache table exists within that database.
44 */
cacheOpen(int bForce)45 static sqlite3 *cacheOpen(int bForce){
46 char *zDbName;
47 sqlite3 *db = 0;
48 int rc;
49 i64 sz;
50
51 zDbName = cacheName();
52 if( zDbName==0 ) return 0;
53 if( bForce==0 ){
54 sz = file_size(zDbName, ExtFILE);
55 if( sz<=0 ){
56 fossil_free(zDbName);
57 return 0;
58 }
59 }
60 rc = sqlite3_open(zDbName, &db);
61 fossil_free(zDbName);
62 if( rc ){
63 sqlite3_close(db);
64 return 0;
65 }
66 sqlite3_busy_timeout(db, 5000);
67 if( sqlite3_table_column_metadata(db,0,"blob","key",0,0,0,0,0)!=SQLITE_OK ){
68 rc = sqlite3_exec(db,
69 "PRAGMA page_size=8192;"
70 "CREATE TABLE IF NOT EXISTS blob(id INTEGER PRIMARY KEY, data BLOB);"
71 "CREATE TABLE IF NOT EXISTS cache("
72 "key TEXT PRIMARY KEY," /* Key used to access the cache */
73 "id INT REFERENCES blob," /* The cache content */
74 "sz INT," /* Size of content in bytes */
75 "tm INT," /* Last access time (unix timestampe) */
76 "nref INT" /* Number of uses */
77 ");"
78 "CREATE TRIGGER IF NOT EXISTS cacheDel AFTER DELETE ON cache BEGIN"
79 " DELETE FROM blob WHERE id=OLD.id;"
80 "END;",
81 0, 0, 0
82 );
83 if( rc!=SQLITE_OK ){
84 sqlite3_close(db);
85 return 0;
86 }
87 }
88 return db;
89 }
90
91 /*
92 ** Attempt to construct a prepared statement for the cache database.
93 */
cacheStmt(sqlite3 * db,const char * zSql)94 static sqlite3_stmt *cacheStmt(sqlite3 *db, const char *zSql){
95 sqlite3_stmt *pStmt = 0;
96 int rc;
97
98 rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
99 if( rc ){
100 sqlite3_finalize(pStmt);
101 pStmt = 0;
102 }
103 return pStmt;
104 }
105
106 /*
107 ** This routine implements an SQL function that renders a large integer
108 ** compactly: ex: 12.3MB
109 */
cache_sizename(sqlite3_context * context,int argc,sqlite3_value ** argv)110 static void cache_sizename(
111 sqlite3_context *context,
112 int argc,
113 sqlite3_value **argv
114 ){
115 char zBuf[30];
116 double v, x;
117 assert( argc==1 );
118 v = sqlite3_value_double(argv[0]);
119 x = v<0.0 ? -v : v;
120 if( x>=1e9 ){
121 sqlite3_snprintf(sizeof(zBuf), zBuf, "%.1fGB", v/1e9);
122 }else if( x>=1e6 ){
123 sqlite3_snprintf(sizeof(zBuf), zBuf, "%.1fMB", v/1e6);
124 }else if( x>=1e3 ){
125 sqlite3_snprintf(sizeof(zBuf), zBuf, "%.1fKB", v/1e3);
126 }else{
127 sqlite3_snprintf(sizeof(zBuf), zBuf, "%gB", v);
128 }
129 sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT);
130 }
131
132 /*
133 ** Register the sizename() SQL function with the SQLite database
134 ** connection.
135 */
cache_register_sizename(sqlite3 * db)136 static void cache_register_sizename(sqlite3 *db){
137 sqlite3_create_function(db, "sizename", 1, SQLITE_UTF8, 0,
138 cache_sizename, 0, 0);
139 }
140
141 /*
142 ** Attempt to write pContent into the cache. If the cache file does
143 ** not exist, then this routine is a no-op. Older cache entries might
144 ** be deleted.
145 */
cache_write(Blob * pContent,const char * zKey)146 void cache_write(Blob *pContent, const char *zKey){
147 sqlite3 *db;
148 sqlite3_stmt *pStmt;
149 int rc = 0;
150 int nKeep;
151
152 db = cacheOpen(0);
153 if( db==0 ) return;
154 sqlite3_busy_timeout(db, 10000);
155 sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, 0);
156 pStmt = cacheStmt(db, "INSERT INTO blob(data) VALUES(?1)");
157 if( pStmt==0 ) goto cache_write_end;
158 sqlite3_bind_blob(pStmt, 1, blob_buffer(pContent), blob_size(pContent),
159 SQLITE_STATIC);
160 if( sqlite3_step(pStmt)!=SQLITE_DONE ) goto cache_write_end;
161 sqlite3_finalize(pStmt);
162 pStmt = cacheStmt(db,
163 "INSERT OR IGNORE INTO cache(key,sz,tm,nref,id)"
164 "VALUES(?1,?2,strftime('%s','now'),1,?3)"
165 );
166 if( pStmt==0 ) goto cache_write_end;
167 sqlite3_bind_text(pStmt, 1, zKey, -1, SQLITE_STATIC);
168 sqlite3_bind_int(pStmt, 2, blob_size(pContent));
169 sqlite3_bind_int(pStmt, 3, sqlite3_last_insert_rowid(db));
170 if( sqlite3_step(pStmt)!=SQLITE_DONE) goto cache_write_end;
171 rc = sqlite3_changes(db);
172
173 /* If the write was successful, truncate the cache to keep at most
174 ** max-cache-entry entries in the cache.
175 **
176 ** The cache entry replacement algorithm is approximately LRU
177 ** (least recently used). However, each access of an entry buys
178 ** that entry an extra hour of grace, so that more commonly accessed
179 ** entries are held in cache longer. The extra "grace" allotted to
180 ** an entry is limited to 2 days worth.
181 */
182 if( rc ){
183 nKeep = db_get_int("max-cache-entry",10);
184 sqlite3_finalize(pStmt);
185 pStmt = cacheStmt(db,
186 "DELETE FROM cache WHERE rowid IN ("
187 "SELECT rowid FROM cache"
188 " ORDER BY (tm + 3600*min(nRef,48)) DESC"
189 " LIMIT -1 OFFSET ?1)");
190 if( pStmt ){
191 sqlite3_bind_int(pStmt, 1, nKeep);
192 sqlite3_step(pStmt);
193 }
194 }
195
196 cache_write_end:
197 sqlite3_finalize(pStmt);
198 sqlite3_exec(db, rc ? "COMMIT" : "ROLLBACK", 0, 0, 0);
199 sqlite3_close(db);
200 }
201
202 /*
203 ** SETTING: max-cache-entry width=10 default=10
204 **
205 ** This is the maximum number of entries to allow in the web-cache
206 ** for tarballs, ZIP-archives, and SQL-archives.
207 */
208
209 /*
210 ** Attempt to read content out of the cache with the given zKey. Return
211 ** non-zero on success and zero if unable to locate the content.
212 **
213 ** Possible reasons for returning zero:
214 ** (1) This server does not implement a cache
215 ** (2) The requested element is not in the cache
216 */
cache_read(Blob * pContent,const char * zKey)217 int cache_read(Blob *pContent, const char *zKey){
218 sqlite3 *db;
219 sqlite3_stmt *pStmt;
220 int rc = 0;
221
222 db = cacheOpen(0);
223 if( db==0 ) return 0;
224 sqlite3_busy_timeout(db, 10000);
225 sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, 0);
226 pStmt = cacheStmt(db,
227 "SELECT blob.data FROM cache, blob"
228 " WHERE cache.key=?1 AND cache.id=blob.id");
229 if( pStmt==0 ) goto cache_read_done;
230 sqlite3_bind_text(pStmt, 1, zKey, -1, SQLITE_STATIC);
231 if( sqlite3_step(pStmt)==SQLITE_ROW ){
232 blob_append(pContent, sqlite3_column_blob(pStmt, 0),
233 sqlite3_column_bytes(pStmt, 0));
234 rc = 1;
235 sqlite3_reset(pStmt);
236 pStmt = cacheStmt(db,
237 "UPDATE cache SET nref=nref+1, tm=strftime('%s','now')"
238 " WHERE key=?1");
239 if( pStmt ){
240 sqlite3_bind_text(pStmt, 1, zKey, -1, SQLITE_STATIC);
241 sqlite3_step(pStmt);
242 }
243 }
244 sqlite3_finalize(pStmt);
245 cache_read_done:
246 sqlite3_exec(db, "COMMIT", 0, 0, 0);
247 sqlite3_close(db);
248 return rc;
249 }
250
251 /*
252 ** Create a cache database for the current repository if no such
253 ** database already exists.
254 */
cache_initialize(void)255 void cache_initialize(void){
256 sqlite3_close(cacheOpen(1));
257 }
258
259 /*
260 ** COMMAND: cache*
261 **
262 ** Usage: %fossil cache SUBCOMMAND
263 **
264 ** Manage the cache used for potentially expensive web pages such as
265 ** /zip and /tarball. SUBCOMMAND can be:
266 **
267 ** clear Remove all entries from the cache.
268 **
269 ** init Create the cache file if it does not already exist.
270 **
271 ** list|ls List the keys and content sizes and other stats for
272 ** all entries currently in the cache.
273 **
274 ** size ?N? Query or set the maximum number of entries in the cache.
275 **
276 ** status Show a summary of the cache status.
277 **
278 ** The cache is stored in a file that is distinct from the repository
279 ** but that is held in the same directory as the repository. The cache
280 ** file can be deleted in order to completely disable the cache.
281 */
cache_cmd(void)282 void cache_cmd(void){
283 const char *zCmd;
284 int nCmd;
285 sqlite3 *db;
286 sqlite3_stmt *pStmt;
287
288 db_find_and_open_repository(0,0);
289 zCmd = g.argc>=3 ? g.argv[2] : "";
290 nCmd = (int)strlen(zCmd);
291 if( nCmd<=1 ){
292 fossil_fatal("Usage: %s cache SUBCOMMAND", g.argv[0]);
293 }
294 if( strncmp(zCmd, "init", nCmd)==0 ){
295 db = cacheOpen(0);
296 sqlite3_close(db);
297 if( db ){
298 fossil_print("cache already exists in file %z\n", cacheName());
299 }else{
300 db = cacheOpen(1);
301 sqlite3_close(db);
302 if( db ){
303 fossil_print("cache created in file %z\n", cacheName());
304 }else{
305 fossil_fatal("unable to create cache file %z", cacheName());
306 }
307 }
308 }else if( strncmp(zCmd, "clear", nCmd)==0 ){
309 db = cacheOpen(0);
310 if( db ){
311 sqlite3_exec(db, "DELETE FROM cache; DELETE FROM blob; VACUUM;",0,0,0);
312 sqlite3_close(db);
313 fossil_print("cache cleared\n");
314 }else{
315 fossil_print("nothing to clear; cache does not exist\n");
316 }
317 }else if( strncmp(zCmd, "list", nCmd)==0
318 || strncmp(zCmd, "ls", nCmd)==0
319 || strncmp(zCmd, "status", nCmd)==0
320 ){
321 db = cacheOpen(0);
322 if( db==0 ){
323 fossil_print("cache does not exist\n");
324 }else{
325 int nEntry = 0;
326 char *zDbName = cacheName();
327 cache_register_sizename(db);
328 pStmt = cacheStmt(db,
329 "SELECT key, sizename(sz), nRef, datetime(tm,'unixepoch')"
330 " FROM cache"
331 " ORDER BY tm DESC"
332 );
333 if( pStmt ){
334 while( sqlite3_step(pStmt)==SQLITE_ROW ){
335 if( zCmd[0]=='l' ){
336 fossil_print("%s %4d %8s %s\n",
337 sqlite3_column_text(pStmt, 3),
338 sqlite3_column_int(pStmt, 2),
339 sqlite3_column_text(pStmt, 1),
340 sqlite3_column_text(pStmt, 0));
341 }
342 nEntry++;
343 }
344 sqlite3_finalize(pStmt);
345 }
346 sqlite3_close(db);
347 fossil_print(
348 "Filename: %s\n"
349 "Entries: %d\n"
350 "max-cache-entry: %d\n"
351 "Cache-file Size: %,lld\n",
352 zDbName,
353 nEntry,
354 db_get_int("max-cache-entry",10),
355 file_size(zDbName, ExtFILE)
356 );
357 fossil_free(zDbName);
358 }
359 }else if( strncmp(zCmd, "size", nCmd)==0 ){
360 if( g.argc>=4 ){
361 int n = atoi(g.argv[3]);
362 if( n>=5 ) db_set_int("max-cache-entry",n,0);
363 }
364 fossil_print("max-cache-entry: %d\n", db_get_int("max-cache-entry",10));
365 }else{
366 fossil_fatal("Unknown subcommand \"%s\"."
367 " Should be one of: clear init list size status", zCmd);
368 }
369 }
370
371 /*
372 ** Given a cache key, find the check-in hash and return it as a separate
373 ** string. The returned string is obtained from fossil_malloc() and must
374 ** be freed by the caller.
375 **
376 ** Return NULL if not found.
377 **
378 ** The key is usually in a format like these:
379 **
380 ** /tarball/HASH/NAME
381 ** /zip/HASH/NAME
382 ** /sqlar/HASH/NAME
383 */
cache_hash_of_key(const char * zKey)384 static char *cache_hash_of_key(const char *zKey){
385 int i;
386 if( zKey==0 ) return 0;
387 if( zKey[0]!='/' ) return 0;
388 zKey++;
389 while( zKey[0] && zKey[0]!='/' ) zKey++;
390 if( zKey[0]==0 ) return 0;
391 zKey++;
392 for(i=0; zKey[i] && zKey[i]!='/'; i++){}
393 if( !validate16(zKey, i) ) return 0;
394 return fossil_strndup(zKey, i);
395 }
396
397
398 /*
399 ** WEBPAGE: cachestat
400 **
401 ** Show information about the webpage cache. Requires Setup privilege.
402 */
cache_page(void)403 void cache_page(void){
404 sqlite3 *db;
405 sqlite3_stmt *pStmt;
406 char zBuf[100];
407
408 login_check_credentials();
409 if( !g.perm.Setup ){ login_needed(0); return; }
410 style_set_current_feature("cache");
411 style_header("Web Cache Status");
412 db = cacheOpen(0);
413 if( db==0 ){
414 @ The web-page cache is disabled for this repository
415 }else{
416 char *zDbName = cacheName();
417 cache_register_sizename(db);
418 pStmt = cacheStmt(db,
419 "SELECT key, sz, nRef, datetime(tm,'unixepoch')"
420 " FROM cache"
421 " ORDER BY (tm + 3600*min(nRef,48)) DESC"
422 );
423 if( pStmt ){
424 @ <ol>
425 while( sqlite3_step(pStmt)==SQLITE_ROW ){
426 const unsigned char *zName = sqlite3_column_text(pStmt,0);
427 char *zHash = cache_hash_of_key((const char*)zName);
428 @ <li><p>%z(href("%R/cacheget?key=%T",zName))%h(zName)</a><br />
429 @ size: %,lld(sqlite3_column_int64(pStmt,1))
430 @ hit-count: %d(sqlite3_column_int(pStmt,2))
431 @ last-access: %s(sqlite3_column_text(pStmt,3)) \
432 if( zHash ){
433 @ %z(href("%R/timeline?c=%S",zHash))check-in</a>\
434 fossil_free(zHash);
435 }
436 @ </p></li>
437
438 }
439 sqlite3_finalize(pStmt);
440 @ </ol>
441 }
442 zDbName = cacheName();
443 bigSizeName(sizeof(zBuf), zBuf, file_size(zDbName, ExtFILE));
444 @ <p>
445 @ cache-file name: %h(zDbName)<br>
446 @ cache-file size: %s(zBuf)<br>
447 @ max-cache-entry: %d(db_get_int("max-cache-entry",10))
448 @ </p>
449 @ <p>
450 @ Use the "<a href="%R/help?cmd=cache">fossil cache</a>" command
451 @ on the command-line to create and configure the web-cache.
452 @ </p>
453 fossil_free(zDbName);
454 sqlite3_close(db);
455 }
456 style_finish_page();
457 }
458
459 /*
460 ** WEBPAGE: cacheget
461 **
462 ** Usage: /cacheget?key=KEY
463 **
464 ** Download a single entry for the cache, identified by KEY.
465 ** This page is normally a hyperlink from the /cachestat page.
466 ** Requires Admin privilege.
467 */
468 void cache_getpage(void){
469 const char *zKey;
470 Blob content;
471
472 login_check_credentials();
473 if( !g.perm.Setup ){ login_needed(0); return; }
474 zKey = PD("key","");
475 blob_zero(&content);
476 if( cache_read(&content, zKey)==0 ){
477 style_set_current_feature("cache");
478 style_header("Cache Download Error");
479 @ The cache does not contain any entry with this key: "%h(zKey)"
480 style_finish_page();
481 return;
482 }
483 cgi_set_content(&content);
484 cgi_set_content_type("application/x-compressed");
485 }
486