1 /******************************************************************************
2  *
3  * Project:  MapCache
4  * Purpose:  MapCache tile caching support file: Berkeley DB cache backend
5  * Author:   Thomas Bonfort and the MapServer team.
6  *
7  ******************************************************************************
8  * Copyright (c) 1996-2011 Regents of the University of Minnesota.
9  *
10  * Permission is hereby granted, free of charge, to any person obtaining a
11  * copy of this software and associated documentation files (the "Software"),
12  * to deal in the Software without restriction, including without limitation
13  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14  * and/or sell copies of the Software, and to permit persons to whom the
15  * Software is furnished to do so, subject to the following conditions:
16  *
17  * The above copyright notice and this permission notice shall be included in
18  * all copies of this Software or works derived from this Software.
19  *
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26  * DEALINGS IN THE SOFTWARE.
27  *****************************************************************************/
28 
29 #include "mapcache.h"
30 #ifdef USE_BDB
31 #include <apr_strings.h>
32 #include <apr_reslist.h>
33 #include <apr_file_info.h>
34 #include <apr_hash.h>
35 #include <string.h>
36 #include <errno.h>
37 #include <time.h>
38 #ifdef APR_HAS_THREADS
39 #include <apr_thread_mutex.h>
40 #endif
41 
42 #ifndef _WIN32
43 #include <unistd.h>
44 #endif
45 
46 #include <db.h>
47 
48 #define PAGESIZE 64*1024
49 #define CACHESIZE 1024*1024
50 
51 typedef struct mapcache_cache_bdb mapcache_cache_bdb;
52 struct mapcache_cache_bdb {
53   mapcache_cache cache;
54   char *basedir;
55   char *key_template;
56 };
57 
58 struct bdb_env {
59   DB* db;
60   DB_ENV *env;
61   int readonly;
62   char *errmsg;
63 };
64 
mapcache_bdb_connection_constructor(mapcache_context * ctx,void ** conn_,void * params)65 void mapcache_bdb_connection_constructor(mapcache_context *ctx, void **conn_, void *params)
66 {
67   int ret;
68   int env_flags;
69   int mode;
70   mapcache_cache_bdb *cache = (mapcache_cache_bdb*)params;
71   char *dbfile = malloc(strlen(cache->basedir)+strlen(cache->cache.name)+strlen("/.db")+1);
72   struct bdb_env *benv = calloc(1,sizeof(struct bdb_env));
73   dbfile[0]=0;
74   strcat(strcat(strcat(strcat(dbfile,cache->basedir),"/"),cache->cache.name),".db"); /*yuk yuk*/
75   *conn_ = benv;
76 
77   ret = db_env_create(&benv->env, 0);
78   if(ret) {
79     ctx->set_error(ctx, 500, "bdb cache failure for db_env_create: %s", db_strerror(ret));
80     goto cleanup_error;
81   }
82   ret = benv->env->set_cachesize(benv->env,0,CACHESIZE,1); /* set a larger cache size than default */
83   if(ret) {
84     ctx->set_error(ctx, 500, "bdb cache failure for db->set_cachesize: %s", db_strerror(ret));
85     goto cleanup_error;
86   }
87   env_flags = DB_INIT_CDB|DB_INIT_MPOOL|DB_CREATE;
88   ret = benv->env->open(benv->env,cache->basedir,env_flags,0);
89   if(ret) {
90     ctx->set_error(ctx,500,"bdb cache failure for env->open: %s", db_strerror(ret));
91     goto cleanup_error;
92   }
93 
94   if ((ret = db_create(&benv->db, benv->env, 0)) != 0) {
95     ctx->set_error(ctx,500,"bdb cache failure for db_create: %s", db_strerror(ret));
96     goto cleanup_error;
97   }
98   mode = DB_BTREE;
99   ret = benv->db->set_pagesize(benv->db,PAGESIZE); /* set pagesize to maximum allowed, as tile data is usually pretty large */
100   if(ret) {
101     ctx->set_error(ctx,500,"bdb cache failure for db->set_pagesize: %s", db_strerror(ret));
102     goto cleanup_error;
103   }
104 
105   if ((ret = benv->db->open(benv->db, NULL, dbfile, NULL, mode, DB_CREATE, 0664)) != 0) {
106     ctx->set_error(ctx,500,"bdb cache failure 1 for db->open: %s", db_strerror(ret));
107     goto cleanup_error;
108   }
109 
110   goto cleanup;
111 
112 cleanup_error:
113   free(benv);
114 cleanup:
115   free(dbfile);
116 }
117 
mapcache_bdb_connection_destructor(void * conn_)118 void mapcache_bdb_connection_destructor(void *conn_)
119 {
120   struct bdb_env *benv = (struct bdb_env*)conn_;
121   benv->db->close(benv->db,0);
122   benv->env->close(benv->env,0);
123   free(benv);
124 }
125 
126 
127 
_bdb_get_conn(mapcache_context * ctx,mapcache_cache_bdb * cache,mapcache_tile * tile,int readonly)128 static mapcache_pooled_connection* _bdb_get_conn(mapcache_context *ctx, mapcache_cache_bdb *cache, mapcache_tile* tile, int readonly) {
129   struct bdb_env *benv;
130   mapcache_pooled_connection *pc;
131   char *conn_key = apr_pstrcat(ctx->pool,readonly?"ro_":"rw_",cache->cache.name,NULL);
132   pc = mapcache_connection_pool_get_connection(ctx,conn_key,mapcache_bdb_connection_constructor, mapcache_bdb_connection_destructor, cache);
133   if(GC_HAS_ERROR(ctx)) return NULL;
134   benv = pc->connection;
135   benv->readonly = readonly;
136   return pc;
137 }
138 
_bdb_release_conn(mapcache_context * ctx,mapcache_cache_bdb * cache,mapcache_tile * tile,mapcache_pooled_connection * pc)139 static void _bdb_release_conn(mapcache_context *ctx, mapcache_cache_bdb *cache, mapcache_tile *tile, mapcache_pooled_connection *pc)
140 {
141   if(GC_HAS_ERROR(ctx)) {
142     mapcache_connection_pool_invalidate_connection(ctx, pc);
143   } else {
144     mapcache_connection_pool_release_connection(ctx,pc);
145   }
146 }
147 
_mapcache_cache_bdb_has_tile(mapcache_context * ctx,mapcache_cache * pcache,mapcache_tile * tile)148 static int _mapcache_cache_bdb_has_tile(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile)
149 {
150   int ret;
151   DBT key;
152   mapcache_cache_bdb *cache = (mapcache_cache_bdb*)pcache;
153   char *skey = mapcache_util_get_tile_key(ctx,tile,cache->key_template,NULL,NULL);
154   mapcache_pooled_connection *pc;
155   struct bdb_env *benv;
156   pc = _bdb_get_conn(ctx,cache,tile,1);
157   if(GC_HAS_ERROR(ctx)) return MAPCACHE_FALSE;
158   benv = pc->connection;
159   memset(&key, 0, sizeof(DBT));
160   key.data = skey;
161   key.size = strlen(skey)+1;
162 
163   ret = benv->db->exists(benv->db, NULL, &key, 0);
164 
165   if(ret == 0) {
166     ret = MAPCACHE_TRUE;
167   } else if(ret == DB_NOTFOUND) {
168     ret = MAPCACHE_FALSE;
169   } else {
170     ctx->set_error(ctx,500,"bdb backend failure on tile_exists: %s",db_strerror(ret));
171     ret= MAPCACHE_FALSE;
172   }
173   _bdb_release_conn(ctx,cache,tile,pc);
174   return ret;
175 }
176 
_mapcache_cache_bdb_delete(mapcache_context * ctx,mapcache_cache * pcache,mapcache_tile * tile)177 static void _mapcache_cache_bdb_delete(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile)
178 {
179   DBT key;
180   int ret;
181   mapcache_cache_bdb *cache = (mapcache_cache_bdb*)pcache;
182   char *skey = mapcache_util_get_tile_key(ctx,tile,cache->key_template,NULL,NULL);
183   mapcache_pooled_connection *pc;
184   struct bdb_env *benv;
185   pc = _bdb_get_conn(ctx,cache,tile,0);
186   GC_CHECK_ERROR(ctx);
187   benv = pc->connection;
188   memset(&key, 0, sizeof(DBT));
189   key.data = skey;
190   key.size = strlen(skey)+1;
191   ret = benv->db->del(benv->db, NULL, &key, 0);
192   if(ret && ret != DB_NOTFOUND) {
193     ctx->set_error(ctx,500,"bdb backend failure on tile_delete: %s",db_strerror(ret));
194   } else {
195     ret = benv->db->sync(benv->db,0);
196     if(ret)
197       ctx->set_error(ctx,500,"bdb backend sync failure on tile_delete: %s",db_strerror(ret));
198   }
199   _bdb_release_conn(ctx,cache,tile,pc);
200 }
201 
_mapcache_cache_bdb_get(mapcache_context * ctx,mapcache_cache * pcache,mapcache_tile * tile)202 static int _mapcache_cache_bdb_get(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile)
203 {
204   DBT key,data;
205   int ret;
206   char *skey;
207   mapcache_cache_bdb *cache = (mapcache_cache_bdb*)pcache;
208   mapcache_pooled_connection *pc;
209   struct bdb_env *benv;
210   pc = _bdb_get_conn(ctx,cache,tile,1);
211   if(GC_HAS_ERROR(ctx)) return MAPCACHE_FALSE;
212   benv = pc->connection;
213   skey = mapcache_util_get_tile_key(ctx,tile,cache->key_template,NULL,NULL);
214   memset(&key, 0, sizeof(DBT));
215   memset(&data, 0, sizeof(DBT));
216   data.flags = DB_DBT_MALLOC;
217   key.data = skey;
218   key.size = strlen(skey)+1;
219 
220   ret = benv->db->get(benv->db, NULL, &key, &data, 0);
221 
222 
223   if(ret == 0) {
224     if(((char*)(data.data))[0] == '#') {
225       tile->encoded_data = mapcache_empty_png_decode(ctx,tile->grid_link->grid->tile_sx, tile->grid_link->grid->tile_sy, (unsigned char*)data.data,&tile->nodata);
226     } else {
227       tile->encoded_data = mapcache_buffer_create(0,ctx->pool);
228       tile->encoded_data->buf = data.data;
229       tile->encoded_data->size = data.size-sizeof(apr_time_t);
230       tile->encoded_data->avail = data.size;
231       apr_pool_cleanup_register(ctx->pool, tile->encoded_data->buf,(void*)free, apr_pool_cleanup_null);
232     }
233     tile->mtime = *((apr_time_t*)(((char*)data.data)+data.size-sizeof(apr_time_t)));
234     ret = MAPCACHE_SUCCESS;
235   } else if(ret == DB_NOTFOUND) {
236     ret = MAPCACHE_CACHE_MISS;
237   } else {
238     ctx->set_error(ctx,500,"bdb backend failure on tile_get: %s",db_strerror(ret));
239     ret = MAPCACHE_FAILURE;
240   }
241   _bdb_release_conn(ctx,cache,tile,pc);
242   return ret;
243 }
244 
245 
_mapcache_cache_bdb_set(mapcache_context * ctx,mapcache_cache * pcache,mapcache_tile * tile)246 static void _mapcache_cache_bdb_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile)
247 {
248   DBT key,data;
249   int ret;
250   apr_time_t now;
251   mapcache_cache_bdb *cache = (mapcache_cache_bdb*)pcache;
252   char *skey = mapcache_util_get_tile_key(ctx,tile,cache->key_template,NULL,NULL);
253   mapcache_pooled_connection *pc;
254   struct bdb_env *benv;
255   now = apr_time_now();
256   memset(&key, 0, sizeof(DBT));
257   memset(&data, 0, sizeof(DBT));
258 
259   key.data = skey;
260   key.size = strlen(skey)+1;
261 
262   if(!tile->raw_image) {
263     tile->raw_image = mapcache_imageio_decode(ctx, tile->encoded_data);
264     GC_CHECK_ERROR(ctx);
265   }
266 
267   if(tile->raw_image->h==256 && tile->raw_image->w==256 && mapcache_image_blank_color(tile->raw_image) != MAPCACHE_FALSE) {
268     data.size = 5+sizeof(apr_time_t);
269     data.data = apr_palloc(ctx->pool,data.size);
270     (((char*)data.data)[0])='#';
271     memcpy(((char*)data.data)+1,tile->raw_image->data,4);
272     memcpy(((char*)data.data)+5,&now,sizeof(apr_time_t));
273   } else {
274     if(!tile->encoded_data) {
275       tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format);
276       GC_CHECK_ERROR(ctx);
277     }
278     mapcache_buffer_append(tile->encoded_data,sizeof(apr_time_t),&now);
279     data.data = tile->encoded_data->buf;
280     data.size = tile->encoded_data->size;
281     tile->encoded_data->size -= sizeof(apr_time_t);
282   }
283 
284   pc = _bdb_get_conn(ctx,cache,tile,0);
285   GC_CHECK_ERROR(ctx);
286   benv = pc->connection;
287 
288 
289   ret = benv->db->put(benv->db,NULL,&key,&data,0);
290   if(ret != 0) {
291     ctx->set_error(ctx,500,"dbd backend failed on tile_set: %s", db_strerror(ret));
292   } else {
293     ret = benv->db->sync(benv->db,0);
294     if(ret)
295       ctx->set_error(ctx,500,"bdb backend sync failure on tile_set: %s",db_strerror(ret));
296   }
297   _bdb_release_conn(ctx,cache,tile,pc);
298 }
299 
_mapcache_cache_bdb_multiset(mapcache_context * ctx,mapcache_cache * pcache,mapcache_tile * tiles,int ntiles)300 static void _mapcache_cache_bdb_multiset(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tiles, int ntiles)
301 {
302   DBT key,data;
303   int ret,i;
304   apr_time_t now;
305   mapcache_cache_bdb *cache = (mapcache_cache_bdb*)pcache;
306   mapcache_pooled_connection *pc;
307   struct bdb_env *benv;
308   now = apr_time_now();
309   memset(&key, 0, sizeof(DBT));
310   memset(&data, 0, sizeof(DBT));
311 
312   pc = _bdb_get_conn(ctx,cache,&tiles[0],0);
313   GC_CHECK_ERROR(ctx);
314   benv = pc->connection;
315 
316   for(i=0; i<ntiles; i++) {
317     char *skey;
318     mapcache_tile *tile;
319     memset(&key, 0, sizeof(DBT));
320     memset(&data, 0, sizeof(DBT));
321     tile = &tiles[i];
322     skey = mapcache_util_get_tile_key(ctx,tile,cache->key_template,NULL,NULL);
323     if(!tile->raw_image) {
324       tile->raw_image = mapcache_imageio_decode(ctx, tile->encoded_data);
325       if(GC_HAS_ERROR(ctx)) {
326         _bdb_release_conn(ctx,cache,&tiles[0],pc);
327         return;
328       }
329     }
330     if(tile->raw_image->h==256 && tile->raw_image->w==256 && mapcache_image_blank_color(tile->raw_image) != MAPCACHE_FALSE) {
331       data.size = 5+sizeof(apr_time_t);
332       data.data = apr_palloc(ctx->pool,data.size);
333       (((char*)data.data)[0])='#';
334       memcpy(((char*)data.data)+1,tile->raw_image->data,4);
335       memcpy(((char*)data.data)+5,&now,sizeof(apr_time_t));
336     } else {
337       if(!tile->encoded_data) {
338         tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format);
339         if(GC_HAS_ERROR(ctx)) {
340           _bdb_release_conn(ctx,cache,&tiles[0],pc);
341           return;
342         }
343       }
344       mapcache_buffer_append(tile->encoded_data,sizeof(apr_time_t),&now);
345       data.data = tile->encoded_data->buf;
346       data.size = tile->encoded_data->size;
347       tile->encoded_data->size -= sizeof(apr_time_t);
348     }
349     key.data = skey;
350     key.size = strlen(skey)+1;
351 
352     ret = benv->db->put(benv->db,NULL,&key,&data,0);
353     if(ret != 0) {
354       ctx->set_error(ctx,500,"dbd backend failed on tile_multiset: %s", db_strerror(ret));
355       break;
356     }
357   }
358   if(ret == 0) {
359     ret = benv->db->sync(benv->db,0);
360     if(ret)
361       ctx->set_error(ctx,500,"bdb backend sync failure on sync in tile_multiset: %s",db_strerror(ret));
362   }
363   _bdb_release_conn(ctx,cache,&tiles[0],pc);
364 }
365 
366 
_mapcache_cache_bdb_configuration_parse_xml(mapcache_context * ctx,ezxml_t node,mapcache_cache * cache,mapcache_cfg * config)367 static void _mapcache_cache_bdb_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *cache, mapcache_cfg *config)
368 {
369   ezxml_t cur_node;
370   mapcache_cache_bdb *dcache = (mapcache_cache_bdb*)cache;
371   if ((cur_node = ezxml_child(node,"base")) != NULL) {
372     dcache->basedir = apr_pstrdup(ctx->pool,cur_node->txt);
373   }
374   if ((cur_node = ezxml_child(node,"key_template")) != NULL) {
375     dcache->key_template = apr_pstrdup(ctx->pool,cur_node->txt);
376   } else {
377     dcache->key_template = apr_pstrdup(ctx->pool,"{tileset}-{grid}-{dim}-{z}-{y}-{x}.{ext}");
378   }
379   if(!dcache->basedir) {
380     ctx->set_error(ctx,500,"dbd cache \"%s\" is missing <base> entry",cache->name);
381     return;
382   }
383 }
384 
385 /**
386  * \private \memberof mapcache_cache_dbd
387  */
_mapcache_cache_bdb_configuration_post_config(mapcache_context * ctx,mapcache_cache * cache,mapcache_cfg * cfg)388 static void _mapcache_cache_bdb_configuration_post_config(mapcache_context *ctx,
389     mapcache_cache *cache, mapcache_cfg *cfg)
390 {
391   mapcache_cache_bdb *dcache = (mapcache_cache_bdb*)cache;
392   apr_status_t rv;
393   apr_dir_t *dir;
394   rv = apr_dir_open(&dir, dcache->basedir, ctx->pool);
395   if(rv != APR_SUCCESS) {
396     char errmsg[120];
397     ctx->set_error(ctx,500,"bdb failed to open directory %s:%s",dcache->basedir,apr_strerror(rv,errmsg,120));
398   }
399 }
400 
401 /**
402  * \brief creates and initializes a mapcache_dbd_cache
403  */
mapcache_cache_bdb_create(mapcache_context * ctx)404 mapcache_cache* mapcache_cache_bdb_create(mapcache_context *ctx)
405 {
406   mapcache_cache_bdb *cache = apr_pcalloc(ctx->pool,sizeof(mapcache_cache_bdb));
407   if(!cache) {
408     ctx->set_error(ctx, 500, "failed to allocate berkeley db cache");
409     return NULL;
410   }
411   cache->cache.metadata = apr_table_make(ctx->pool,3);
412   cache->cache.type = MAPCACHE_CACHE_BDB;
413   cache->cache._tile_delete = _mapcache_cache_bdb_delete;
414   cache->cache._tile_get = _mapcache_cache_bdb_get;
415   cache->cache._tile_exists = _mapcache_cache_bdb_has_tile;
416   cache->cache._tile_set = _mapcache_cache_bdb_set;
417   cache->cache._tile_multi_set = _mapcache_cache_bdb_multiset;
418   cache->cache.configuration_post_config = _mapcache_cache_bdb_configuration_post_config;
419   cache->cache.configuration_parse_xml = _mapcache_cache_bdb_configuration_parse_xml;
420   cache->basedir = NULL;
421   cache->key_template = NULL;
422   return (mapcache_cache*)cache;
423 }
424 
425 #else
mapcache_cache_bdb_create(mapcache_context * ctx)426 mapcache_cache* mapcache_cache_bdb_create(mapcache_context *ctx) {
427   ctx->set_error(ctx,400,"BERKELEYDB support not compiled in this version");
428   return NULL;
429 }
430 #endif
431 
432 /* vim: ts=2 sts=2 et sw=2
433 */
434