1 /******************************************************************************
2 *
3 * Project: MapCache
4 * Purpose: MapCache tile caching support file: Berkeley DB cache backend
5 * Author: Thomas Bonfort and the MapServer team.
6 *
7 ******************************************************************************
8 * Copyright (c) 1996-2011 Regents of the University of Minnesota.
9 *
10 * Permission is hereby granted, free of charge, to any person obtaining a
11 * copy of this software and associated documentation files (the "Software"),
12 * to deal in the Software without restriction, including without limitation
13 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14 * and/or sell copies of the Software, and to permit persons to whom the
15 * Software is furnished to do so, subject to the following conditions:
16 *
17 * The above copyright notice and this permission notice shall be included in
18 * all copies of this Software or works derived from this Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26 * DEALINGS IN THE SOFTWARE.
27 *****************************************************************************/
28
29 #include "mapcache.h"
30 #ifdef USE_BDB
31 #include <apr_strings.h>
32 #include <apr_reslist.h>
33 #include <apr_file_info.h>
34 #include <apr_hash.h>
35 #include <string.h>
36 #include <errno.h>
37 #include <time.h>
38 #ifdef APR_HAS_THREADS
39 #include <apr_thread_mutex.h>
40 #endif
41
42 #ifndef _WIN32
43 #include <unistd.h>
44 #endif
45
46 #include <db.h>
47
48 #define PAGESIZE 64*1024
49 #define CACHESIZE 1024*1024
50
51 typedef struct mapcache_cache_bdb mapcache_cache_bdb;
52 struct mapcache_cache_bdb {
53 mapcache_cache cache;
54 char *basedir;
55 char *key_template;
56 };
57
58 struct bdb_env {
59 DB* db;
60 DB_ENV *env;
61 int readonly;
62 char *errmsg;
63 };
64
mapcache_bdb_connection_constructor(mapcache_context * ctx,void ** conn_,void * params)65 void mapcache_bdb_connection_constructor(mapcache_context *ctx, void **conn_, void *params)
66 {
67 int ret;
68 int env_flags;
69 int mode;
70 mapcache_cache_bdb *cache = (mapcache_cache_bdb*)params;
71 char *dbfile = malloc(strlen(cache->basedir)+strlen(cache->cache.name)+strlen("/.db")+1);
72 struct bdb_env *benv = calloc(1,sizeof(struct bdb_env));
73 dbfile[0]=0;
74 strcat(strcat(strcat(strcat(dbfile,cache->basedir),"/"),cache->cache.name),".db"); /*yuk yuk*/
75 *conn_ = benv;
76
77 ret = db_env_create(&benv->env, 0);
78 if(ret) {
79 ctx->set_error(ctx, 500, "bdb cache failure for db_env_create: %s", db_strerror(ret));
80 goto cleanup_error;
81 }
82 ret = benv->env->set_cachesize(benv->env,0,CACHESIZE,1); /* set a larger cache size than default */
83 if(ret) {
84 ctx->set_error(ctx, 500, "bdb cache failure for db->set_cachesize: %s", db_strerror(ret));
85 goto cleanup_error;
86 }
87 env_flags = DB_INIT_CDB|DB_INIT_MPOOL|DB_CREATE;
88 ret = benv->env->open(benv->env,cache->basedir,env_flags,0);
89 if(ret) {
90 ctx->set_error(ctx,500,"bdb cache failure for env->open: %s", db_strerror(ret));
91 goto cleanup_error;
92 }
93
94 if ((ret = db_create(&benv->db, benv->env, 0)) != 0) {
95 ctx->set_error(ctx,500,"bdb cache failure for db_create: %s", db_strerror(ret));
96 goto cleanup_error;
97 }
98 mode = DB_BTREE;
99 ret = benv->db->set_pagesize(benv->db,PAGESIZE); /* set pagesize to maximum allowed, as tile data is usually pretty large */
100 if(ret) {
101 ctx->set_error(ctx,500,"bdb cache failure for db->set_pagesize: %s", db_strerror(ret));
102 goto cleanup_error;
103 }
104
105 if ((ret = benv->db->open(benv->db, NULL, dbfile, NULL, mode, DB_CREATE, 0664)) != 0) {
106 ctx->set_error(ctx,500,"bdb cache failure 1 for db->open: %s", db_strerror(ret));
107 goto cleanup_error;
108 }
109
110 goto cleanup;
111
112 cleanup_error:
113 free(benv);
114 cleanup:
115 free(dbfile);
116 }
117
mapcache_bdb_connection_destructor(void * conn_)118 void mapcache_bdb_connection_destructor(void *conn_)
119 {
120 struct bdb_env *benv = (struct bdb_env*)conn_;
121 benv->db->close(benv->db,0);
122 benv->env->close(benv->env,0);
123 free(benv);
124 }
125
126
127
_bdb_get_conn(mapcache_context * ctx,mapcache_cache_bdb * cache,mapcache_tile * tile,int readonly)128 static mapcache_pooled_connection* _bdb_get_conn(mapcache_context *ctx, mapcache_cache_bdb *cache, mapcache_tile* tile, int readonly) {
129 struct bdb_env *benv;
130 mapcache_pooled_connection *pc;
131 char *conn_key = apr_pstrcat(ctx->pool,readonly?"ro_":"rw_",cache->cache.name,NULL);
132 pc = mapcache_connection_pool_get_connection(ctx,conn_key,mapcache_bdb_connection_constructor, mapcache_bdb_connection_destructor, cache);
133 if(GC_HAS_ERROR(ctx)) return NULL;
134 benv = pc->connection;
135 benv->readonly = readonly;
136 return pc;
137 }
138
_bdb_release_conn(mapcache_context * ctx,mapcache_cache_bdb * cache,mapcache_tile * tile,mapcache_pooled_connection * pc)139 static void _bdb_release_conn(mapcache_context *ctx, mapcache_cache_bdb *cache, mapcache_tile *tile, mapcache_pooled_connection *pc)
140 {
141 if(GC_HAS_ERROR(ctx)) {
142 mapcache_connection_pool_invalidate_connection(ctx, pc);
143 } else {
144 mapcache_connection_pool_release_connection(ctx,pc);
145 }
146 }
147
_mapcache_cache_bdb_has_tile(mapcache_context * ctx,mapcache_cache * pcache,mapcache_tile * tile)148 static int _mapcache_cache_bdb_has_tile(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile)
149 {
150 int ret;
151 DBT key;
152 mapcache_cache_bdb *cache = (mapcache_cache_bdb*)pcache;
153 char *skey = mapcache_util_get_tile_key(ctx,tile,cache->key_template,NULL,NULL);
154 mapcache_pooled_connection *pc;
155 struct bdb_env *benv;
156 pc = _bdb_get_conn(ctx,cache,tile,1);
157 if(GC_HAS_ERROR(ctx)) return MAPCACHE_FALSE;
158 benv = pc->connection;
159 memset(&key, 0, sizeof(DBT));
160 key.data = skey;
161 key.size = strlen(skey)+1;
162
163 ret = benv->db->exists(benv->db, NULL, &key, 0);
164
165 if(ret == 0) {
166 ret = MAPCACHE_TRUE;
167 } else if(ret == DB_NOTFOUND) {
168 ret = MAPCACHE_FALSE;
169 } else {
170 ctx->set_error(ctx,500,"bdb backend failure on tile_exists: %s",db_strerror(ret));
171 ret= MAPCACHE_FALSE;
172 }
173 _bdb_release_conn(ctx,cache,tile,pc);
174 return ret;
175 }
176
_mapcache_cache_bdb_delete(mapcache_context * ctx,mapcache_cache * pcache,mapcache_tile * tile)177 static void _mapcache_cache_bdb_delete(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile)
178 {
179 DBT key;
180 int ret;
181 mapcache_cache_bdb *cache = (mapcache_cache_bdb*)pcache;
182 char *skey = mapcache_util_get_tile_key(ctx,tile,cache->key_template,NULL,NULL);
183 mapcache_pooled_connection *pc;
184 struct bdb_env *benv;
185 pc = _bdb_get_conn(ctx,cache,tile,0);
186 GC_CHECK_ERROR(ctx);
187 benv = pc->connection;
188 memset(&key, 0, sizeof(DBT));
189 key.data = skey;
190 key.size = strlen(skey)+1;
191 ret = benv->db->del(benv->db, NULL, &key, 0);
192 if(ret && ret != DB_NOTFOUND) {
193 ctx->set_error(ctx,500,"bdb backend failure on tile_delete: %s",db_strerror(ret));
194 } else {
195 ret = benv->db->sync(benv->db,0);
196 if(ret)
197 ctx->set_error(ctx,500,"bdb backend sync failure on tile_delete: %s",db_strerror(ret));
198 }
199 _bdb_release_conn(ctx,cache,tile,pc);
200 }
201
_mapcache_cache_bdb_get(mapcache_context * ctx,mapcache_cache * pcache,mapcache_tile * tile)202 static int _mapcache_cache_bdb_get(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile)
203 {
204 DBT key,data;
205 int ret;
206 char *skey;
207 mapcache_cache_bdb *cache = (mapcache_cache_bdb*)pcache;
208 mapcache_pooled_connection *pc;
209 struct bdb_env *benv;
210 pc = _bdb_get_conn(ctx,cache,tile,1);
211 if(GC_HAS_ERROR(ctx)) return MAPCACHE_FALSE;
212 benv = pc->connection;
213 skey = mapcache_util_get_tile_key(ctx,tile,cache->key_template,NULL,NULL);
214 memset(&key, 0, sizeof(DBT));
215 memset(&data, 0, sizeof(DBT));
216 data.flags = DB_DBT_MALLOC;
217 key.data = skey;
218 key.size = strlen(skey)+1;
219
220 ret = benv->db->get(benv->db, NULL, &key, &data, 0);
221
222
223 if(ret == 0) {
224 if(((char*)(data.data))[0] == '#') {
225 tile->encoded_data = mapcache_empty_png_decode(ctx,tile->grid_link->grid->tile_sx, tile->grid_link->grid->tile_sy, (unsigned char*)data.data,&tile->nodata);
226 } else {
227 tile->encoded_data = mapcache_buffer_create(0,ctx->pool);
228 tile->encoded_data->buf = data.data;
229 tile->encoded_data->size = data.size-sizeof(apr_time_t);
230 tile->encoded_data->avail = data.size;
231 apr_pool_cleanup_register(ctx->pool, tile->encoded_data->buf,(void*)free, apr_pool_cleanup_null);
232 }
233 tile->mtime = *((apr_time_t*)(((char*)data.data)+data.size-sizeof(apr_time_t)));
234 ret = MAPCACHE_SUCCESS;
235 } else if(ret == DB_NOTFOUND) {
236 ret = MAPCACHE_CACHE_MISS;
237 } else {
238 ctx->set_error(ctx,500,"bdb backend failure on tile_get: %s",db_strerror(ret));
239 ret = MAPCACHE_FAILURE;
240 }
241 _bdb_release_conn(ctx,cache,tile,pc);
242 return ret;
243 }
244
245
_mapcache_cache_bdb_set(mapcache_context * ctx,mapcache_cache * pcache,mapcache_tile * tile)246 static void _mapcache_cache_bdb_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile)
247 {
248 DBT key,data;
249 int ret;
250 apr_time_t now;
251 mapcache_cache_bdb *cache = (mapcache_cache_bdb*)pcache;
252 char *skey = mapcache_util_get_tile_key(ctx,tile,cache->key_template,NULL,NULL);
253 mapcache_pooled_connection *pc;
254 struct bdb_env *benv;
255 now = apr_time_now();
256 memset(&key, 0, sizeof(DBT));
257 memset(&data, 0, sizeof(DBT));
258
259 key.data = skey;
260 key.size = strlen(skey)+1;
261
262 if(!tile->raw_image) {
263 tile->raw_image = mapcache_imageio_decode(ctx, tile->encoded_data);
264 GC_CHECK_ERROR(ctx);
265 }
266
267 if(tile->raw_image->h==256 && tile->raw_image->w==256 && mapcache_image_blank_color(tile->raw_image) != MAPCACHE_FALSE) {
268 data.size = 5+sizeof(apr_time_t);
269 data.data = apr_palloc(ctx->pool,data.size);
270 (((char*)data.data)[0])='#';
271 memcpy(((char*)data.data)+1,tile->raw_image->data,4);
272 memcpy(((char*)data.data)+5,&now,sizeof(apr_time_t));
273 } else {
274 if(!tile->encoded_data) {
275 tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format);
276 GC_CHECK_ERROR(ctx);
277 }
278 mapcache_buffer_append(tile->encoded_data,sizeof(apr_time_t),&now);
279 data.data = tile->encoded_data->buf;
280 data.size = tile->encoded_data->size;
281 tile->encoded_data->size -= sizeof(apr_time_t);
282 }
283
284 pc = _bdb_get_conn(ctx,cache,tile,0);
285 GC_CHECK_ERROR(ctx);
286 benv = pc->connection;
287
288
289 ret = benv->db->put(benv->db,NULL,&key,&data,0);
290 if(ret != 0) {
291 ctx->set_error(ctx,500,"dbd backend failed on tile_set: %s", db_strerror(ret));
292 } else {
293 ret = benv->db->sync(benv->db,0);
294 if(ret)
295 ctx->set_error(ctx,500,"bdb backend sync failure on tile_set: %s",db_strerror(ret));
296 }
297 _bdb_release_conn(ctx,cache,tile,pc);
298 }
299
_mapcache_cache_bdb_multiset(mapcache_context * ctx,mapcache_cache * pcache,mapcache_tile * tiles,int ntiles)300 static void _mapcache_cache_bdb_multiset(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tiles, int ntiles)
301 {
302 DBT key,data;
303 int ret,i;
304 apr_time_t now;
305 mapcache_cache_bdb *cache = (mapcache_cache_bdb*)pcache;
306 mapcache_pooled_connection *pc;
307 struct bdb_env *benv;
308 now = apr_time_now();
309 memset(&key, 0, sizeof(DBT));
310 memset(&data, 0, sizeof(DBT));
311
312 pc = _bdb_get_conn(ctx,cache,&tiles[0],0);
313 GC_CHECK_ERROR(ctx);
314 benv = pc->connection;
315
316 for(i=0; i<ntiles; i++) {
317 char *skey;
318 mapcache_tile *tile;
319 memset(&key, 0, sizeof(DBT));
320 memset(&data, 0, sizeof(DBT));
321 tile = &tiles[i];
322 skey = mapcache_util_get_tile_key(ctx,tile,cache->key_template,NULL,NULL);
323 if(!tile->raw_image) {
324 tile->raw_image = mapcache_imageio_decode(ctx, tile->encoded_data);
325 if(GC_HAS_ERROR(ctx)) {
326 _bdb_release_conn(ctx,cache,&tiles[0],pc);
327 return;
328 }
329 }
330 if(tile->raw_image->h==256 && tile->raw_image->w==256 && mapcache_image_blank_color(tile->raw_image) != MAPCACHE_FALSE) {
331 data.size = 5+sizeof(apr_time_t);
332 data.data = apr_palloc(ctx->pool,data.size);
333 (((char*)data.data)[0])='#';
334 memcpy(((char*)data.data)+1,tile->raw_image->data,4);
335 memcpy(((char*)data.data)+5,&now,sizeof(apr_time_t));
336 } else {
337 if(!tile->encoded_data) {
338 tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format);
339 if(GC_HAS_ERROR(ctx)) {
340 _bdb_release_conn(ctx,cache,&tiles[0],pc);
341 return;
342 }
343 }
344 mapcache_buffer_append(tile->encoded_data,sizeof(apr_time_t),&now);
345 data.data = tile->encoded_data->buf;
346 data.size = tile->encoded_data->size;
347 tile->encoded_data->size -= sizeof(apr_time_t);
348 }
349 key.data = skey;
350 key.size = strlen(skey)+1;
351
352 ret = benv->db->put(benv->db,NULL,&key,&data,0);
353 if(ret != 0) {
354 ctx->set_error(ctx,500,"dbd backend failed on tile_multiset: %s", db_strerror(ret));
355 break;
356 }
357 }
358 if(ret == 0) {
359 ret = benv->db->sync(benv->db,0);
360 if(ret)
361 ctx->set_error(ctx,500,"bdb backend sync failure on sync in tile_multiset: %s",db_strerror(ret));
362 }
363 _bdb_release_conn(ctx,cache,&tiles[0],pc);
364 }
365
366
_mapcache_cache_bdb_configuration_parse_xml(mapcache_context * ctx,ezxml_t node,mapcache_cache * cache,mapcache_cfg * config)367 static void _mapcache_cache_bdb_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *cache, mapcache_cfg *config)
368 {
369 ezxml_t cur_node;
370 mapcache_cache_bdb *dcache = (mapcache_cache_bdb*)cache;
371 if ((cur_node = ezxml_child(node,"base")) != NULL) {
372 dcache->basedir = apr_pstrdup(ctx->pool,cur_node->txt);
373 }
374 if ((cur_node = ezxml_child(node,"key_template")) != NULL) {
375 dcache->key_template = apr_pstrdup(ctx->pool,cur_node->txt);
376 } else {
377 dcache->key_template = apr_pstrdup(ctx->pool,"{tileset}-{grid}-{dim}-{z}-{y}-{x}.{ext}");
378 }
379 if(!dcache->basedir) {
380 ctx->set_error(ctx,500,"dbd cache \"%s\" is missing <base> entry",cache->name);
381 return;
382 }
383 }
384
385 /**
386 * \private \memberof mapcache_cache_dbd
387 */
_mapcache_cache_bdb_configuration_post_config(mapcache_context * ctx,mapcache_cache * cache,mapcache_cfg * cfg)388 static void _mapcache_cache_bdb_configuration_post_config(mapcache_context *ctx,
389 mapcache_cache *cache, mapcache_cfg *cfg)
390 {
391 mapcache_cache_bdb *dcache = (mapcache_cache_bdb*)cache;
392 apr_status_t rv;
393 apr_dir_t *dir;
394 rv = apr_dir_open(&dir, dcache->basedir, ctx->pool);
395 if(rv != APR_SUCCESS) {
396 char errmsg[120];
397 ctx->set_error(ctx,500,"bdb failed to open directory %s:%s",dcache->basedir,apr_strerror(rv,errmsg,120));
398 }
399 }
400
401 /**
402 * \brief creates and initializes a mapcache_dbd_cache
403 */
mapcache_cache_bdb_create(mapcache_context * ctx)404 mapcache_cache* mapcache_cache_bdb_create(mapcache_context *ctx)
405 {
406 mapcache_cache_bdb *cache = apr_pcalloc(ctx->pool,sizeof(mapcache_cache_bdb));
407 if(!cache) {
408 ctx->set_error(ctx, 500, "failed to allocate berkeley db cache");
409 return NULL;
410 }
411 cache->cache.metadata = apr_table_make(ctx->pool,3);
412 cache->cache.type = MAPCACHE_CACHE_BDB;
413 cache->cache._tile_delete = _mapcache_cache_bdb_delete;
414 cache->cache._tile_get = _mapcache_cache_bdb_get;
415 cache->cache._tile_exists = _mapcache_cache_bdb_has_tile;
416 cache->cache._tile_set = _mapcache_cache_bdb_set;
417 cache->cache._tile_multi_set = _mapcache_cache_bdb_multiset;
418 cache->cache.configuration_post_config = _mapcache_cache_bdb_configuration_post_config;
419 cache->cache.configuration_parse_xml = _mapcache_cache_bdb_configuration_parse_xml;
420 cache->basedir = NULL;
421 cache->key_template = NULL;
422 return (mapcache_cache*)cache;
423 }
424
425 #else
mapcache_cache_bdb_create(mapcache_context * ctx)426 mapcache_cache* mapcache_cache_bdb_create(mapcache_context *ctx) {
427 ctx->set_error(ctx,400,"BERKELEYDB support not compiled in this version");
428 return NULL;
429 }
430 #endif
431
432 /* vim: ts=2 sts=2 et sw=2
433 */
434