1 /******************************************************************************
2 * $Id$
3 *
4 * Project: MapServer
5 * Purpose: MapCache tile caching support file: couchbase cache backend.
6 * Author: Michael Downey and the MapServer team.
7 *
8 ******************************************************************************
9 * Copyright (c) 1996-2011 Regents of the University of Minnesota.
10 *
11 * Permission is hereby granted, free of charge, to any person obtaining a
12 * copy of this software and associated documentation files (the "Software"),
13 * to deal in the Software without restriction, including without limitation
14 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
15 * and/or sell copies of the Software, and to permit persons to whom the
16 * Software is furnished to do so, subject to the following conditions:
17 *
18 * The above copyright notice and this permission notice shall be included in
19 * all copies of this Software or works derived from this Software.
20 *
21 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
22 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
24 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
26 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27 * DEALINGS IN THE SOFTWARE.
28 *****************************************************************************/
29
30 #include "mapcache.h"
31
32 #ifdef USE_COUCHBASE
33
34 #include <apr_strings.h>
35 #include <string.h>
36 #include <errno.h>
37 #include <libcouchbase/couchbase.h>
38
39 typedef struct mapcache_cache_couchbase mapcache_cache_couchbase;
40
41 /**\class mapcache_cache_couchbase
42 * \brief a mapcache_cache on couchbase servers
43 * \implements mapcache_cache
44 */
45 struct mapcache_cache_couchbase {
46 mapcache_cache cache;
47 // apr_reslist_t *connection_pool;
48 char *host;
49 char *username;
50 char *password;
51 char *bucket;
52 mapcache_context *ctx;
53 };
54
55
56 typedef struct getStruct
57 {
58 mapcache_buffer *tileBuffer;
59 libcouchbase_error_t error;
60 } getStruct_t;
61
62 /* Not sure if we need this. */
_couchbase_error_callback(libcouchbase_t instance,libcouchbase_error_t error,const char * errinfo)63 static void _couchbase_error_callback(libcouchbase_t instance,
64 libcouchbase_error_t error,
65 const char *errinfo)
66 {
67 /* Ignore timeouts... */
68 if (error != LIBCOUCHBASE_ETIMEDOUT) {
69 fprintf(stderr, "\nFATAL ERROR: %s\n",
70 libcouchbase_strerror(instance, error));
71 if (errinfo && strlen(errinfo) != 0) {
72 fprintf(stderr, "\t\"%s\"\n", errinfo);
73 }
74 }
75 }
76
_couchbase_get_callback(libcouchbase_t instance,const void * cookie,libcouchbase_error_t error,const void * key,libcouchbase_size_t nkey,const void * bytes,libcouchbase_size_t nbytes,libcouchbase_uint32_t flags,libcouchbase_cas_t cas)77 static void _couchbase_get_callback(libcouchbase_t instance,
78 const void *cookie,
79 libcouchbase_error_t error,
80 const void *key, libcouchbase_size_t nkey,
81 const void *bytes, libcouchbase_size_t nbytes,
82 libcouchbase_uint32_t flags, libcouchbase_cas_t cas)
83 {
84 (void)instance;
85 (void)key;
86 (void)nkey;
87 (void)flags;
88 (void)cas;
89
90 if (cookie)
91 {
92 getStruct_t *request = (getStruct_t*)cookie;
93
94 request->error = error;
95
96 if (error == LIBCOUCHBASE_SUCCESS && request->tileBuffer)
97 {
98 mapcache_buffer_append(request->tileBuffer, nbytes, (void*)bytes);
99 }
100 }
101 }
102
_couchbase_store_callback(libcouchbase_t instance,const void * cookie,libcouchbase_storage_t unknown,libcouchbase_error_t error,const void * unknown2,libcouchbase_size_t unknown3,libcouchbase_cas_t cas)103 static void _couchbase_store_callback(libcouchbase_t instance,
104 const void* cookie,
105 libcouchbase_storage_t unknown,
106 libcouchbase_error_t error,
107 const void* unknown2,
108 libcouchbase_size_t unknown3,
109 libcouchbase_cas_t cas)
110 {
111 (void)instance;
112 (void)unknown;
113 (void)unknown2;
114 (void)unknown3;
115 (void)cas;
116
117 libcouchbase_error_t* userError = (libcouchbase_error_t*)cookie;
118
119 *userError = error;
120 }
121
_couchbase_reslist_get_connection(void ** conn_,void * params,apr_pool_t * pool)122 static apr_status_t _couchbase_reslist_get_connection(void **conn_, void *params, apr_pool_t *pool) {
123 mapcache_cache_couchbase *cache = (mapcache_cache_couchbase*)params;
124
125 libcouchbase_t *instance = apr_pcalloc(pool,sizeof(libcouchbase_t));
126 const char *host = cache->host;
127 const char *username = cache->username;
128 const char *passwd = cache->password;
129 const char *bucket = "default";
130
131 *instance = libcouchbase_create(host, username, passwd, bucket, NULL);
132 if (*instance == NULL) {
133 return APR_EGENERAL;
134 }
135
136 libcouchbase_set_error_callback(*instance, _couchbase_error_callback);
137 libcouchbase_set_get_callback(*instance, _couchbase_get_callback);
138 libcouchbase_set_storage_callback(*instance, _couchbase_store_callback);
139
140 if (libcouchbase_connect(*instance) != LIBCOUCHBASE_SUCCESS) {
141 return APR_EGENERAL;
142 }
143
144 /* Wait for the connect to compelete */
145 libcouchbase_wait(*instance);
146
147 *conn_ = instance;
148 return APR_SUCCESS;
149 }
150
_couchbase_reslist_free_connection(void * conn_,void * params,apr_pool_t * pool)151 static apr_status_t _couchbase_reslist_free_connection(void *conn_, void *params, apr_pool_t *pool) {
152 libcouchbase_t *instance = (libcouchbase_t*)conn_;
153 libcouchbase_destroy(*instance);
154 return APR_SUCCESS;
155 }
156
_couchbase_get_connection(mapcache_context * ctx,mapcache_tile * tile)157 static libcouchbase_t* _couchbase_get_connection(mapcache_context *ctx, mapcache_tile *tile)
158 {
159 apr_status_t rv;
160 libcouchbase_t *instance;
161 mapcache_cache_couchbase *cache = (mapcache_cache_couchbase*)tile->tileset->cache;
162
163 rv = apr_reslist_acquire(cache->connection_pool, (void **)&instance);
164 if(rv != APR_SUCCESS) {
165 ctx->set_error(ctx, 500, "failed to aquire connection to couchbase backend: %s", ctx->get_error_message(ctx));
166 return NULL;
167 }
168
169 return instance;
170 }
171
_couchbase_release_connection(mapcache_tile * tile,libcouchbase_t * instance)172 static void _couchbase_release_connection(mapcache_tile *tile, libcouchbase_t* instance)
173 {
174 mapcache_cache_couchbase* cache = (mapcache_cache_couchbase*)tile->tileset->cache;
175 apr_reslist_release(cache->connection_pool, (void*)instance);
176 }
177
_couchbase_invalidate_connection(mapcache_tile * tile,libcouchbase_t * instance)178 static void _couchbase_invalidate_connection(mapcache_tile *tile, libcouchbase_t* instance)
179 {
180 mapcache_cache_couchbase* cache = (mapcache_cache_couchbase*)tile->tileset->cache;
181 apr_reslist_invalidate(cache->connection_pool, (void*)instance);
182 }
183
_mapcache_cache_couchbase_has_tile(mapcache_context * ctx,mapcache_tile * tile)184 static int _mapcache_cache_couchbase_has_tile(mapcache_context *ctx, mapcache_tile *tile) {
185 char *key[1];
186 libcouchbase_t *instance;
187 libcouchbase_error_t error;
188 size_t keySize[1];
189 getStruct_t request;
190
191 key[0] = mapcache_util_get_tile_key(ctx, tile, NULL, " \r\n\t\f\e\a\b", "#");
192 if(GC_HAS_ERROR(ctx)) {
193 return MAPCACHE_FALSE;
194 }
195
196 keySize[0] = strlen(key[0]);
197
198 instance = _couchbase_get_connection(ctx, tile);
199 request.tileBuffer = 0;
200 request.error = LIBCOUCHBASE_KEY_ENOENT;
201 error = libcouchbase_mget(*instance, &request, 1, (const void * const*)key, keySize, 0);
202 if (error != LIBCOUCHBASE_SUCCESS) {
203 ctx->set_error(ctx, 500, "couchbase: failed to get key %s: %s", key, libcouchbase_strerror(*instance, error));
204 _couchbase_invalidate_connection(tile, instance);
205 return MAPCACHE_FALSE;
206 }
207
208 libcouchbase_wait(*instance);
209
210 error = request.error;
211 if (error != LIBCOUCHBASE_SUCCESS) {
212 ctx->set_error(ctx, 500, "couchbase: failed to get key %s: %s", key, libcouchbase_strerror(*instance, error));
213 _couchbase_invalidate_connection(tile, instance);
214 return MAPCACHE_FALSE;
215 }
216
217 _couchbase_release_connection(tile, instance);
218 return MAPCACHE_TRUE;
219 }
220
_mapcache_cache_couchbase_delete(mapcache_context * ctx,mapcache_tile * tile)221 static void _mapcache_cache_couchbase_delete(mapcache_context *ctx, mapcache_tile *tile) {
222 char *key;
223 libcouchbase_t *instance;
224 libcouchbase_error_t error;
225
226 key = mapcache_util_get_tile_key(ctx, tile,NULL," \r\n\t\f\e\a\b","#");
227 GC_CHECK_ERROR(ctx);
228
229 instance = _couchbase_get_connection(ctx, tile);
230
231 error = libcouchbase_remove(*instance, 0, key, strlen(key), 0);
232 if (error != LIBCOUCHBASE_SUCCESS) {
233 ctx->set_error(ctx, 500, "couchbase: failed to delete key %s: %s", key, libcouchbase_strerror(*instance, error));
234 }
235
236 libcouchbase_wait(*instance);
237
238 error = libcouchbase_get_last_error(*instance);
239 if (error != LIBCOUCHBASE_SUCCESS) {
240 ctx->set_error(ctx, 500, "couchbase: failed to delete key %s: %s", key, libcouchbase_strerror(*instance, error));
241 }
242
243 _couchbase_release_connection(tile, instance);
244 }
245
246 /**
247 * \brief get content of given tile
248 *
249 * fills the mapcache_tile::data of the given tile with content stored on the couchbase server
250 * \private \memberof mapcache_cache_couchbase
251 * \sa mapcache_cache::tile_get()
252 */
_mapcache_cache_couchbase_get(mapcache_context * ctx,mapcache_tile * tile)253 static int _mapcache_cache_couchbase_get(mapcache_context *ctx, mapcache_tile *tile) {
254 char *key[1];
255 size_t keySize[1];
256 libcouchbase_t *instance;
257 libcouchbase_error_t error;
258 getStruct_t request;
259
260 key[0] = mapcache_util_get_tile_key(ctx, tile, NULL, " \r\n\t\f\e\a\b", "#");
261 if(GC_HAS_ERROR(ctx)) {
262 return MAPCACHE_FAILURE;
263 }
264
265 keySize[0] = strlen(key[0]);
266
267 tile->encoded_data = mapcache_buffer_create(0, ctx->pool);
268
269 libcouchbase_time_t expires = 86400;
270 if(tile->tileset->auto_expire)
271 expires = tile->tileset->auto_expire;
272
273 instance = _couchbase_get_connection(ctx, tile);
274 if (GC_HAS_ERROR(ctx)) {
275 return MAPCACHE_FAILURE;
276 }
277
278 request.tileBuffer = tile->encoded_data;
279 error = libcouchbase_mget(*instance, &request, 1, (const void * const*)key, keySize, &expires);
280 if (error != LIBCOUCHBASE_SUCCESS) {
281 ctx->set_error(ctx, 500, "couchbase cache returned error on mget %s", libcouchbase_strerror(*instance, error));
282 _couchbase_invalidate_connection(tile, instance);
283 return MAPCACHE_FAILURE;
284 }
285
286 libcouchbase_wait(*instance);
287
288 if(request.error != LIBCOUCHBASE_SUCCESS) {
289 _couchbase_release_connection(tile, instance);
290 return MAPCACHE_CACHE_MISS;
291 }
292
293 if (tile->encoded_data->size == 0) {
294 _couchbase_release_connection(tile, instance);
295 ctx->set_error(ctx, 500, "couchbase cache returned 0-length data for tile %d %d %d", tile->x, tile->y, tile->z);
296 return MAPCACHE_FAILURE;
297 }
298
299 apr_time_t now = apr_time_now();
300 tile->mtime = now;
301
302 _couchbase_release_connection(tile, instance);
303 return MAPCACHE_SUCCESS;
304 }
305
306 /**
307 * \brief push tile data to couchbase
308 *
309 * writes the content of mapcache_tile::data to the configured couchbased instance(s)
310 * \private \memberof mapcache_cache_couchbase
311 * \sa mapcache_cache::tile_set()
312 */
_mapcache_cache_couchbase_set(mapcache_context * ctx,mapcache_tile * tile)313 static void _mapcache_cache_couchbase_set(mapcache_context *ctx, mapcache_tile *tile) {
314 char *key;
315 libcouchbase_t *instance;
316 libcouchbase_error_t error;
317 const int max_retries = 3;
318 int retries = max_retries;
319 apr_interval_time_t delay;
320
321 /* set expiration to one day if not configured */
322 libcouchbase_time_t expires = 86400;
323 if(tile->tileset->auto_expire)
324 expires = tile->tileset->auto_expire;
325
326 mapcache_cache_couchbase *cache = (mapcache_cache_couchbase*)tile->tileset->cache;
327 key = mapcache_util_get_tile_key(ctx, tile, NULL, " \r\n\t\f\e\a\b", "#");
328 GC_CHECK_ERROR(ctx);
329
330 if(!tile->encoded_data) {
331 tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format);
332 GC_CHECK_ERROR(ctx);
333 }
334
335 instance = _couchbase_get_connection(ctx, tile);
336 GC_CHECK_ERROR(ctx);
337
338 do
339 {
340 error = libcouchbase_store(*instance, &error, LIBCOUCHBASE_SET, key, strlen(key), tile->encoded_data->buf, tile->encoded_data->size, 0, expires, 0);
341 if (error != LIBCOUCHBASE_SUCCESS) {
342 _couchbase_release_connection(tile, instance);
343 ctx->set_error(ctx, 500, "failed to store tile %d %d %d to couchbase cache %s due to eror %s.",
344 tile->x, tile->y, tile->z, cache->cache.name, libcouchbase_strerror(*instance, error));
345 return;
346 }
347
348 libcouchbase_wait(*instance);
349
350 if (error == LIBCOUCHBASE_ETMPFAIL) {
351 if (retries > 0) {
352 delay = 100000 * (1 << (max_retries - retries)); // Do an exponential back off of starting at 100 milliseconds
353 apr_sleep(delay);
354 }
355 else {
356 _couchbase_release_connection(tile, instance);
357 ctx->set_error(ctx, 500, "failed to store tile %d %d %d to couchbase cache %s due to %s. Maximum number of retries used.",
358 tile->x, tile->y, tile->z, cache->cache.name, libcouchbase_strerror(*instance, error));
359 return;
360 }
361
362 --retries;
363 }
364
365 else if (error != LIBCOUCHBASE_SUCCESS) {
366 _couchbase_release_connection(tile, instance);
367 ctx->set_error(ctx, 500, "failed to store tile %d %d %d to couchbase cache %s due to error %s.",
368 tile->x, tile->y, tile->z, cache->cache.name, libcouchbase_strerror(*instance, error));
369 return;
370 }
371 }
372 while (error == LIBCOUCHBASE_ETMPFAIL);
373
374 _couchbase_release_connection(tile, instance);
375 }
376
377 /**
378 * \private \memberof mapcache_cache_couchbase
379 */
_mapcache_cache_couchbase_configuration_parse_xml(mapcache_context * ctx,ezxml_t node,mapcache_cache * cache,mapcache_cfg * config)380 static void _mapcache_cache_couchbase_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *cache, mapcache_cfg *config) {
381 ezxml_t cur_node;
382 apr_status_t rv;
383 mapcache_cache_couchbase *dcache = (mapcache_cache_couchbase*)cache;
384 int servercount = 0;
385
386 for(cur_node = ezxml_child(node,"server"); cur_node; cur_node = cur_node->next) {
387 servercount++;
388 }
389
390 if(!servercount) {
391 ctx->set_error(ctx, 400, "couchbase cache %s has no <server>s configured", cache->name);
392 return;
393 }
394
395 if(servercount > 1) {
396 ctx->set_error(ctx, 400, "couchbase cache %s has more than 1 server configured", cache->name);
397 return;
398 }
399
400 cur_node = ezxml_child(node, "server");
401 ezxml_t xhost = ezxml_child(cur_node, "host"); /* Host should contain server:port */
402 ezxml_t xusername = ezxml_child(cur_node, "username");
403 ezxml_t xpasswd = ezxml_child(cur_node, "password");
404 ezxml_t xbucket = ezxml_child(cur_node, "bucket");
405
406 if(!xhost || !xhost->txt || ! *xhost->txt) {
407 ctx->set_error(ctx, 400, "cache %s: <server> with no <host>", cache->name);
408 return;
409 } else {
410 dcache->host = apr_pstrdup(ctx->pool, xhost->txt);
411 if (dcache->host == NULL) {
412 ctx->set_error(ctx, 400, "cache %s: failed to allocate host string!", cache->name);
413 return;
414 }
415 }
416
417 if(xusername && xusername->txt && *xusername->txt) {
418 dcache->username = apr_pstrdup(ctx->pool, xusername->txt);
419 }
420
421 if(xpasswd && xpasswd->txt && *xpasswd->txt) {
422 dcache->password = apr_pstrdup(ctx->pool, xpasswd->txt);
423 }
424
425 if(xbucket && xbucket->txt && *xbucket->txt) {
426 dcache->bucket = apr_pstrdup(ctx->pool, xbucket->txt);
427 }
428
429 dcache->ctx = ctx;
430
431 rv = apr_reslist_create(&(dcache->connection_pool),
432 0 /* min */,
433 10 /* soft max */,
434 200 /* hard max */,
435 60*1000000 /*60 seconds, ttl*/,
436 _couchbase_reslist_get_connection, /* resource constructor */
437 _couchbase_reslist_free_connection, /* resource destructor */
438 dcache, ctx->pool);
439 if(rv != APR_SUCCESS) {
440 ctx->set_error(ctx, 500, "failed to create couchbase connection pool");
441 return;
442 }
443 }
444
445 /**
446 * \private \memberof mapcache_cache_couchbase
447 */
_mapcache_cache_couchbase_configuration_post_config(mapcache_context * ctx,mapcache_cache * cache,mapcache_cfg * cfg)448 static void _mapcache_cache_couchbase_configuration_post_config(mapcache_context *ctx, mapcache_cache *cache, mapcache_cfg *cfg) {
449 }
450
451 /**
452 * \brief creates and initializes a mapcache_couchbase_cache
453 */
mapcache_cache_couchbase_create(mapcache_context * ctx)454 mapcache_cache* mapcache_cache_couchbase_create(mapcache_context *ctx) {
455 mapcache_cache_couchbase *cache = apr_pcalloc(ctx->pool,sizeof(mapcache_cache_couchbase));
456 if(!cache) {
457 ctx->set_error(ctx, 500, "failed to allocate couchbase cache");
458 return NULL;
459 }
460
461 cache->cache.metadata = apr_table_make(ctx->pool, 3);
462 cache->cache.type = MAPCACHE_CACHE_COUCHBASE;
463 cache->cache.tile_get = _mapcache_cache_couchbase_get;
464 cache->cache.tile_exists = _mapcache_cache_couchbase_has_tile;
465 cache->cache.tile_set = _mapcache_cache_couchbase_set;
466 cache->cache.tile_delete = _mapcache_cache_couchbase_delete;
467 cache->cache.configuration_parse_xml = _mapcache_cache_couchbase_configuration_parse_xml;
468 cache->cache.configuration_post_config = _mapcache_cache_couchbase_configuration_post_config;
469 cache->host = NULL;
470 cache->username = NULL;
471 cache->password = NULL;
472 cache->bucket = NULL;
473
474 return (mapcache_cache*)cache;
475 }
476
477 #else
mapcache_cache_couchbase_create(mapcache_context * ctx)478 mapcache_cache* mapcache_cache_couchbase_create(mapcache_context *ctx) {
479 ctx->set_error(ctx,400,"COUCHBASE support not compiled in this version");
480 return NULL;
481 }
482 #endif
483
484 /* vim: ai ts=3 sts=3 et sw=3
485 */
486