1 /******************************************************************************
2  * $Id$
3  *
4  * Project:  MapServer
5  * Purpose:  MapCache tile caching support file: couchbase cache backend.
6  * Author:   Michael Downey and the MapServer team.
7  *
8  ******************************************************************************
9  * Copyright (c) 1996-2011 Regents of the University of Minnesota.
10  *
11  * Permission is hereby granted, free of charge, to any person obtaining a
12  * copy of this software and associated documentation files (the "Software"),
13  * to deal in the Software without restriction, including without limitation
14  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
15  * and/or sell copies of the Software, and to permit persons to whom the
16  * Software is furnished to do so, subject to the following conditions:
17  *
18  * The above copyright notice and this permission notice shall be included in
19  * all copies of this Software or works derived from this Software.
20  *
21  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
22  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
24  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
26  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27  * DEALINGS IN THE SOFTWARE.
28  *****************************************************************************/
29 
30 #include "mapcache.h"
31 
32 #ifdef USE_COUCHBASE
33 
34 #include <apr_strings.h>
35 #include <string.h>
36 #include <errno.h>
37 #include <libcouchbase/couchbase.h>
38 
39 typedef struct mapcache_cache_couchbase mapcache_cache_couchbase;
40 
41 /**\class mapcache_cache_couchbase
42  * \brief a mapcache_cache on couchbase servers
43  * \implements mapcache_cache
44  */
45 struct mapcache_cache_couchbase {
46    mapcache_cache cache;
47 //   apr_reslist_t *connection_pool;
48    char *host;
49    char *username;
50    char *password;
51    char *bucket;
52    mapcache_context *ctx;
53 };
54 
55 
56 typedef struct getStruct
57 {
58    mapcache_buffer *tileBuffer;
59    libcouchbase_error_t error;
60 } getStruct_t;
61 
62 /* Not sure if we need this. */
_couchbase_error_callback(libcouchbase_t instance,libcouchbase_error_t error,const char * errinfo)63 static void _couchbase_error_callback(libcouchbase_t instance,
64                                       libcouchbase_error_t error,
65                                       const char *errinfo)
66 {
67    /* Ignore timeouts... */
68    if (error != LIBCOUCHBASE_ETIMEDOUT) {
69       fprintf(stderr, "\nFATAL ERROR: %s\n",
70               libcouchbase_strerror(instance, error));
71       if (errinfo && strlen(errinfo) != 0) {
72          fprintf(stderr, "\t\"%s\"\n", errinfo);
73       }
74    }
75 }
76 
_couchbase_get_callback(libcouchbase_t instance,const void * cookie,libcouchbase_error_t error,const void * key,libcouchbase_size_t nkey,const void * bytes,libcouchbase_size_t nbytes,libcouchbase_uint32_t flags,libcouchbase_cas_t cas)77 static void _couchbase_get_callback(libcouchbase_t instance,
78                                     const void *cookie,
79                                     libcouchbase_error_t error,
80                                     const void *key, libcouchbase_size_t nkey,
81                                     const void *bytes, libcouchbase_size_t nbytes,
82                                     libcouchbase_uint32_t flags, libcouchbase_cas_t cas)
83 {
84    (void)instance;
85    (void)key;
86    (void)nkey;
87    (void)flags;
88    (void)cas;
89 
90    if (cookie)
91    {
92        getStruct_t *request = (getStruct_t*)cookie;
93 
94        request->error = error;
95 
96        if (error == LIBCOUCHBASE_SUCCESS && request->tileBuffer)
97        {
98            mapcache_buffer_append(request->tileBuffer, nbytes, (void*)bytes);
99        }
100     }
101 }
102 
_couchbase_store_callback(libcouchbase_t instance,const void * cookie,libcouchbase_storage_t unknown,libcouchbase_error_t error,const void * unknown2,libcouchbase_size_t unknown3,libcouchbase_cas_t cas)103 static void _couchbase_store_callback(libcouchbase_t instance,
104                                       const void* cookie,
105                                       libcouchbase_storage_t unknown,
106                                       libcouchbase_error_t error,
107                                       const void* unknown2,
108                                       libcouchbase_size_t unknown3,
109                                       libcouchbase_cas_t cas)
110 {
111    (void)instance;
112    (void)unknown;
113    (void)unknown2;
114    (void)unknown3;
115    (void)cas;
116 
117    libcouchbase_error_t* userError = (libcouchbase_error_t*)cookie;
118 
119    *userError = error;
120 }
121 
_couchbase_reslist_get_connection(void ** conn_,void * params,apr_pool_t * pool)122 static apr_status_t _couchbase_reslist_get_connection(void **conn_, void *params, apr_pool_t *pool) {
123    mapcache_cache_couchbase *cache = (mapcache_cache_couchbase*)params;
124 
125    libcouchbase_t *instance = apr_pcalloc(pool,sizeof(libcouchbase_t));
126    const char *host = cache->host;
127    const char *username = cache->username;
128    const char *passwd = cache->password;
129    const char *bucket = "default";
130 
131   *instance = libcouchbase_create(host, username, passwd, bucket, NULL);
132    if (*instance == NULL) {
133       return APR_EGENERAL;
134    }
135 
136    libcouchbase_set_error_callback(*instance, _couchbase_error_callback);
137    libcouchbase_set_get_callback(*instance, _couchbase_get_callback);
138    libcouchbase_set_storage_callback(*instance, _couchbase_store_callback);
139 
140    if (libcouchbase_connect(*instance) != LIBCOUCHBASE_SUCCESS) {
141        return APR_EGENERAL;
142    }
143 
144    /* Wait for the connect to compelete */
145    libcouchbase_wait(*instance);
146 
147    *conn_ = instance;
148    return APR_SUCCESS;
149 }
150 
_couchbase_reslist_free_connection(void * conn_,void * params,apr_pool_t * pool)151 static apr_status_t _couchbase_reslist_free_connection(void *conn_, void *params, apr_pool_t *pool) {
152    libcouchbase_t *instance = (libcouchbase_t*)conn_;
153    libcouchbase_destroy(*instance);
154    return APR_SUCCESS;
155 }
156 
_couchbase_get_connection(mapcache_context * ctx,mapcache_tile * tile)157 static libcouchbase_t* _couchbase_get_connection(mapcache_context *ctx, mapcache_tile *tile)
158 {
159    apr_status_t rv;
160    libcouchbase_t *instance;
161    mapcache_cache_couchbase *cache = (mapcache_cache_couchbase*)tile->tileset->cache;
162 
163    rv = apr_reslist_acquire(cache->connection_pool, (void **)&instance);
164    if(rv != APR_SUCCESS) {
165       ctx->set_error(ctx, 500, "failed to aquire connection to couchbase backend: %s", ctx->get_error_message(ctx));
166       return NULL;
167    }
168 
169    return instance;
170 }
171 
_couchbase_release_connection(mapcache_tile * tile,libcouchbase_t * instance)172 static void _couchbase_release_connection(mapcache_tile *tile, libcouchbase_t* instance)
173 {
174    mapcache_cache_couchbase* cache = (mapcache_cache_couchbase*)tile->tileset->cache;
175    apr_reslist_release(cache->connection_pool, (void*)instance);
176 }
177 
_couchbase_invalidate_connection(mapcache_tile * tile,libcouchbase_t * instance)178 static void _couchbase_invalidate_connection(mapcache_tile *tile, libcouchbase_t* instance)
179 {
180    mapcache_cache_couchbase* cache = (mapcache_cache_couchbase*)tile->tileset->cache;
181    apr_reslist_invalidate(cache->connection_pool, (void*)instance);
182 }
183 
_mapcache_cache_couchbase_has_tile(mapcache_context * ctx,mapcache_tile * tile)184 static int _mapcache_cache_couchbase_has_tile(mapcache_context *ctx, mapcache_tile *tile) {
185    char *key[1];
186    libcouchbase_t *instance;
187    libcouchbase_error_t error;
188    size_t keySize[1];
189    getStruct_t request;
190 
191    key[0] = mapcache_util_get_tile_key(ctx, tile, NULL, " \r\n\t\f\e\a\b", "#");
192    if(GC_HAS_ERROR(ctx)) {
193       return MAPCACHE_FALSE;
194    }
195 
196    keySize[0] = strlen(key[0]);
197 
198    instance = _couchbase_get_connection(ctx, tile);
199    request.tileBuffer = 0;
200    request.error = LIBCOUCHBASE_KEY_ENOENT;
201    error = libcouchbase_mget(*instance, &request, 1, (const void * const*)key, keySize, 0);
202    if (error != LIBCOUCHBASE_SUCCESS) {
203       ctx->set_error(ctx, 500, "couchbase: failed to get key %s: %s", key, libcouchbase_strerror(*instance, error));
204       _couchbase_invalidate_connection(tile, instance);
205       return MAPCACHE_FALSE;
206    }
207 
208    libcouchbase_wait(*instance);
209 
210    error = request.error;
211    if (error != LIBCOUCHBASE_SUCCESS) {
212       ctx->set_error(ctx, 500, "couchbase: failed to get key %s: %s", key, libcouchbase_strerror(*instance, error));
213       _couchbase_invalidate_connection(tile, instance);
214       return MAPCACHE_FALSE;
215    }
216 
217    _couchbase_release_connection(tile, instance);
218    return MAPCACHE_TRUE;
219 }
220 
_mapcache_cache_couchbase_delete(mapcache_context * ctx,mapcache_tile * tile)221 static void _mapcache_cache_couchbase_delete(mapcache_context *ctx, mapcache_tile *tile) {
222    char *key;
223    libcouchbase_t *instance;
224    libcouchbase_error_t error;
225 
226    key = mapcache_util_get_tile_key(ctx, tile,NULL," \r\n\t\f\e\a\b","#");
227    GC_CHECK_ERROR(ctx);
228 
229    instance = _couchbase_get_connection(ctx, tile);
230 
231    error = libcouchbase_remove(*instance, 0, key, strlen(key), 0);
232    if (error != LIBCOUCHBASE_SUCCESS) {
233       ctx->set_error(ctx, 500, "couchbase: failed to delete key %s: %s", key, libcouchbase_strerror(*instance, error));
234    }
235 
236    libcouchbase_wait(*instance);
237 
238    error = libcouchbase_get_last_error(*instance);
239    if (error != LIBCOUCHBASE_SUCCESS) {
240       ctx->set_error(ctx, 500, "couchbase: failed to delete key %s: %s", key, libcouchbase_strerror(*instance, error));
241    }
242 
243    _couchbase_release_connection(tile, instance);
244 }
245 
246 /**
247  * \brief get content of given tile
248  *
249  * fills the mapcache_tile::data of the given tile with content stored on the couchbase server
250  * \private \memberof mapcache_cache_couchbase
251  * \sa mapcache_cache::tile_get()
252  */
_mapcache_cache_couchbase_get(mapcache_context * ctx,mapcache_tile * tile)253 static int _mapcache_cache_couchbase_get(mapcache_context *ctx, mapcache_tile *tile) {
254    char *key[1];
255    size_t keySize[1];
256    libcouchbase_t *instance;
257    libcouchbase_error_t error;
258    getStruct_t request;
259 
260    key[0] = mapcache_util_get_tile_key(ctx, tile, NULL, " \r\n\t\f\e\a\b", "#");
261    if(GC_HAS_ERROR(ctx)) {
262       return MAPCACHE_FAILURE;
263    }
264 
265    keySize[0] = strlen(key[0]);
266 
267    tile->encoded_data = mapcache_buffer_create(0, ctx->pool);
268 
269    libcouchbase_time_t expires = 86400;
270    if(tile->tileset->auto_expire)
271       expires = tile->tileset->auto_expire;
272 
273    instance = _couchbase_get_connection(ctx, tile);
274    if (GC_HAS_ERROR(ctx)) {
275       return MAPCACHE_FAILURE;
276    }
277 
278    request.tileBuffer = tile->encoded_data;
279    error = libcouchbase_mget(*instance, &request, 1, (const void * const*)key, keySize, &expires);
280    if (error != LIBCOUCHBASE_SUCCESS) {
281       ctx->set_error(ctx, 500, "couchbase cache returned error on mget %s", libcouchbase_strerror(*instance, error));
282       _couchbase_invalidate_connection(tile, instance);
283       return MAPCACHE_FAILURE;
284    }
285 
286    libcouchbase_wait(*instance);
287 
288    if(request.error != LIBCOUCHBASE_SUCCESS) {
289        _couchbase_release_connection(tile, instance);
290        return MAPCACHE_CACHE_MISS;
291    }
292 
293    if (tile->encoded_data->size == 0) {
294       _couchbase_release_connection(tile, instance);
295       ctx->set_error(ctx, 500, "couchbase cache returned 0-length data for tile %d %d %d", tile->x, tile->y, tile->z);
296       return MAPCACHE_FAILURE;
297    }
298 
299    apr_time_t now = apr_time_now();
300    tile->mtime = now;
301 
302    _couchbase_release_connection(tile, instance);
303    return MAPCACHE_SUCCESS;
304 }
305 
306 /**
307  * \brief push tile data to couchbase
308  *
309  * writes the content of mapcache_tile::data to the configured couchbased instance(s)
310  * \private \memberof mapcache_cache_couchbase
311  * \sa mapcache_cache::tile_set()
312  */
_mapcache_cache_couchbase_set(mapcache_context * ctx,mapcache_tile * tile)313 static void _mapcache_cache_couchbase_set(mapcache_context *ctx, mapcache_tile *tile) {
314    char *key;
315    libcouchbase_t *instance;
316    libcouchbase_error_t error;
317    const int max_retries = 3;
318    int retries = max_retries;
319    apr_interval_time_t delay;
320 
321    /* set expiration to one day if not configured */
322    libcouchbase_time_t expires = 86400;
323    if(tile->tileset->auto_expire)
324       expires = tile->tileset->auto_expire;
325 
326    mapcache_cache_couchbase *cache = (mapcache_cache_couchbase*)tile->tileset->cache;
327    key = mapcache_util_get_tile_key(ctx, tile, NULL, " \r\n\t\f\e\a\b", "#");
328    GC_CHECK_ERROR(ctx);
329 
330    if(!tile->encoded_data) {
331       tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format);
332       GC_CHECK_ERROR(ctx);
333    }
334 
335    instance = _couchbase_get_connection(ctx, tile);
336    GC_CHECK_ERROR(ctx);
337 
338    do
339    {
340       error = libcouchbase_store(*instance, &error, LIBCOUCHBASE_SET, key, strlen(key), tile->encoded_data->buf, tile->encoded_data->size, 0, expires, 0);
341       if (error != LIBCOUCHBASE_SUCCESS) {
342           _couchbase_release_connection(tile, instance);
343           ctx->set_error(ctx, 500, "failed to store tile %d %d %d to couchbase cache %s due to eror %s.",
344                          tile->x, tile->y, tile->z, cache->cache.name, libcouchbase_strerror(*instance, error));
345           return;
346       }
347 
348       libcouchbase_wait(*instance);
349 
350       if (error == LIBCOUCHBASE_ETMPFAIL) {
351           if (retries > 0) {
352               delay = 100000 * (1 << (max_retries - retries));	// Do an exponential back off of starting at 100 milliseconds
353               apr_sleep(delay);
354           }
355           else {
356               _couchbase_release_connection(tile, instance);
357               ctx->set_error(ctx, 500, "failed to store tile %d %d %d to couchbase cache %s due to %s. Maximum number of retries used.",
358                              tile->x, tile->y, tile->z, cache->cache.name, libcouchbase_strerror(*instance, error));
359               return;
360           }
361 
362           --retries;
363       }
364 
365       else if (error != LIBCOUCHBASE_SUCCESS) {
366           _couchbase_release_connection(tile, instance);
367           ctx->set_error(ctx, 500, "failed to store tile %d %d %d to couchbase cache %s due to error %s.",
368                          tile->x, tile->y, tile->z, cache->cache.name, libcouchbase_strerror(*instance, error));
369           return;
370       }
371    }
372    while (error == LIBCOUCHBASE_ETMPFAIL);
373 
374    _couchbase_release_connection(tile, instance);
375 }
376 
377 /**
378  * \private \memberof mapcache_cache_couchbase
379  */
_mapcache_cache_couchbase_configuration_parse_xml(mapcache_context * ctx,ezxml_t node,mapcache_cache * cache,mapcache_cfg * config)380 static void _mapcache_cache_couchbase_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *cache, mapcache_cfg *config) {
381    ezxml_t cur_node;
382    apr_status_t rv;
383    mapcache_cache_couchbase *dcache = (mapcache_cache_couchbase*)cache;
384    int servercount = 0;
385 
386    for(cur_node = ezxml_child(node,"server"); cur_node; cur_node = cur_node->next) {
387       servercount++;
388    }
389 
390    if(!servercount) {
391       ctx->set_error(ctx, 400, "couchbase cache %s has no <server>s configured", cache->name);
392       return;
393    }
394 
395    if(servercount > 1) {
396       ctx->set_error(ctx, 400, "couchbase cache %s has more than 1 server configured", cache->name);
397       return;
398    }
399 
400    cur_node = ezxml_child(node, "server");
401    ezxml_t xhost = ezxml_child(cur_node, "host");   /* Host should contain server:port */
402    ezxml_t xusername = ezxml_child(cur_node, "username");
403    ezxml_t xpasswd = ezxml_child(cur_node, "password");
404    ezxml_t xbucket = ezxml_child(cur_node, "bucket");
405 
406    if(!xhost || !xhost->txt || ! *xhost->txt) {
407       ctx->set_error(ctx, 400, "cache %s: <server> with no <host>", cache->name);
408       return;
409    } else {
410       dcache->host = apr_pstrdup(ctx->pool, xhost->txt);
411       if (dcache->host == NULL) {
412           ctx->set_error(ctx, 400, "cache %s: failed to allocate host string!", cache->name);
413           return;
414       }
415    }
416 
417    if(xusername && xusername->txt && *xusername->txt) {
418       dcache->username = apr_pstrdup(ctx->pool, xusername->txt);
419    }
420 
421    if(xpasswd && xpasswd->txt && *xpasswd->txt) {
422       dcache->password = apr_pstrdup(ctx->pool, xpasswd->txt);
423    }
424 
425    if(xbucket && xbucket->txt && *xbucket->txt) {
426       dcache->bucket = apr_pstrdup(ctx->pool, xbucket->txt);
427    }
428 
429    dcache->ctx = ctx;
430 
431    rv = apr_reslist_create(&(dcache->connection_pool),
432          0 /* min */,
433          10 /* soft max */,
434          200 /* hard max */,
435          60*1000000 /*60 seconds, ttl*/,
436          _couchbase_reslist_get_connection, /* resource constructor */
437          _couchbase_reslist_free_connection, /* resource destructor */
438          dcache, ctx->pool);
439    if(rv != APR_SUCCESS) {
440       ctx->set_error(ctx, 500, "failed to create couchbase connection pool");
441       return;
442    }
443 }
444 
445 /**
446  * \private \memberof mapcache_cache_couchbase
447  */
_mapcache_cache_couchbase_configuration_post_config(mapcache_context * ctx,mapcache_cache * cache,mapcache_cfg * cfg)448 static void _mapcache_cache_couchbase_configuration_post_config(mapcache_context *ctx, mapcache_cache *cache, mapcache_cfg *cfg) {
449 }
450 
451 /**
452  * \brief creates and initializes a mapcache_couchbase_cache
453  */
mapcache_cache_couchbase_create(mapcache_context * ctx)454 mapcache_cache* mapcache_cache_couchbase_create(mapcache_context *ctx) {
455    mapcache_cache_couchbase *cache = apr_pcalloc(ctx->pool,sizeof(mapcache_cache_couchbase));
456    if(!cache) {
457       ctx->set_error(ctx, 500, "failed to allocate couchbase cache");
458       return NULL;
459    }
460 
461    cache->cache.metadata = apr_table_make(ctx->pool, 3);
462    cache->cache.type = MAPCACHE_CACHE_COUCHBASE;
463    cache->cache.tile_get = _mapcache_cache_couchbase_get;
464    cache->cache.tile_exists = _mapcache_cache_couchbase_has_tile;
465    cache->cache.tile_set = _mapcache_cache_couchbase_set;
466    cache->cache.tile_delete = _mapcache_cache_couchbase_delete;
467    cache->cache.configuration_parse_xml = _mapcache_cache_couchbase_configuration_parse_xml;
468    cache->cache.configuration_post_config = _mapcache_cache_couchbase_configuration_post_config;
469    cache->host = NULL;
470    cache->username = NULL;
471    cache->password = NULL;
472    cache->bucket = NULL;
473 
474    return (mapcache_cache*)cache;
475 }
476 
477 #else
mapcache_cache_couchbase_create(mapcache_context * ctx)478 mapcache_cache* mapcache_cache_couchbase_create(mapcache_context *ctx) {
479   ctx->set_error(ctx,400,"COUCHBASE support not compiled in this version");
480   return NULL;
481 }
482 #endif
483 
484 /* vim: ai ts=3 sts=3 et sw=3
485 */
486