1 /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 /* 3 * Copyright 2013 Couchbase, Inc. 4 * 5 * Licensed under the Apache License, Version 2.0 (the "License"); 6 * you may not use this file except in compliance with the License. 7 * You may obtain a copy of the License at 8 * 9 * http://www.apache.org/licenses/LICENSE-2.0 10 * 11 * Unless required by applicable law or agreed to in writing, software 12 * distributed under the License is distributed on an "AS IS" BASIS, 13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 * See the License for the specific language governing permissions and 15 * limitations under the License. 16 */ 17 18 #ifndef LCB_CLCONFIG_H 19 #define LCB_CLCONFIG_H 20 21 #include "hostlist.h" 22 #include <list> 23 #include <lcbio/timer-ng.h> 24 #include <lcbio/timer-cxx.h> 25 26 /** @file */ 27 28 /** 29 * @defgroup lcb-confmon Cluster Configuration Management 30 * 31 * @brief Monitors the retrieval and application of new cluster topology maps 32 * (vBucket Configurations) 33 * 34 * @details 35 * This module attempts to implement the 'Configuration Provider' interface 36 * described at https://docs.google.com/document/d/1bSMt0Sj1uQtm0OYolQaJDJg4sASfoCEwU6_gjm1he8s/edit 37 * 38 * The model is fairly complex though significantly more maintainable and 39 * testable than the previous model. The basic idea is as follows: 40 * 41 * 42 * <ol> 43 * 44 * <li> 45 * There is a _Configuration Monitor_ object (Confmon) which acts 46 * as the configuration supervisor. It is responsible for returning 47 * configuration objects to those entities which request it. 48 * </li> 49 * 50 * <li> 51 * There are multiple _Configuration Provider_ (Provider) objects. 52 * These providers aggregate configurations from multiple sources and 53 * implement a common interface to: 54 * 55 * <ol> 56 * <li>Return a _quick_ configuration without fetching from network or disk 57 * (see Provider::get_cached())</i> 58 59 * <li>Schedule a refresh to retrieve the latest configuration from the 60 * network (see Provider::refresh())</li> 61 * 62 * <li>Notify the monitor that it has received a new configuration. The 63 * monitor itself will determine whether or not to accept the new 64 * configuration by examining the configuration and determining if it is more 65 * recent than the one currently in used. See lcb_confmon_set_next()</li> 66 * </ol></li> 67 * 68 * <li> 69 * _Configuration Info_ objects. These objects are refcounted wrappers 70 * around vbucket configuration handles. They have a refcount and also an 71 * integer which can be used to compare with other objects for 'freshness'. 72 * See ConfigInfo 73 * </li> 74 * 75 * <li> 76 * _Configuration Listeners_. These are registered with the global supervisor 77 * and are invoked whenever a new valid configuration is detected. This is 78 * really only ever used during bootstrap or testing where we are explicitly 79 * waiting for a configuration without having any actual commands to schedule. 80 * See Listener 81 * </li> 82 * </ol> 83 */ 84 85 /** 86 *@addtogroup lcb-confmon 87 *@{ 88 */ 89 90 namespace lcb { 91 namespace clconfig { 92 93 /** 94 * @brief Enumeration of the various config providers available. 95 * The type of methods available. These are enumerated in order of preference 96 */ 97 enum Method { 98 /** File-based "configcache" provider. Implemented in bc_file.c */ 99 CLCONFIG_FILE, 100 /** New-style config-over-memcached provider. Implemented in bc_cccp.c */ 101 CLCONFIG_CCCP, 102 /** Old-style streaming HTTP provider. Implemented in bc_http.c */ 103 CLCONFIG_HTTP, 104 /** Raw memcached provided */ 105 CLCONFIG_MCRAW, 106 /** Cluster administration provider. Static config with services */ 107 CLCONFIG_CLADMIN, 108 109 CLCONFIG_MAX, 110 111 /** Ephemeral source, used for tests */ 112 CLCONFIG_PHONY 113 }; 114 115 116 /** Event types propagated to listeners */ 117 enum EventType { 118 /** Called when a new configuration is being set in confmon */ 119 CLCONFIG_EVENT_GOT_NEW_CONFIG, 120 121 /** Called when _any_ configuration is received via set_enxt */ 122 CLCONFIG_EVENT_GOT_ANY_CONFIG, 123 124 /** Called when all providers have been tried */ 125 CLCONFIG_EVENT_PROVIDERS_CYCLED, 126 127 /** The monitor has stopped */ 128 CLCONFIG_EVENT_MONITOR_STOPPED 129 }; 130 131 132 /** @brief Possible confmon states */ 133 enum State { 134 /** The monitor is idle and not requesting a new configuration */ 135 CONFMON_S_INACTIVE = 0, 136 137 /** The monitor is actively requesting a configuration */ 138 CONFMON_S_ACTIVE = 1 << 0, 139 140 /** The monitor is fetching a configuration, but is in a throttle state */ 141 CONFMON_S_ITERGRACE = 1 << 1 142 }; 143 144 145 struct Provider; 146 struct Listener; 147 class ConfigInfo; 148 149 /** 150 * This object contains the information needed for libcouchbase to deal with 151 * when retrieving new configs. 152 */ 153 struct Confmon { 154 /** 155 * @brief Create a new configuration monitor. 156 * This function creates a new `confmon` object which can be used to manage 157 * configurations and their providers. 158 * 159 * @param settings pointer to LCB settings 160 * @param iot pointer socket IO routines 161 * @param instance LCB handle 162 * 163 * Once the confmon object has been created you may enable or disable various 164 * providers (see lcb_confmon_set_provider_active()). Once no more providers 165 * remain to be activated you should call lcb_confmon_prepare() once. Then 166 * call the rest of the functions. 167 */ 168 Confmon(lcb_settings* settings, lcbio_pTABLE iot, lcb_t instance); 169 void destroy() { delete this; } 170 ~Confmon(); 171 172 /** 173 * Get the provider following the current provider, or NULL if this is 174 * the last provider in the list. 175 * @param cur The current provider. 176 * @return The next provider, or NULL if no more providers remain. 177 */ 178 Provider *next_active(Provider *cur); 179 180 /** 181 * Gets the first active provider. 182 * @return the first provider, or NULL if no providers exist. 183 */ 184 Provider *first_active(); 185 186 /** 187 * Prepares the configuration monitor object for operations. This will insert 188 * all the enabled providers into a list. Call this function each time a 189 * provider has been enabled. 190 */ 191 void prepare(); 192 193 /** 194 * Set a given provider as being 'active'. This will activate the 195 * provider as well as call #prepare() to update the list. 196 * @param type The ID of the provider to activate 197 * @param enabled true for activate, false for deactivate 198 */ 199 void set_active(Method type, bool enabled); 200 201 /** 202 * @brief Request a configuration refresh 203 * 204 * Start traversing the list of current providers, requesting a new 205 * configuration for each. This function will asynchronously loop through all 206 * providers until one provides a new configuration. 207 * 208 * You may call #stop() to asynchronously break out of the loop. 209 * If the confmon is already in a refreshing state 210 * (i.e. #is_refreshing()) returns true then this function does 211 * nothing. 212 * 213 * This function is reentrant safe and may be called at any time. 214 * 215 * @see lcb_confmon_add_listener() 216 * @see #stop() 217 * @see #is_refreshing() 218 */ 219 void start(); 220 221 /** 222 * @brief Cancel a pending configuration refresh. 223 * 224 * Stops the monitor. This will call Provider::pause() for each active 225 * provider. Typically called before destruction or when a new configuration 226 * has been found. 227 * 228 * This function is safe to call anywhere. If the monitor is already stopped 229 * then this function does nothing. 230 * 231 * @see #start() 232 * @see #is_refreshing() 233 */ 234 void stop(); 235 236 /** 237 * @brief Check if the monitor is waiting for a new config from a provider 238 * @return true if refreshing, false if idle 239 */ 240 bool is_refreshing() const { 241 return (state & CONFMON_S_ACTIVE) != 0; 242 } 243 244 /** 245 * Get the current configuration 246 * @return The configuration 247 */ 248 ConfigInfo* get_config() const { 249 return config; 250 } 251 252 /** 253 * Get the last error which occurred on this object 254 * @return The last error 255 */ 256 lcb_error_t get_last_error() const { 257 return last_error; 258 } 259 260 /** 261 * @brief Get the current monitor state 262 * @return a set of flags consisting of @ref State values. 263 */ 264 int get_state() const { 265 return state; 266 } 267 268 void stop_real(); 269 void do_next_provider(); 270 int do_set_next(ConfigInfo*, bool notify_miss); 271 void invoke_listeners(EventType, ConfigInfo*); 272 273 /** 274 * @brief Indicate that a provider has failed and advance the monitor 275 * 276 * Indicate that the current provider has failed to obtain a new configuration. 277 * This is always called by a provider and should be invoked when the provider 278 * has encountered an internal error which caused it to be unable to fetch 279 * the configuration. 280 * 281 * Note that this function is safe to call from any provider at any time. If 282 * the provider is not the current provider then it is treated as an async 283 * push notification failure and ignored. This function is _not_ safe to call 284 * from consumers of providers 285 * 286 * Once this is called, the confmon instance will either roll over to the next 287 * provider or enter the inactive state depending on the configuration and 288 * whether the current provider is the last provider in the list. 289 * 290 * @param which reference to provider, which has been failed 291 * @param why error code 292 */ 293 void provider_failed(Provider *which, lcb_error_t why); 294 295 /** 296 * @brief Indicate that a provider has successfuly retrieved a configuration. 297 * 298 * Indicates that the provider has fetched a new configuration from the network 299 * and that confmon should attempt to propagate it. It has similar semantics 300 * to lcb_confmon_provider_failed() except that the second argument is a config 301 * object rather than an error code. The second argument must not be `NULL` 302 * 303 * The monitor will compare the new config against the current config. 304 * If the new config does not feature any changes from the current config then 305 * it is ignored and the confmon instance will proceed to the next provider. 306 * This is done through a direct call to provider_failed(provider, LCB_SUCCESS). 307 * 308 * This function should _not_ be called outside of an asynchronous provider's 309 * handler. 310 * 311 * @param which the provider which yielded the new configuration 312 * @param config the new configuration 313 */ 314 void provider_got_config(Provider *which, ConfigInfo* config); 315 316 /** 317 * Dump information about the monitor 318 * @param fp the file to which information should be written 319 */ 320 void dump(FILE *fp); 321 322 Provider* get_provider(Method m) const { 323 return all_providers[m]; 324 } 325 326 /** 327 * @brief Register a listener to be invoked on state changes and events 328 * 329 * Adds a 'listener' object to be called at each configuration update. The 330 * listener may co-exist with other listeners (though it should never be added 331 * twice). When a new configuration is received and accept, the listener's 332 * Listener::callback field will be invoked with it. 333 * 334 * The callback will continue to be invoked for each new configuration received 335 * until remove_listener is called. Note that the listener is not allocated 336 * by the confmon and its responsibility is the user's 337 * 338 * @param lsn the listener. The listener's contents are not copied into 339 * confmon and should thus remain valid until it is removed 340 */ 341 void add_listener(Listener* lsn); 342 343 /** 344 * @brief Unregister (and remove) a listener added via lcb_confmon_add_listener() 345 * @param lsn the listener 346 */ 347 void remove_listener(Listener *lsn); 348 349 /**Current provider. This provider may either fail or succeed. 350 * In either case unless the provider can provide us with a specific 351 * config which is newer than the one we have, it will roll over to the 352 * next provider. */ 353 Provider *cur_provider; 354 355 /** All providers we know about. Currently this means the 'builtin' providers */ 356 Provider* all_providers[CLCONFIG_MAX]; 357 358 /** The current configuration pointer. This contains the most recent accepted 359 * configuration */ 360 ConfigInfo * config; 361 362 typedef std::list<Listener*> ListenerList; 363 /** List of listeners for events */ 364 ListenerList listeners; 365 366 lcb_settings *settings; 367 lcb_error_t last_error; 368 lcbio_pTABLE iot; 369 370 /** This is the async handle for a reentrant start */ 371 lcb::io::Timer<Confmon, &Confmon::do_next_provider> as_start; 372 373 /** Async handle for a reentrant stop */ 374 lcb::io::Timer<Confmon, &Confmon::stop_real> as_stop; 375 376 /* CONFMON_S_* values. Used internally */ 377 int state; 378 379 /** Last time the provider was stopped. As a microsecond timestamp */ 380 lcb_uint64_t last_stop_us; 381 382 typedef std::list<Provider*> ProviderList; 383 ProviderList active_providers; 384 385 lcb_t instance; 386 size_t active_provider_list_id; 387 }; 388 389 /** 390 * The base structure of a provider. This structure is intended to be 391 * 'subclassed' by implementors. 392 */ 393 struct Provider { 394 Provider(Confmon*, Method type_); 395 396 /** Destroy the resources created by this provider. */ 397 virtual ~Provider(); 398 399 /** 400 * Get the current map known to this provider. This should not perform 401 * any blocking operations. Providers which use a push model may use 402 * this method as an asynchronous return value for a previously-received 403 * configuration. 404 */ 405 virtual ConfigInfo* get_cached() = 0; 406 407 408 /** 409 * Request a new configuration. This will be called by the manager when 410 * the cached configuration (i.e. 'get_cached') is deemed invalid. Thus 411 * this function should unconditionally try to schedule getting the 412 * newest configuration it can. When the configuration has been received, 413 * the provider may call provider_success or provider_failed. 414 * 415 * @note 416 * The PROVIDER is responsible for terminating its own 417 * process. In other words there is no safeguard within confmon itself 418 * against a provider taking an excessively long time; therefore a provider 419 * should implement a timeout mechanism of its choice to promptly deliver 420 * a success or failure. 421 */ 422 virtual lcb_error_t refresh() = 0; 423 424 /** 425 * Callback invoked to the provider to indicate that it should cease 426 * performing any "Active" configuration changes. Note that this is only 427 * a hint and a provider may perform its own hooking based on this. In any 428 * event receiving this callback is indicating that the provider will not 429 * be needed again in quite some time. How long this "time" is can range 430 * between 0 seconds and several minutes depending on how a user has 431 * configured the client. 432 * 433 * @return true if actually paused 434 */ 435 virtual bool pause() { 436 return false; 437 } 438 439 /** 440 * Called when a new configuration has been received. 441 * 442 * @param config the current configuration. 443 * Note that this should only update the server list and do nothing 444 * else. 445 */ 446 virtual void config_updated(lcbvb_CONFIG* config) { 447 (void)config; 448 } 449 450 /** 451 * Retrieve the list of nodes from this provider, if applicable 452 * 453 * @return A list of nodes, or NULL if the provider does not have a list 454 */ 455 virtual const lcb::Hostlist* get_nodes() const { 456 return NULL; 457 } 458 459 /** 460 * Call to change the configured nodes of this provider. 461 * 462 * @param l The list of nodes to apply 463 */ 464 virtual void configure_nodes(const lcb::Hostlist& l) { 465 (void)l; 466 } 467 468 /** 469 * Dump state information. This callback is optional 470 * 471 * @param f the file to write to 472 */ 473 virtual void dump(FILE *f) const { 474 (void)f; 475 } 476 477 void enable() { 478 enabled = 1; 479 } 480 481 virtual void enable(void *) { 482 lcb_assert("Must be implemented in subclass if used" && 0); 483 } 484 485 /** The type of provider */ 486 const Method type; 487 488 /** Whether this provider has been disabled/enabled explicitly by a user */ 489 bool enabled; 490 491 /** The parent manager object */ 492 Confmon *parent; 493 494 lcb_settings& settings() const { 495 return *parent->settings; 496 } 497 }; 498 499 Provider *new_cccp_provider(Confmon*); 500 Provider *new_file_provider(Confmon*); 501 Provider *new_http_provider(Confmon*); 502 Provider *new_mcraw_provider(Confmon*); 503 Provider *new_cladmin_provider(Confmon*); 504 505 506 /** @brief refcounted object encapsulating a vbucket config */ 507 class ConfigInfo { 508 public: 509 /** 510 * Creates a new configuration wrapper object containing the vbucket config 511 * pointed to by 'config'. Its initial refcount will be set to 1. 512 * 513 * @param vbc a newly parsed configuration 514 * @param origin the type of provider from which the config originated. 515 * @return a new ConfigInfo object. This should be incref()'d/decref()'d 516 * as needed. 517 */ 518 static ConfigInfo* create(lcbvb_CONFIG *vbc, Method origin) { 519 return new ConfigInfo(vbc, origin); 520 } 521 /** 522 * @brief Compares two info structures and determine which one is newer 523 * 524 * This function returns an integer less than 525 * zero, zero or greater than zero if the first argument is considered older 526 * than, equal to, or later than the second argument. 527 * 528 * @param config anoother config 529 * @see lcbvb_get_revision 530 * @see ConfigInfo::cmpclock 531 */ 532 int compare(const ConfigInfo& config); 533 534 /** 535 * @brief Increment the refcount on a config object 536 */ 537 void incref() { refcount++; } 538 539 /** 540 * @brief Decrement the refcount on a config object. 541 * Decrement the refcount. If the internal refcount reaches 0 then the internal 542 * members (including the vbucket config handle itself) will be freed. 543 */ 544 void decref() { 545 if (!--refcount) { 546 delete this; 547 } 548 } 549 550 operator lcbvb_CONFIG*() const { 551 return vbc; 552 } 553 554 Method get_origin() const { 555 return origin; 556 } 557 558 /** Actual configuration */ 559 lcbvb_CONFIG* vbc; 560 561 private: 562 ConfigInfo(lcbvb_CONFIG *vbc, Method origin); 563 564 ~ConfigInfo(); 565 566 /** Comparative clock with which to compare */ 567 uint64_t cmpclock; 568 569 /** Reference counter */ 570 unsigned int refcount; 571 572 /** Origin provider type which produced this config */ 573 Method origin; 574 }; 575 576 /** 577 * @brief Listener for events 578 * One or more listeners may be installed into the confmon which will have 579 * a callback invoked on significant vbucket events. See clconfig_event_t 580 * for a variety of events the listener can know. 581 */ 582 struct Listener { 583 virtual ~Listener() { 584 } 585 586 /** Linked list node */ 587 lcb_list_t ll; 588 589 /** 590 * Callback invoked for significant events 591 * 592 * @param event the event which took place 593 * @param config the configuration associated with the event. Note that 594 * `config` may also be NULL 595 */ 596 virtual void clconfig_lsn(EventType event, ConfigInfo *config) = 0; 597 }; 598 599 /* Method-specific setup methods.. */ 600 601 /** 602 * @name File Provider-specific APIs 603 * @{ 604 */ 605 606 /** 607 * Sets the input/output filename for the file provider. This also enables 608 * the file provider. 609 * @param p the provider 610 * @param f the filename (if NULL, a temporary filename will be created) 611 * @param ro whether the client will never modify the file 612 * @return true on success, false on failure. 613 */ 614 bool file_set_filename(Provider *p, const char *f, bool ro); 615 616 /** 617 * Retrieve the filename for the provider 618 * @param p The provider of type CLCONFIG_FILE 619 * @return the current filename being used. 620 */ 621 const char* file_get_filename(Provider *p); 622 void file_set_readonly(Provider *p, bool val); 623 /**@}*/ 624 625 /** 626 * @name HTTP Provider-specific APIs 627 * @{ 628 */ 629 630 /** 631 * Get the socket representing the current REST connection to the cluster 632 * (if applicable) 633 * @param p The provider of type CLCONFIG_HTTP 634 * @return 635 */ 636 const lcbio_SOCKET* http_get_conn(const Provider *p); 637 638 static inline const lcbio_SOCKET* http_get_conn(Confmon *c) { 639 return http_get_conn(c->get_provider(CLCONFIG_HTTP)); 640 } 641 642 /** 643 * Get the hostname for the current REST connection to the cluster 644 * @param p The provider of type CLCONFIG_HTTP 645 * @return 646 */ 647 const lcb_host_t * http_get_host(const Provider *p); 648 static inline const lcb_host_t* http_get_host(Confmon *c) { 649 return http_get_host(c->get_provider(CLCONFIG_HTTP)); 650 } 651 /**@}*/ 652 653 /** 654 * @name CCCP Provider-specific APIs 655 * @{ 656 */ 657 658 /** 659 * Note, to initialize the CCCP provider, you should use 660 * cccp->enable(instance); 661 */ 662 663 /** 664 * @brief Notify the CCCP provider about a new configuration from a 665 * `NOT_MY_VBUCKET` response 666 * 667 * This should be called by the packet handler when a configuration has been 668 * received as a payload to a response with the error of `NOT_MY_VBUCKET`. 669 * 670 * @param provider The CCCP provider 671 * @param host The hostname (without the port) on which the packet was received 672 * @param data The configuration JSON blob 673 * @return LCB_SUCCESS, or an error code if the configuration could not be 674 * set 675 */ 676 lcb_error_t 677 cccp_update(Provider *provider, const char *host, const char *data); 678 679 /** 680 * @brief Notify the CCCP provider about a configuration received from a 681 * `CMD_GET_CLUSTER_CONFIG` response. 682 * 683 * @param cookie The cookie object attached to the packet 684 * @param err The error code for the reply 685 * @param bytes The payload pointer 686 * @param nbytes Size of payload 687 * @param origin Host object from which the packet was received 688 */ 689 void cccp_update(const void *cookie, lcb_error_t err, 690 const void *bytes, size_t nbytes, const lcb_host_t *origin); 691 692 /**@}*/ 693 694 /**@name Raw Memcached (MCRAW) Provider-specific APIs 695 * @{*/ 696 /**@}*/ 697 /**@}*/ 698 699 } // clconfig 700 } // lcb 701 #endif /* LCB_CLCONFIG_H */ 702