1 /* Copyright (C) 2014 InfiniDB, Inc. 2 3 This program is free software; you can redistribute it and/or 4 modify it under the terms of the GNU General Public License 5 as published by the Free Software Foundation; version 2 of 6 the License. 7 8 This program is distributed in the hope that it will be useful, 9 but WITHOUT ANY WARRANTY; without even the implied warranty of 10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 GNU General Public License for more details. 12 13 You should have received a copy of the GNU General Public License 14 along with this program; if not, write to the Free Software 15 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 16 MA 02110-1301, USA. */ 17 18 /****************************************************************************** 19 * $Id: extentmap.h 1936 2013-07-09 22:10:29Z dhall $ 20 * 21 *****************************************************************************/ 22 23 /** @file 24 * class ExtentMap 25 */ 26 27 #ifndef _EXTENTMAP_H_ 28 #define _EXTENTMAP_H_ 29 30 #include <sys/types.h> 31 #include <vector> 32 #include <set> 33 #ifdef _MSC_VER 34 #include <unordered_map> 35 #else 36 #include <tr1/unordered_map> 37 #endif 38 //#define NDEBUG 39 #include <cassert> 40 #include <boost/interprocess/shared_memory_object.hpp> 41 #include <boost/interprocess/mapped_region.hpp> 42 43 #include "shmkeys.h" 44 #include "brmtypes.h" 45 #include "mastersegmenttable.h" 46 #include "undoable.h" 47 48 #include "brmshmimpl.h" 49 #include "exceptclasses.h" 50 51 #ifdef NONE 52 #undef NONE 53 #endif 54 #ifdef READ 55 #undef READ 56 #endif 57 #ifdef WRITE 58 #undef WRITE 59 #endif 60 61 #if defined(_MSC_VER) && defined(xxxEXTENTMAP_DLLEXPORT) 62 #define EXPORT __declspec(dllexport) 63 #else 64 #define EXPORT 65 #endif 66 67 namespace oam 68 { 69 typedef std::vector<uint16_t> DBRootConfigList; 70 } 71 72 namespace idbdatafile 73 { 74 class IDBDataFile; 75 } 76 77 namespace BRM 78 { 79 80 // assumed column width when calculating dictionary store extent size 81 #define DICT_COL_WIDTH 8 82 83 // valid values for EMEntry.status 84 const int16_t EXTENTSTATUSMIN(0); // equal to minimum valid status value 85 const int16_t EXTENTAVAILABLE(0); 86 const int16_t EXTENTUNAVAILABLE(1); 87 const int16_t EXTENTOUTOFSERVICE(2); 88 const int16_t EXTENTSTATUSMAX(2); // equal to maximum valid status value 89 90 enum partition_type_enum 91 { 92 PART_DEFAULT = 0, 93 PART_CASUAL, 94 PART_RANGE 95 }; 96 typedef partition_type_enum EMPartitionType_t; 97 typedef int64_t RangePartitionData_t; 98 99 const char CP_INVALID = 0; 100 const char CP_UPDATING = 1; 101 const char CP_VALID = 2; 102 103 struct EMCasualPartition_struct 104 { 105 RangePartitionData_t hi_val; // This needs to be reinterpreted as unsigned for uint64_t column types. 106 RangePartitionData_t lo_val; 107 int32_t sequenceNum; 108 char isValid; //CP_INVALID - No min/max and no DML in progress. CP_UPDATING - Update in progress. CP_VALID- min/max is valid 109 EXPORT EMCasualPartition_struct(); 110 EXPORT EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum); 111 EXPORT EMCasualPartition_struct(const EMCasualPartition_struct& em); 112 EXPORT EMCasualPartition_struct& operator= (const EMCasualPartition_struct& em); 113 }; 114 typedef EMCasualPartition_struct EMCasualPartition_t; 115 116 struct EMPartition_struct 117 { 118 EMCasualPartition_t cprange; 119 }; 120 typedef EMPartition_struct EMPartition_t; 121 122 struct EMEntry 123 { 124 InlineLBIDRange range; 125 int fileID; 126 uint32_t blockOffset; 127 HWM_t HWM; 128 uint32_t partitionNum; // starts at 0 129 uint16_t segmentNum; // starts at 0 130 uint16_t dbRoot; // starts at 1 to match Columnstore.xml 131 uint16_t colWid; 132 int16_t status; //extent avail for query or not, or out of service 133 EMPartition_t partition; 134 EXPORT EMEntry(); 135 EXPORT EMEntry(const EMEntry&); 136 EXPORT EMEntry& operator= (const EMEntry&); 137 EXPORT bool operator< (const EMEntry&) const; 138 }; 139 140 // Bug 2989, moved from joblist 141 struct ExtentSorter 142 { operatorExtentSorter143 bool operator()(const EMEntry& e1, const EMEntry& e2) 144 { 145 if (e1.dbRoot < e2.dbRoot) 146 return true; 147 148 if (e1.dbRoot == e2.dbRoot && e1.partitionNum < e2.partitionNum) 149 return true; 150 151 if (e1.dbRoot == e2.dbRoot && e1.partitionNum == e2.partitionNum && e1.blockOffset < e2.blockOffset) 152 return true; 153 154 if (e1.dbRoot == e2.dbRoot && e1.partitionNum == e2.partitionNum && e1.blockOffset == e2.blockOffset && e1.segmentNum < e2.segmentNum) 155 return true; 156 157 return false; 158 } 159 }; 160 161 class ExtentMapImpl 162 { 163 public: ~ExtentMapImpl()164 ~ExtentMapImpl(){}; 165 166 static ExtentMapImpl* makeExtentMapImpl(unsigned key, off_t size, bool readOnly = false); refreshShm()167 static void refreshShm() 168 { 169 if (fInstance) 170 { 171 delete fInstance; 172 fInstance = NULL; 173 } 174 } 175 grow(unsigned key,off_t size)176 inline void grow(unsigned key, off_t size) 177 #ifdef NDEBUG 178 { 179 fExtMap.grow(key, size); 180 } 181 #else 182 { 183 int rc = fExtMap.grow(key, size); 184 idbassert(rc == 0); 185 } 186 #endif makeReadOnly()187 inline void makeReadOnly() 188 { 189 fExtMap.setReadOnly(); 190 } clear(unsigned key,off_t size)191 inline void clear(unsigned key, off_t size) 192 { 193 fExtMap.clear(key, size); 194 } swapout(BRMShmImpl & rhs)195 inline void swapout(BRMShmImpl& rhs) 196 { 197 fExtMap.swap(rhs); 198 rhs.destroy(); 199 } key()200 inline unsigned key() const 201 { 202 return fExtMap.key(); 203 } 204 get()205 inline EMEntry* get() const 206 { 207 return reinterpret_cast<EMEntry*>(fExtMap.fMapreg.get_address()); 208 } 209 210 private: 211 ExtentMapImpl(unsigned key, off_t size, bool readOnly = false); 212 ExtentMapImpl(const ExtentMapImpl& rhs); 213 ExtentMapImpl& operator=(const ExtentMapImpl& rhs); 214 215 BRMShmImpl fExtMap; 216 217 static boost::mutex fInstanceMutex; 218 static ExtentMapImpl* fInstance; 219 }; 220 221 class FreeListImpl 222 { 223 public: ~FreeListImpl()224 ~FreeListImpl(){}; 225 226 static FreeListImpl* makeFreeListImpl(unsigned key, off_t size, bool readOnly = false); refreshShm()227 static void refreshShm() 228 { 229 if (fInstance) 230 { 231 delete fInstance; 232 fInstance = NULL; 233 } 234 } 235 grow(unsigned key,off_t size)236 inline void grow(unsigned key, off_t size) 237 #ifdef NDEBUG 238 { 239 fFreeList.grow(key, size); 240 } 241 #else 242 { 243 int rc = fFreeList.grow(key, size); 244 idbassert(rc == 0); 245 } 246 #endif makeReadOnly()247 inline void makeReadOnly() 248 { 249 fFreeList.setReadOnly(); 250 } clear(unsigned key,off_t size)251 inline void clear(unsigned key, off_t size) 252 { 253 fFreeList.clear(key, size); 254 } swapout(BRMShmImpl & rhs)255 inline void swapout(BRMShmImpl& rhs) 256 { 257 fFreeList.swap(rhs); 258 rhs.destroy(); 259 } key()260 inline unsigned key() const 261 { 262 return fFreeList.key(); 263 } 264 get()265 inline InlineLBIDRange* get() const 266 { 267 return reinterpret_cast<InlineLBIDRange*>(fFreeList.fMapreg.get_address()); 268 } 269 270 private: 271 FreeListImpl(unsigned key, off_t size, bool readOnly = false); 272 FreeListImpl(const FreeListImpl& rhs); 273 FreeListImpl& operator=(const FreeListImpl& rhs); 274 275 BRMShmImpl fFreeList; 276 277 static boost::mutex fInstanceMutex; 278 static FreeListImpl* fInstance; 279 }; 280 281 /** @brief This class encapsulates the extent map functionality of the system 282 * 283 * This class encapsulates the extent map functionality of the system. It 284 * is currently implemented in the quickest-to-write (aka dumb) way to 285 * get something working into the hands of the other developers ASAP. 286 * The Extent Map shared data should be implemented in a more scalable 287 * structure such as a tree or hash table. 288 */ 289 class ExtentMap : public Undoable 290 { 291 public: 292 EXPORT ExtentMap(); 293 EXPORT ~ExtentMap(); 294 295 /** @brief Loads the ExtentMap entries from a file 296 * 297 * Loads the ExtentMap entries from a file. This will 298 * clear out any existing entries. The intention is that before 299 * the system starts, an external tool instantiates a single Extent 300 * Map and loads the stored entries. 301 * @param filename The file to load from. 302 * @note Throws an ios_base::failure exception on an IO error, runtime_error 303 * if the file "looks" bad. 304 */ 305 EXPORT void load(const std::string& filename, bool fixFL = false); 306 307 /** @brief Saves the ExtentMap entries to a file 308 * 309 * Saves the ExtentMap entries to a file. 310 * @param filename The file to save to. 311 */ 312 EXPORT void save(const std::string& filename); 313 314 // @bug 1509. Added new version of lookup below. 315 /** @brief Returns the first and last LBID in the range for a given LBID 316 * 317 * Get the first and last LBID for the extent that contains the given LBID. 318 * @param LBID (in) The lbid to search for 319 * @param firstLBID (out) The first lbid for the extent 320 * @param lastLBID (out) the last lbid for the extent 321 * @return 0 on success, -1 on error 322 */ 323 EXPORT int lookup(LBID_t LBID, LBID_t& firstLBID, LBID_t& lastLBID); 324 325 // @bug 1055+. New functions added for multiple files per OID enhancement. 326 327 /** @brief Look up the OID and file block offset assiciated with an LBID 328 * 329 * Look up the OID and file block offset assiciated with an LBID 330 * @param LBID (in) The lbid to search for 331 * @param OID (out) The OID associated with lbid 332 * @param dbRoot (out) The db root containing the LBID 333 * @param partitionNum (out) The partition containing the LBID 334 * @param segmentNum (out) The segment containing the LBID 335 * @param fileBlockOffset (out) The file block offset associated 336 * with LBID 337 * @return 0 on success, -1 on error 338 */ 339 EXPORT int lookupLocal(LBID_t LBID, int& OID, uint16_t& dbRoot, uint32_t& partitionNum, uint16_t& segmentNum, uint32_t& fileBlockOffset); 340 341 /** @brief Look up the LBID associated with a given OID, offset, partition, and segment. 342 * 343 * Look up the LBID associated with a given OID, offset, partition, and segment. 344 * @param OID (in) The OID to look up 345 * @param fileBlockOffset (in) The file block offset 346 * @param partitionNum (in) The partition containing the lbid 347 * @param segmentNum (in) The segement containing the lbid 348 * @param LBID (out) The LBID associated with the given offset of the OID. 349 * @return 0 on success, -1 on error 350 */ 351 EXPORT int lookupLocal(int OID, uint32_t partitionNum, uint16_t segmentNum, uint32_t fileBlockOffset, LBID_t& LBID); 352 353 /** @brief Look up the LBID associated with a given dbroot, OID, offset, 354 * partition, and segment. 355 * 356 * Look up LBID associated with a given OID, offset, partition, and segment. 357 * @param OID (in) The OID to look up 358 * @param fileBlockOffset (in) The file block offset 359 * @param partitionNum (in) The partition containing the lbid 360 * @param segmentNum (in) The segement containing the lbid 361 * @param LBID (out) The LBID associated with the given offset of the OID. 362 * @return 0 on success, -1 on error 363 */ 364 EXPORT int lookupLocal_DBroot(int OID, uint16_t dbroot, 365 uint32_t partitionNum, uint16_t segmentNum, uint32_t fileBlockOffset, 366 LBID_t& LBID); 367 368 // @bug 1055-. 369 370 /** @brief Look up the starting LBID associated with a given OID, 371 * partition, segment, and offset. 372 * 373 * @param OID (in) The OID to look up 374 * @param partitionNum (in) The partition containing the lbid 375 * @param segmentNum (in) The segement containing the lbid 376 * @param fileBlockOffset (in) The file block offset 377 * @param LBID (out) The starting LBID associated with the extent 378 * containing the given offset 379 * @return 0 on success, -1 on error 380 */ 381 int lookupLocalStartLbid(int OID, 382 uint32_t partitionNum, 383 uint16_t segmentNum, 384 uint32_t fileBlockOffset, 385 LBID_t& LBID); 386 387 /** @brief Get a complete list of LBID ranges assigned to an OID 388 * 389 * Get a complete list of LBID ranges assigned to an OID. 390 */ 391 EXPORT void lookup(OID_t oid, LBIDRange_v& ranges); 392 393 /** @brief Allocate a "stripe" of extents for columns in a table (in DBRoot) 394 * 395 * If this is the first extent for the OID/DBRoot, it will start at 396 * file offset 0. If space for the OID already exists, the new 397 * extent will "logically" be appended to the end of the already- 398 * allocated space, although the extent may reside in a different 399 * physical file as indicated by dbRoot, partition, and segment. 400 * Partition and segment numbers are 0 based, dbRoot is 1 based. 401 * 402 * Allocate a "stripe" of extents for the specified columns and DBRoot 403 * @param cols (in) List of column OIDs and column widths 404 * @param dbRoot (in) DBRoot for requested extents. 405 * @param partitionNum (in/out) Partition number in file path. 406 * If allocating OID's first extent for this DBRoot, then 407 * partitionNum is input, else it is an output arg. 408 * @param segmentNum (out) Segment number selected for new extents. 409 * @param extents (out) list of lbids, numBlks, and fbo for new extents 410 * @return 0 on success, -1 on error 411 */ 412 EXPORT void createStripeColumnExtents( 413 const std::vector<CreateStripeColumnExtentsArgIn>& cols, 414 uint16_t dbRoot, 415 uint32_t& partitionNum, 416 uint16_t& segmentNum, 417 std::vector<CreateStripeColumnExtentsArgOut>& extents); 418 419 /** @brief Allocates an extent for a column file 420 * 421 * Allocates an extent for the specified OID and DBroot. 422 * If this is the first extent for the OID/DBRoot, it will start at 423 * file offset 0. If space for the OID already exists, the new 424 * extent will "logically" be appended to the end of the already- 425 * allocated space, although the extent may reside in a different 426 * physical file as indicated by dbRoot, partition, and segment. 427 * Partition and segment numbers are 0 based, dbRoot is 1 based. 428 * 429 * @param OID (in) The OID requesting the extent. 430 * @param colWidth (in) Column width of the OID. 431 * @param dbRoot (in) DBRoot where extent is to be added. 432 * @param colDataType (in) the column type 433 * @param partitionNum (in/out) Partition number in file path. 434 * If allocating OID's first extent for this DBRoot, then 435 * partitionNum is input, else it is an output arg. 436 * @param segmentNum (out) Segment number assigned to the extent. 437 * @param lbid (out) The first LBID of the extent created. 438 * @param allocdsize (out) The total number of LBIDs allocated. 439 * @param startBlockOffset (out) The first block of the extent created. 440 * @param useLock Grab ExtentMap and FreeList WRITE lock to perform work 441 */ 442 // @bug 4091: To be deprecated as public function. Should just be a 443 // private function used by createStripeColumnExtents(). 444 EXPORT void createColumnExtent_DBroot(int OID, 445 uint32_t colWidth, 446 uint16_t dbRoot, 447 execplan::CalpontSystemCatalog::ColDataType colDataType, 448 uint32_t& partitionNum, 449 uint16_t& segmentNum, 450 LBID_t& lbid, 451 int& allocdsize, 452 uint32_t& startBlockOffset, 453 bool useLock = true); 454 455 /** @brief Allocates extent for exact file that is specified 456 * 457 * Allocates an extent for the exact file specified by OID, DBRoot, 458 * partition, and segment. 459 * If this is the first extent for the OID/DBRoot, it will start at 460 * file offset 0. If space for the OID already exists, the new 461 * extent will "logically" be appended to the end of the already- 462 * allocated space. 463 * Partition and segment numbers are 0 based, dbRoot is 1 based. 464 * 465 * @param OID (in) The OID requesting the extent. 466 * @param colWidth (in) Column width of the OID. 467 * @param dbRoot (in) DBRoot where extent is to be added. 468 * @param partitionNum (in) Partition number in file path. 469 * If allocating OID's first extent for this DBRoot, then 470 * partitionNum is input, else it is an output arg. 471 * @param segmentNum (in) Segment number in file path. 472 * If allocating OID's first extent for this DBRoot, then 473 * segmentNum is input, else it is an output arg. 474 * @param colDataType (in) the column type 475 * @param lbid (out) The first LBID of the extent created. 476 * @param allocdSize (out) The total number of LBIDs allocated. 477 * @param startBlockOffset (out) The first block of the extent created. 478 */ 479 EXPORT void createColumnExtentExactFile(int OID, 480 uint32_t colWidth, 481 uint16_t dbRoot, 482 uint32_t partitionNum, 483 uint16_t segmentNum, 484 execplan::CalpontSystemCatalog::ColDataType colDataType, 485 LBID_t& lbid, 486 int& allocdsize, 487 uint32_t& startBlockOffset); 488 489 /** @brief Allocates an extent for a dictionary store file 490 * 491 * Allocates an extent for the specified dictionary store OID, 492 * dbRoot, partition number, and segment number. These should 493 * correlate with those belonging to the corresponding token file. 494 * The first extent for each store file will start at file offset 0. 495 * Other extents will be appended to the end of the already- 496 * allocated space for the same store file. 497 * Partition and segment numbers are 0 based, dbRoot is 1 based. 498 * 499 * @param OID (in) The OID requesting the extent. 500 * @param dbRoot (in) DBRoot to assign to the extent. 501 * @param partitionNum (in) Partition number to assign to the extent. 502 * @param segmentNum (in) Segment number to assign to the extent. 503 * @param lbid (out) The first LBID of the extent created. 504 * @param allocdsize (out) The total number of LBIDs allocated. 505 */ 506 EXPORT void createDictStoreExtent(int OID, 507 uint16_t dbRoot, 508 uint32_t partitionNum, 509 uint16_t segmentNum, 510 LBID_t& lbid, 511 int& allocdsize); 512 513 /** @brief Rollback (delete) a set of extents for the specified OID. 514 * 515 * Deletes all the extents that logically follow the specified 516 * column extent; and sets the HWM for the specified extent. 517 * @param oid OID of the extents to be deleted. 518 * @param partitionNum Last partition to be kept. 519 * @param segmentNum Last segment in partitionNum to be kept. 520 * @param hwm HWM to be assigned to the last extent that is kept. 521 */ 522 EXPORT void rollbackColumnExtents(int oid, 523 uint32_t partitionNum, 524 uint16_t segmentNum, 525 HWM_t hwm); 526 527 /** @brief Rollback (delete) set of extents for specified OID & DBRoot. 528 * 529 * Deletes all the extents that logically follow the specified 530 * column extent; and sets the HWM for the specified extent. 531 * @param oid OID of the extents to be deleted. 532 * @param bDeleteAll Flag indicates if all extents for oid and dbroot are 533 * to be deleted, else part#, seg#, and HWM are used. 534 * @param dbRoot DBRoot of the extents to be deleted. 535 * @param partitionNum Last partition to be kept. 536 * @param segmentNum Last segment in partitionNum to be kept. 537 * @param hwm HWM to be assigned to the last extent that is kept. 538 */ 539 EXPORT void rollbackColumnExtents_DBroot(int oid, 540 bool bDeleteAll, 541 uint16_t dbRoot, 542 uint32_t partitionNum, 543 uint16_t segmentNum, 544 HWM_t hwm); 545 546 /** @brief delete of column extents for the specified extents. 547 * 548 * Deletes the extents that logically follow the specified 549 * column extent in extentsInfo. It use the same algorithm as in 550 * rollbackColumnExtents. 551 * @param extentInfo the information for extents 552 */ 553 EXPORT void deleteEmptyColExtents(const ExtentsInfoMap_t& extentsInfo); 554 555 /** @brief delete of dictionary extents for the specified extents. 556 * 557 * Arguments specify the last stripe for all the oids. Any extents after this are 558 * deleted. The hwm's of the extents in the last stripe are updated 559 * based on the hwm in extentsInfo. It use the same algorithm as in 560 * rollbackDictStoreExtents. 561 * @param extentInfo the information for extents to be resetted 562 */ 563 EXPORT void deleteEmptyDictStoreExtents(const ExtentsInfoMap_t& extentsInfo); 564 565 /** @brief Rollback (delete) a set of dict store extents for an OID. 566 * 567 * Arguments specify the last stripe. Any extents after this are 568 * deleted. The hwm's of the extents in the last stripe are updated 569 * based on the contents of the hwm vector. If hwms is a partial list, 570 * (as in the first stripe of a partition), then any extents in sub- 571 * sequent segment files for that partition are deleted. 572 * @param oid OID of the extents to be deleted or updated. 573 * @param partitionNum Last partition to be kept. 574 * @param hwms Vector of hwms for the last partition to be kept. 575 */ 576 EXPORT void rollbackDictStoreExtents(int oid, 577 uint32_t partitionNum, 578 const std::vector<HWM_t>& hwms); 579 580 /** @brief Rollback (delete) a set of dict store extents for an OID & DBRoot 581 * 582 * Arguments specify the last stripe. Any extents after this are 583 * deleted. The hwm's of the extents in the last stripe are updated 584 * based on the contents of the hwm vector. If hwms is a partial list, 585 * (as in the first stripe of a partition), then any extents in sub- 586 * sequent segment files for that partition are deleted. If hwms is empty 587 * then all the extents in dbRoot are deleted. 588 * @param oid OID of the extents to be deleted or updated. 589 * @param dbRoot DBRoot of the extents to be deleted. 590 * @param partitionNum Last partition to be kept. 591 * @param segNums Vector of segment files in last partition to be kept. 592 * @param hwms Vector of hwms for the last partition to be kept. 593 */ 594 EXPORT void rollbackDictStoreExtents_DBroot(int oid, 595 uint16_t dbRoot, 596 uint32_t partitionNum, 597 const std::vector<uint16_t>& segNums, 598 const std::vector<HWM_t>& hwms); 599 600 /** @brief Deallocates all extents associated with OID 601 * 602 * Deallocates all extents associated with OID 603 * @param OID The OID to delete 604 */ 605 EXPORT void deleteOID(int OID); 606 607 /** @brief Deallocates all extents associated with each OID 608 * 609 * Deallocates all extents associated with each OID 610 * @param OIDs The OIDs to delete 611 */ 612 EXPORT void deleteOIDs(const OidsMap_t& OIDs); 613 614 /** @brief Check if any of the given partitions is the last one of a DBroot 615 * 616 * This is for partitioning operations to use. The last partition of a DBroot 617 * can not be dropped or disabled. 618 * 619 * @param OID (in) The OID 620 * @param partitionNums (in) The logical partition numbers to check. 621 * @return true if any of the partitions in the set is the last partition of 622 * a DBroot. 623 */ 624 625 /** @brief Gets the last local high water mark of an OID for a given dbRoot 626 * 627 * Get last local high water mark of an OID for a given dbRoot, relative to 628 * a segment file. The partition and segment numbers for the pertinent 629 * segment are also returned. 630 631 * @param OID (in) The OID 632 * @param dbRoot (in) The relevant DBRoot 633 * @param partitionNum (out) The relevant partition number 634 * @param segmentNum (out) The relevant segment number 635 * @param status (out) State of the extent (Available, OutOfService, etc) 636 * @param bFound (out) Indicates whether an extent was found for dbRoot 637 * @return The last file block number written to in the last 638 * partition/segment file for the given OID. 639 */ 640 EXPORT HWM_t getLastHWM_DBroot(int OID, uint16_t dbRoot, 641 uint32_t& partitionNum, uint16_t& segmentNum, 642 int& status, bool& bFound); 643 644 /** @brief Gets the current high water mark of an OID,partition,segment 645 * 646 * Get current local high water mark of an OID, partition, segment; 647 * where HWM is relative to the specific segment file. 648 * @param OID (in) The OID 649 * @param partitionNum (in) The relevant partition number 650 * @param segmentNum (in) The relevant segment number 651 * @param status (out) State of the extent (Available, OutOfService, etc) 652 * @return The last file block number written to in the specified 653 * partition/segment file for the given OID. 654 */ 655 EXPORT HWM_t getLocalHWM(int OID, uint32_t partitionNum, 656 uint16_t segmentNum, int& status); 657 658 /** @brief Sets the current high water mark of an OID,partition,segment 659 * 660 * Sets the current local high water mark of an OID, partition, segment; 661 * where HWM is relative to the specific segment file. 662 * @param OID The OID 663 * @param partitionNum (in) The relevant partition number 664 * @param segmentNum (in) The relevant segment number 665 * @param HWM The high water mark to record 666 */ 667 EXPORT void setLocalHWM(int OID, uint32_t partitionNum, 668 uint16_t segmentNum, HWM_t HWM, bool firstNode, 669 bool uselock = true); 670 671 EXPORT void bulkSetHWM(const std::vector<BulkSetHWMArg>&, bool firstNode); 672 673 EXPORT void bulkUpdateDBRoot(const std::vector<BulkUpdateDBRootArg>&); 674 675 /** @brief Get HWM information about last segment file for each DBRoot 676 * assigned to a specific PM. 677 * 678 * Vector will contain an entry for each DBRoot. If no "available" extents 679 * are found for a DBRoot, then totalBlocks will be 0 (and hwmExtentIndex 680 * will be -1) for that DBRoot. 681 * @param OID The oid of interest. 682 * @param pmNumber The PM number of interest. 683 * @param emDbRootHwmInfos The vector of DbRoot/HWM related objects. 684 */ 685 EXPORT void getDbRootHWMInfo(int OID, uint16_t pmNumber, 686 EmDbRootHWMInfo_v& emDbRootHwmInfos); 687 688 /** @brief Get the status (AVAILABLE, OUTOFSERVICE, etc) for the 689 * segment file represented by the specified OID, part# and seg#. 690 * 691 * Unlike many of the other DBRM functions, this function does 692 * not throw an exception if no extent is found; the "found" 693 * flag indicates whether an extent was found or not. 694 * 695 * @param oid (in) The OID of interest 696 * @param partitionNum (in) The partition number of interest 697 * @param segmentNum (in) The segment number of interest 698 * @param bFound (out) Indicates if extent was found or not 699 * @param status (out) The state of the extents in the specified 700 * segment file. 701 */ 702 EXPORT void getExtentState(int OID, uint32_t partitionNum, 703 uint16_t segmentNum, bool& bFound, int& status); 704 705 /** @brief Gets the extents of a given OID 706 * 707 * Gets the extents of a given OID. The returned entries will 708 * be NULL-terminated and will have to be destroyed individually 709 * using delete. 710 * @note Untested 711 * @param OID (in) The OID to get the extents for. 712 * @param entries (out) A snapshot of the OID's Extent Map entries 713 * sorted by starting LBID; note that The Real Entries can change at 714 * any time. 715 * @param sorted (in) indicates if output is to be sorted 716 * @param notFoundErr (in) indicates if no extents is considered an err 717 * @param incOutOfService (in) include/exclude out of service extents 718 */ 719 EXPORT void getExtents(int OID, std::vector<struct EMEntry>& entries, 720 bool sorted = true, bool notFoundErr = true, 721 bool incOutOfService = false); 722 723 /** @brief Gets the extents of a given OID under specified dbroot 724 * 725 * Gets the extents of a given OID under specified dbroot. The returned entries will 726 * be NULL-terminated and will have to be destroyed individually 727 * using delete. 728 * @param OID (in) The OID to get the extents for. 729 * @param entries (out) A snapshot of the OID's Extent Map entries for the dbroot 730 * @param dbroot (in) the specified dbroot 731 */ 732 EXPORT void getExtents_dbroot(int OID, std::vector<struct EMEntry>& entries, 733 const uint16_t dbroot); 734 735 /** @brief Gets the number of extents for the specified OID and DBRoot 736 * 737 * @param OID (in) The OID of interest 738 * @param dbroot (in) The DBRoot of interest 739 * @param incOutOfService (in) include/exclude out of service extents 740 * @param numExtents (out) number of extents found for OID and dbroot 741 * @return 0 on success, non-0 on error (see brmtypes.h) 742 */ 743 EXPORT void getExtentCount_dbroot(int OID, uint16_t dbroot, 744 bool incOutOfService, uint64_t& numExtents); 745 746 /** @brief Gets the size of an extent in rows 747 * 748 * Gets the size of an extent in rows. 749 * @return The number of rows in an extent. 750 */ 751 EXPORT unsigned getExtentSize(); //dmc-consider deprecating 752 EXPORT unsigned getExtentRows(); 753 754 /** @brief Gets the DBRoot for the specified system catalog OID 755 * 756 * Function should only be called for System Catalog OIDs, as it assumes 757 * the OID is fully contained on a single DBRoot, returning the first 758 * DBRoot found. This only makes sence for a System Catalog 759 * OID, because all other column OIDs can span multiple DBRoots. 760 * 761 * @param oid The system catalog OID 762 * @param dbRoot (out) the DBRoot holding the system catalog OID 763 */ 764 EXPORT void getSysCatDBRoot(OID_t oid, uint16_t& dbRoot); 765 766 /** @brief Delete a Partition for the specified OID(s). 767 * 768 * @param oids (in) the OIDs of interest. 769 * @param partitionNums (in) the set of partitions to be deleted. 770 */ 771 EXPORT void deletePartition(const std::set<OID_t>& oids, 772 const std::set<LogicalPartition>& partitionNums, std::string& emsg); 773 774 /** @brief Mark a Partition for the specified OID(s) as out of service. 775 * 776 * @param oids (in) the OIDs of interest. 777 * @param partitionNums (in) the set of partitions to be marked out of service. 778 */ 779 EXPORT void markPartitionForDeletion(const std::set<OID_t>& oids, 780 const std::set<LogicalPartition>& partitionNums, std::string& emsg); 781 782 /** @brief Mark all Partition for the specified OID(s) as out of service. 783 * 784 * @param oids (in) the OIDs of interest. 785 */ 786 EXPORT void markAllPartitionForDeletion(const std::set<OID_t>& oids); 787 788 /** @brief Restore a Partition for the specified OID(s). 789 * 790 * @param oids (in) the OIDs of interest. 791 * @param partitionNums (in) the set of partitions to be restored. 792 */ 793 EXPORT void restorePartition(const std::set<OID_t>& oids, 794 const std::set<LogicalPartition>& partitionNums, std::string& emsg); 795 796 /** @brief Get the list of out-of-service partitions for a given OID 797 * 798 * @param OID (in) the OID of interest. 799 * @param partitionNums (out) the out-of-service partitions for the oid. 800 * partitionNums will be in sorted order. 801 */ 802 EXPORT void getOutOfServicePartitions(OID_t oid, 803 std::set<LogicalPartition>& partitionNums); 804 805 /** @brief Delete all extent map rows for the specified dbroot 806 * 807 * @param dbroot (in) the dbroot 808 */ 809 EXPORT void deleteDBRoot(uint16_t dbroot); 810 811 /** @brief Is the specified DBRoot empty with no extents. 812 * Throws exception if extentmap shared memory is not loaded. 813 * 814 * @param dbroot DBRoot of interest 815 */ 816 EXPORT bool isDBRootEmpty(uint16_t dbroot); 817 818 /** @brief Performs internal consistency checks (for testing only) 819 * 820 * Performs internal consistency checks (for testing only). 821 * @note It's incomplete 822 * @return 0 if all tests pass, -1 (or throws logic_error) if not. 823 */ 824 EXPORT int checkConsistency(); 825 826 EXPORT void setReadOnly(); 827 828 EXPORT virtual void undoChanges(); 829 830 EXPORT virtual void confirmChanges(); 831 832 EXPORT int markInvalid(const LBID_t lbid, 833 const execplan::CalpontSystemCatalog::ColDataType colDataType); 834 EXPORT int markInvalid(const std::vector<LBID_t>& lbids, 835 const std::vector<execplan::CalpontSystemCatalog::ColDataType>& colDataTypes); 836 837 EXPORT int setMaxMin(const LBID_t lbidRange, const int64_t max, const int64_t min, const int32_t seqNum, 838 bool firstNode); 839 840 // @bug 1970. Added setExtentsMaxMin function below. 841 842 /** @brief Updates the extents in the passed map of CPMaxMin objects. 843 * @param cpMap - The key must be the first LBID in the range. 844 * The values are a CPMaxMin struct with the 845 * min, max, and sequence. 846 * @param firstNode - if true, logs a debugging msg when CP data is updated 847 * @return 0 if all tests pass, -1 (or throws logic_error) if not. 848 */ 849 EXPORT void setExtentsMaxMin(const CPMaxMinMap_t& cpMap, bool firstNode, bool useLock = true); 850 851 /** @brief Merges the CP info for the extents contained in cpMap. 852 * @param cpMap - The key must be the starting LBID in the range. 853 * @return 0 if all tests pass, -1 (or throws logic_error) if not. 854 */ 855 void mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMap, bool useLock = true); 856 857 EXPORT int getMaxMin(const LBID_t lbidRange, int64_t& max, int64_t& min, int32_t& seqNum); 858 empty()859 inline bool empty() 860 { 861 if (fEMShminfo == 0) 862 { 863 grabEMEntryTable(BRM::ExtentMap::READ); 864 releaseEMEntryTable(BRM::ExtentMap::READ); 865 } 866 867 return (fEMShminfo->currentSize == 0); 868 } 869 870 EXPORT std::vector<InlineLBIDRange> getFreeListEntries(); 871 872 EXPORT void dumpTo(std::ostream& os); 873 EXPORT const bool* getEMLockStatus(); 874 EXPORT const bool* getEMFLLockStatus(); 875 876 #ifdef BRM_DEBUG 877 EXPORT void printEM() const; 878 EXPORT void printEM(const OID_t& oid) const; 879 EXPORT void printEM(const EMEntry& em) const; 880 EXPORT void printFL() const; 881 #endif 882 883 private: 884 static const size_t EM_INCREMENT_ROWS = 100; 885 static const size_t EM_INITIAL_SIZE = EM_INCREMENT_ROWS * 10 * sizeof(EMEntry); 886 static const size_t EM_INCREMENT = EM_INCREMENT_ROWS * sizeof(EMEntry); 887 static const size_t EM_FREELIST_INITIAL_SIZE = 50 * sizeof(InlineLBIDRange); 888 static const size_t EM_FREELIST_INCREMENT = 50 * sizeof(InlineLBIDRange); 889 890 ExtentMap(const ExtentMap& em); 891 ExtentMap& operator=(const ExtentMap& em); 892 893 EMEntry* fExtentMap; 894 InlineLBIDRange* fFreeList; 895 key_t fCurrentEMShmkey; 896 key_t fCurrentFLShmkey; 897 MSTEntry* fEMShminfo; 898 MSTEntry* fFLShminfo; 899 const MasterSegmentTable fMST; 900 bool r_only; 901 typedef std::tr1::unordered_map<int, oam::DBRootConfigList*> PmDbRootMap_t; 902 PmDbRootMap_t fPmDbRootMap; 903 time_t fCacheTime; // timestamp associated with config cache 904 905 int numUndoRecords; 906 bool flLocked, emLocked; 907 static boost::mutex mutex; // @bug5355 - made mutex static 908 boost::mutex fConfigCacheMutex; // protect access to Config Cache 909 910 enum OPS 911 { 912 NONE, 913 READ, 914 WRITE 915 }; 916 917 OPS EMLock, FLLock; 918 919 LBID_t _createColumnExtent_DBroot(uint32_t size, int OID, 920 uint32_t colWidth, 921 uint16_t dbRoot, 922 execplan::CalpontSystemCatalog::ColDataType colDataType, 923 uint32_t& partitionNum, 924 uint16_t& segmentNum, 925 uint32_t& startBlockOffset); 926 LBID_t _createColumnExtentExactFile(uint32_t size, int OID, 927 uint32_t colWidth, 928 uint16_t dbRoot, 929 uint32_t partitionNum, 930 uint16_t segmentNum, 931 execplan::CalpontSystemCatalog::ColDataType colDataType, 932 uint32_t& startBlockOffset); 933 LBID_t _createDictStoreExtent(uint32_t size, int OID, 934 uint16_t dbRoot, 935 uint32_t partitionNum, 936 uint16_t segmentNum); 937 bool isValidCPRange(int64_t max, int64_t min, execplan::CalpontSystemCatalog::ColDataType type) const; 938 void deleteExtent(int emIndex); 939 LBID_t getLBIDsFromFreeList(uint32_t size); 940 void reserveLBIDRange(LBID_t start, uint8_t size); // used by load() to allocate pre-existing LBIDs 941 942 key_t chooseEMShmkey(); //see the code for how keys are segmented 943 key_t chooseFLShmkey(); //see the code for how keys are segmented 944 void grabEMEntryTable(OPS op); 945 void grabFreeList(OPS op); 946 void releaseEMEntryTable(OPS op); 947 void releaseFreeList(OPS op); 948 void growEMShmseg(size_t nrows = 0); 949 void growFLShmseg(); 950 void finishChanges(); 951 952 EXPORT unsigned getFilesPerColumnPartition(); 953 unsigned getExtentsPerSegmentFile(); 954 unsigned getDbRootCount(); 955 void getPmDbRoots(int pm, std::vector<int>& dbRootList); 956 void checkReloadConfig(); 957 ShmKeys fShmKeys; 958 959 bool fDebug; 960 961 int _markInvalid(const LBID_t lbid, const execplan::CalpontSystemCatalog::ColDataType colDataType); 962 963 void loadVersion4(std::ifstream& in); 964 void loadVersion4(idbdatafile::IDBDataFile* in); 965 966 ExtentMapImpl* fPExtMapImpl; 967 FreeListImpl* fPFreeListImpl; 968 }; 969 970 inline std::ostream& operator<<(std::ostream& os, ExtentMap& rhs) 971 { 972 rhs.dumpTo(os); 973 return os; 974 } 975 976 } //namespace 977 978 #undef EXPORT 979 980 #endif 981