1 /*========================================================================= 2 3 Program: Visualization Toolkit 4 Module: vtkPKdTree.h 5 6 Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen 7 All rights reserved. 8 See Copyright.txt or http://www.kitware.com/Copyright.htm for details. 9 10 This software is distributed WITHOUT ANY WARRANTY; without even 11 the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR 12 PURPOSE. See the above copyright notice for more information. 13 14 =========================================================================*/ 15 /*---------------------------------------------------------------------------- 16 Copyright (c) Sandia Corporation 17 See Copyright.txt or http://www.paraview.org/HTML/Copyright.html for details. 18 ----------------------------------------------------------------------------*/ 19 20 /** 21 * @class vtkPKdTree 22 * @brief Build a k-d tree decomposition of a list of points. 23 * 24 * 25 * Build, in parallel, a k-d tree decomposition of one or more 26 * vtkDataSets distributed across processors. We assume each 27 * process has read in one portion of a large distributed data set. 28 * When done, each process has access to the k-d tree structure, 29 * can obtain information about which process contains 30 * data for each spatial region, and can depth sort the spatial 31 * regions. 32 * 33 * This class can also assign spatial regions to processors, based 34 * on one of several region assignment schemes. By default 35 * a contiguous, convex region is assigned to each process. Several 36 * queries return information about how many and what cells I have 37 * that lie in a region assigned to another process. 38 * 39 * @sa 40 * vtkKdTree 41 */ 42 43 #ifndef vtkPKdTree_h 44 #define vtkPKdTree_h 45 46 #include "vtkFiltersParallelModule.h" // For export macro 47 #include "vtkKdTree.h" 48 #include <string> // Instead of using char* 49 #include <vector> // For automatic array memory management 50 51 class vtkMultiProcessController; 52 class vtkCommunicator; 53 class vtkSubGroup; 54 class vtkIntArray; 55 class vtkKdNode; 56 57 class VTKFILTERSPARALLEL_EXPORT vtkPKdTree : public vtkKdTree 58 { 59 public: 60 vtkTypeMacro(vtkPKdTree, vtkKdTree); 61 62 63 void PrintSelf(ostream& os, vtkIndent indent) override; 64 void PrintTiming(ostream& os, vtkIndent indent) override; 65 void PrintTables(ostream& os, vtkIndent indent); 66 67 static vtkPKdTree *New(); 68 69 /** 70 * Build the spatial decomposition. Call this explicitly 71 * after changing any parameters affecting the build of the 72 * tree. It must be called by all processes in the parallel 73 * application, or it will hang. 74 */ 75 void BuildLocator() override; 76 77 /** 78 * Get the total number of cells distributed across the data 79 * files read by all processes. You must have called BuildLocator 80 * before calling this method. 81 */ GetTotalNumberOfCells()82 vtkIdType GetTotalNumberOfCells(){return this->TotalNumCells;} 83 84 /** 85 * Create tables of counts of cells per process per region. 86 * These tables can be accessed with queries like 87 * "HasData", "GetProcessCellCountForRegion", and so on. 88 * You must have called BuildLocator() beforehand. This 89 * method must be called by all processes or it will hang. 90 * Returns 1 on error, 0 when no error. 91 */ 92 int CreateProcessCellCountData(); 93 94 /** 95 * A convenience function which compiles the global 96 * bounds of the data arrays across processes. 97 * These bounds can be accessed with 98 * "GetCellArrayGlobalRange" and "GetPointArrayGlobalRange". 99 * This method must be called by all processes or it will hang. 100 * Returns 1 on error, 0 when no error. 101 */ 102 int CreateGlobalDataArrayBounds(); 103 104 //@{ 105 /** 106 * Set/Get the communicator object 107 */ 108 void SetController(vtkMultiProcessController *c); 109 vtkGetObjectMacro(Controller, vtkMultiProcessController); 110 //@} 111 112 //@{ 113 /** 114 * The PKdTree class can assign spatial regions to processors after 115 * building the k-d tree, using one of several partitioning criteria. 116 * These functions Set/Get whether this assignment is computed. 117 * The default is "Off", no assignment is computed. If "On", and 118 * no assignment scheme is specified, contiguous assignment will be 119 * computed. Specifying an assignment scheme (with AssignRegions*()) 120 * automatically turns on RegionAssignment. 121 */ 122 vtkGetMacro(RegionAssignment, int); 123 //@} 124 125 static const int NoRegionAssignment; 126 static const int ContiguousAssignment; 127 static const int UserDefinedAssignment; 128 static const int RoundRobinAssignment; 129 130 /** 131 * Assign spatial regions to processes via a user defined map. 132 * The user-supplied map is indexed by region ID, and provides a 133 * process ID for each region. 134 */ 135 int AssignRegions(int *map, int numRegions); 136 137 /** 138 * Let the PKdTree class assign a process to each region in a 139 * round robin fashion. If the k-d tree has not yet been 140 * built, the regions will be assigned after BuildLocator executes. 141 */ 142 int AssignRegionsRoundRobin(); 143 144 /** 145 * Let the PKdTree class assign a process to each region 146 * by assigning contiguous sets of spatial regions to each 147 * process. The set of regions assigned to each process will 148 * always have a union that is a convex space (a box). 149 * If the k-d tree has not yet been built, the regions 150 * will be assigned after BuildLocator executes. 151 */ 152 int AssignRegionsContiguous(); 153 154 /** 155 * Returns the region assignment map where index is the region and value is 156 * the processes id for that region. 157 */ GetRegionAssignmentMap()158 const int* GetRegionAssignmentMap() 159 { return &this->RegionAssignmentMap[0]; } 160 161 //@{ 162 /** 163 * / Returns the number of regions in the region assignment map. 164 */ GetRegionAssignmentMapLength()165 int GetRegionAssignmentMapLength() 166 { return static_cast<int>(this->RegionAssignmentMap.size()); } 167 //@} 168 169 /** 170 * Writes the list of region IDs assigned to the specified 171 * process. Regions IDs start at 0 and increase by 1 from there. 172 * Returns the number of regions in the list. 173 */ 174 int GetRegionAssignmentList(int procId, vtkIntArray *list); 175 176 /** 177 * The k-d tree spatial regions have been assigned to processes. 178 * Given a point on the boundary of one of the regions, this 179 * method creates a list of all processes whose region 180 * boundaries include that point. This may be required when 181 * looking for processes that have cells adjacent to the cells 182 * of a given process. 183 */ 184 void GetAllProcessesBorderingOnPoint(float x, float y, float z, 185 vtkIntArray *list); 186 187 /** 188 * Returns the ID of the process assigned to the region. 189 */ 190 int GetProcessAssignedToRegion(int regionId); 191 192 /** 193 * Returns 1 if the process has data for the given region, 194 * 0 otherwise. 195 */ 196 int HasData(int processId, int regionId); 197 198 /** 199 * Returns the number of cells the specified process has in the 200 * specified region. 201 */ 202 int GetProcessCellCountForRegion(int processId, int regionId); 203 204 /** 205 * Returns the total number of processes that have data 206 * falling within this spatial region. 207 */ 208 int GetTotalProcessesInRegion(int regionId); 209 210 /** 211 * Adds the list of processes having data for the given 212 * region to the supplied list, returns the number of 213 * processes added. 214 */ 215 int GetProcessListForRegion(int regionId, vtkIntArray *processes); 216 217 /** 218 * Writes the number of cells each process has for the region 219 * to the supplied list of length len. Returns the number of 220 * cell counts written. The order of the cell counts corresponds 221 * to the order of process IDs in the process list returned by 222 * GetProcessListForRegion. 223 */ 224 int GetProcessesCellCountForRegion(int regionId, int *count, int len); 225 226 /** 227 * Returns the total number of spatial regions that a given 228 * process has data for. 229 */ 230 int GetTotalRegionsForProcess(int processId); 231 232 /** 233 * Adds the region IDs for which this process has data to 234 * the supplied vtkIntArray. Returns the number of regions. 235 */ 236 int GetRegionListForProcess(int processId, vtkIntArray *regions); 237 238 /** 239 * Writes to the supplied integer array the number of cells this 240 * process has for each region. Returns the number of 241 * cell counts written. The order of the cell counts corresponds 242 * to the order of region IDs in the region list returned by 243 * GetRegionListForProcess. 244 */ 245 int GetRegionsCellCountForProcess(int ProcessId, int *count, int len); 246 247 //@{ 248 /** 249 * After regions have been assigned to processes, I may want to know 250 * which cells I have that are in the regions assigned to a particular 251 * process. 252 253 * This method takes a process ID and two vtkIdLists. It 254 * writes to the first list the IDs of the cells 255 * contained in the process' regions. (That is, their cell 256 * centroid is contained in the region.) To the second list it 257 * write the IDs of the cells which intersect the process' regions 258 * but whose cell centroid lies elsewhere. 259 260 * The total number of cell IDs written to both lists is returned. 261 * Either list pointer passed in can be nullptr, and it will be ignored. 262 * If there are multiple data sets, you must specify which data set 263 * you wish cell IDs for. 264 265 * The caller should delete these two lists when done. This method 266 * uses the cell lists created in vtkKdTree::CreateCellLists(). 267 * If the cell lists for the process' regions do not exist, this 268 * method will first build the cell lists for all regions by calling 269 * CreateCellLists(). You must remember to DeleteCellLists() when 270 * done with all calls to this method, as cell lists can require a 271 * great deal of memory. 272 */ 273 vtkIdType GetCellListsForProcessRegions(int ProcessId, int set, 274 vtkIdList *inRegionCells, vtkIdList *onBoundaryCells); 275 vtkIdType GetCellListsForProcessRegions(int ProcessId, vtkDataSet *set, 276 vtkIdList *inRegionCells, vtkIdList *onBoundaryCells); 277 vtkIdType GetCellListsForProcessRegions(int ProcessId, 278 vtkIdList *inRegionCells, 279 vtkIdList *onBoundaryCells); 280 //@} 281 282 /** 283 * Return a list of all processes in order from front to back given a 284 * vector direction of projection. Use this to do visibility sorts 285 * in parallel projection mode. `orderedList' will be resized to the number 286 * of processes. The return value is the number of processes. 287 * \pre orderedList_exists: orderedList!=0 288 */ 289 int ViewOrderAllProcessesInDirection(const double directionOfProjection[3], 290 vtkIntArray *orderedList); 291 292 /** 293 * Return a list of all processes in order from front to back given a 294 * camera position. Use this to do visibility sorts in perspective 295 * projection mode. `orderedList' will be resized to the number 296 * of processes. The return value is the number of processes. 297 * \pre orderedList_exists: orderedList!=0 298 */ 299 int ViewOrderAllProcessesFromPosition(const double cameraPosition[3], 300 vtkIntArray *orderedList); 301 302 /** 303 * An added feature of vtkPKdTree is that it will calculate the 304 * the global range of field arrays across all processes. You 305 * call CreateGlobalDataArrayBounds() to do this calculation. 306 * Then the following methods return the ranges. 307 * Returns 1 on error, 0 otherwise. 308 */ 309 310 int GetCellArrayGlobalRange(const char *name, float range[2]); 311 int GetPointArrayGlobalRange(const char *name, float range[2]); 312 int GetCellArrayGlobalRange(const char *name, double range[2]); 313 int GetPointArrayGlobalRange(const char *name, double range[2]); 314 315 int GetCellArrayGlobalRange(int arrayIndex, double range[2]); 316 int GetPointArrayGlobalRange(int arrayIndex, double range[2]); 317 int GetCellArrayGlobalRange(int arrayIndex, float range[2]); 318 int GetPointArrayGlobalRange(int arrayIndex, float range[2]); 319 320 protected: 321 322 vtkPKdTree(); 323 ~vtkPKdTree() override; 324 325 void SingleProcessBuildLocator(); 326 int MultiProcessBuildLocator(double *bounds); 327 328 private: 329 330 int RegionAssignment; 331 332 vtkMultiProcessController *Controller; 333 334 vtkSubGroup *SubGroup; 335 336 void StrDupWithNew(const char *s, std::string& output); 337 338 int NumProcesses; 339 int MyId; 340 341 // basic tables - each region is the responsibility of one process, but 342 // one process may be assigned many regions 343 344 std::vector<int> RegionAssignmentMap; // indexed by region ID 345 std::vector<std::vector<int> > ProcessAssignmentMap; // indexed by process ID 346 std::vector<int> NumRegionsAssigned; // indexed by process ID 347 348 int UpdateRegionAssignment(); 349 350 // basic tables reflecting the data that was read from disk 351 // by each process 352 353 std::vector<char> DataLocationMap; // by process, by region 354 355 std::vector<int> NumProcessesInRegion; // indexed by region ID 356 std::vector<std::vector<int> > ProcessList; // indexed by region ID 357 358 std::vector<int> NumRegionsInProcess; // indexed by process ID 359 std::vector<std::vector<int> > ParallelRegionList; // indexed by process ID 360 361 std::vector<std::vector<vtkIdType> > CellCountList; // indexed by region ID 362 363 std::vector<double> CellDataMin; // global range for data arrays 364 std::vector<double> CellDataMax; 365 std::vector<double> PointDataMin; 366 std::vector<double> PointDataMax; 367 std::vector<std::string> CellDataName; 368 std::vector<std::string> PointDataName; 369 int NumCellArrays; 370 int NumPointArrays; 371 372 // distribution of indices for select operation 373 374 int BuildGlobalIndexLists(vtkIdType ncells); 375 376 std::vector<vtkIdType> StartVal; 377 std::vector<vtkIdType> EndVal; 378 std::vector<vtkIdType> NumCells; 379 vtkIdType TotalNumCells; 380 381 // local share of points to be partitioned, and local cache 382 383 int WhoHas(int pos); 384 int _whoHas(int L, int R, int pos); 385 float *GetLocalVal(int pos); 386 float *GetLocalValNext(int pos); 387 void SetLocalVal(int pos, float *val); 388 void ExchangeVals(int pos1, int pos2); 389 void ExchangeLocalVals(int pos1, int pos2); 390 391 // keep PtArray and PtArray2 as dynamically allocated since it's set as a return 392 // value from parent class method 393 float *PtArray; 394 float *PtArray2; 395 float *CurrentPtArray; // just pointer to other memory but never allocated 396 float *NextPtArray; // just pointer to other memory but never allocated 397 int PtArraySize; 398 399 std::vector<int> SelectBuffer; 400 401 // Parallel build of k-d tree 402 403 int AllCheckForFailure(int rc, const char *where, const char *how); 404 void AllCheckParameters(); 405 406 //@{ 407 /** 408 * Return the global bounds over all processes. Returns true 409 * if successful and false otherwise. 410 */ 411 bool VolumeBounds(double*); 412 int DivideRegion(vtkKdNode *kd, int L, int level, int tag); 413 int BreadthFirstDivide(double *bounds); 414 void enQueueNode(vtkKdNode *kd, int L, int level, int tag); 415 //@} 416 417 int Select(int dim, int L, int R); 418 void _select(int L, int R, int K, int dim); 419 void DoTransfer(int from, int to, int fromIndex, int toIndex, int count); 420 421 int *PartitionAboutMyValue(int L, int R, int K, int dim); 422 int *PartitionAboutOtherValue(int L, int R, float T, int dim); 423 int *PartitionSubArray(int L, int R, int K, int dim, int p1, int p2); 424 425 int CompleteTree(); 426 #ifdef YIELDS_INCONSISTENT_REGION_BOUNDARIES 427 void RetrieveData(vtkKdNode *kd, int *buf); 428 #else 429 void ReduceData(vtkKdNode *kd, int *sources); 430 void BroadcastData(vtkKdNode *kd); 431 #endif 432 433 void GetDataBounds(int L, int K, int R, float dataBounds[12]); 434 void GetLocalMinMax(int L, int R, int me, float *min, float *max); 435 436 static int FillOutTree(vtkKdNode *kd, int level); 437 static int ComputeDepth(vtkKdNode *kd); 438 static void PackData(vtkKdNode *kd, double *data); 439 static void UnpackData(vtkKdNode *kd, double *data); 440 static void CheckFixRegionBoundaries(vtkKdNode *tree); 441 442 // list management 443 444 int AllocateDoubleBuffer(); 445 void FreeDoubleBuffer(); 446 void SwitchDoubleBuffer(); 447 void AllocateSelectBuffer(); 448 void FreeSelectBuffer(); 449 450 void InitializeGlobalIndexLists(); 451 void AllocateAndZeroGlobalIndexLists(); 452 void FreeGlobalIndexLists(); 453 void InitializeRegionAssignmentLists(); 454 void AllocateAndZeroRegionAssignmentLists(); 455 void FreeRegionAssignmentLists(); 456 void InitializeProcessDataLists(); 457 void AllocateAndZeroProcessDataLists(); 458 void FreeProcessDataLists(); 459 void InitializeFieldArrayMinMax(); 460 void AllocateAndZeroFieldArrayMinMax(); 461 void FreeFieldArrayMinMax(); 462 463 void ReleaseTables(); 464 465 // Assigning regions to processors 466 467 void AddProcessRegions(int procId, vtkKdNode *kd); 468 void BuildRegionListsForProcesses(); 469 470 // Gather process/region data totals 471 472 bool CollectLocalRegionProcessData(std::vector<int>&); // returns false for failure 473 int BuildRegionProcessTables(); 474 int BuildFieldArrayMinMax(); 475 void AddEntry(int *list, int len, int id); 476 #ifdef VTK_USE_64BIT_IDS 477 void AddEntry(vtkIdType *list, int len, vtkIdType id); 478 #endif 479 static int BinarySearch(vtkIdType *list, int len, vtkIdType which); 480 481 static int FindNextLocalArrayIndex(const char *n, const std::vector<std::string>& names, 482 int len, int start=0); 483 484 vtkPKdTree(const vtkPKdTree&) = delete; 485 void operator=(const vtkPKdTree&) = delete; 486 }; 487 488 #endif 489