1 ////////////////////////////////////////////////////////////////////////////////////// 2 // This file is distributed under the University of Illinois/NCSA Open Source License. 3 // See LICENSE file in top directory for details. 4 // 5 // Copyright (c) 2020 QMCPACK developers. 6 // 7 // File developed by: Peter Doak, doakpw@ornl.gov, Oak Ridge National Laboratory 8 // 9 // File refactored from: QMCDriver.h 10 ////////////////////////////////////////////////////////////////////////////////////// 11 12 13 /** 14 * @file 15 * Declaration of QMCDriverNew 16 * 17 * This will replace QMCDriver once unified drivers are finished 18 * the general documentation from QMCDriver.h must be moved before then 19 * 20 * This driver base class should be generic with respect to precision, 21 * value type, device execution, and ... 22 * It should contain no typdefs not related to compiler bugs or platform workarounds 23 * 24 */ 25 26 #ifndef QMCPLUSPLUS_QMCDRIVERNEW_H 27 #define QMCPLUSPLUS_QMCDRIVERNEW_H 28 29 #include <type_traits> 30 31 #include "Configuration.h" 32 #include "Utilities/PooledData.h" 33 #include "Utilities/TimerManager.h" 34 #include "Utilities/ScopedProfiler.h" 35 #include "QMCDrivers/MCPopulation.h" 36 #include "QMCDrivers/QMCDriverInterface.h" 37 #include "QMCDrivers/GreenFunctionModifiers/DriftModifierBase.h" 38 #include "QMCDrivers/QMCDriverInput.h" 39 #include "QMCDrivers/ContextForSteps.h" 40 #include "OhmmsApp/ProjectData.h" 41 #include "MultiWalkerDispatchers.h" 42 #include "DriverWalkerTypes.h" 43 44 class Communicate; 45 46 namespace qmcplusplus 47 { 48 //forward declarations: Do not include headers if not needed 49 class HDFWalkerOutput; 50 class TraceManager; 51 class EstimatorManagerNew; 52 class TrialWaveFunction; 53 class QMCHamiltonian; 54 55 namespace testing 56 { 57 class DMCBatchedTest; 58 class VMCBatchedTest; 59 class QMCDriverNewTestWrapper; 60 } // namespace testing 61 62 /** @ingroup QMCDrivers 63 * @{ 64 * @brief QMCDriverNew Base class for Unified Drivers 65 * 66 * # General Principals 67 * * Parameters used unchanged from input object are not copied into class state 68 * * The driver state machine should be as minimal as possible. 69 * * In non performance critical areas favor clarity over clever optimizations. 70 */ 71 class QMCDriverNew : public QMCDriverInterface, public MPIObjectBase 72 { 73 public: 74 using RealType = QMCTraits::RealType; 75 using IndexType = QMCTraits::IndexType; 76 using FullPrecRealType = QMCTraits::FullPrecRealType; 77 /** separate but similar to QMCModeEnum 78 * 79 * a code smell 80 */ 81 enum 82 { 83 QMC_UPDATE_MODE, 84 QMC_MULTIPLE, 85 QMC_OPTIMIZE, 86 QMC_WARMUP 87 }; 88 89 using MCPWalker = MCPopulation::MCPWalker; 90 using WFBuffer = MCPopulation::WFBuffer; 91 92 using SetNonLocalMoveHandler = std::function<void(QMCHamiltonian&)>; 93 /** bits to classify QMCDriver 94 * 95 * - qmc_driver_mode[QMC_UPDATE_MODE]? particle-by-particle: walker-by-walker 96 * - qmc_driver_mode[QMC_MULTIPLE]? multiple H/Psi : single H/Psi 97 * - qmc_driver_mode[QMC_OPTIMIZE]? optimization : vmc/dmc/rmc 98 */ 99 std::bitset<QMC_MODE_MAX> qmc_driver_mode_; 100 101 protected: 102 void endBlock(); 103 /** This is a data structure strictly for QMCDriver and its derived classes 104 * 105 * i.e. its nested in scope for a reason 106 */ 107 struct AdjustedWalkerCounts 108 { 109 IndexType global_walkers; 110 std::vector<IndexType> walkers_per_rank; 111 std::vector<IndexType> walkers_per_crowd; 112 RealType reserve_walkers; 113 }; 114 115 public: 116 /// Constructor. 117 QMCDriverNew(const ProjectData& project_data, 118 QMCDriverInput&& input, 119 MCPopulation&& population, 120 const std::string timer_prefix, 121 Communicate* comm, 122 const std::string& QMC_driver_type, 123 SetNonLocalMoveHandler = &QMCDriverNew::defaultSetNonLocalMoveHandler); 124 125 ///Move Constructor 126 QMCDriverNew(QMCDriverNew&&) = default; 127 ///Copy Constructor (disabled). 128 QMCDriverNew(const QMCDriverNew&) = delete; 129 ///Copy operator (disabled). 130 QMCDriverNew& operator=(const QMCDriverNew&) = delete; 131 132 virtual ~QMCDriverNew() override; 133 134 bool putQMCInfo(xmlNodePtr cur); 135 136 /** Adjust populations local walkers to this number 137 * @param nwalkers number of walkers to add 138 * 139 */ 140 void makeLocalWalkers(int nwalkers, 141 RealType reserve, 142 const ParticleAttrib<TinyVector<QMCTraits::RealType, 3>>& positions); 143 get_drift_modifier()144 DriftModifierBase& get_drift_modifier() const { return *drift_modifier_; } 145 146 /** record the state of the block 147 * @param block current block 148 * 149 * virtual function with a default implementation 150 */ 151 virtual void recordBlock(int block) override; 152 153 /** finalize a qmc section 154 * @param block current block 155 * @param dumpwalkers if true, dump walkers 156 * 157 * Accumulate energy and weight is written to a hdf5 file. 158 * Finialize the estimators 159 */ 160 bool finalize(int block, bool dumpwalkers = true); 161 162 ///return current step current()163 inline IndexType current() const { return current_step_; } 164 165 /** Set the status of the QMCDriver 166 * @param aname the root file name, ignored 167 * @param h5name root name of the master hdf5 file containing previous qmcrun 168 * @param append if true, the run is a continuation of the previous qmc 169 * 170 * All output files will be of 171 * the form "aname.s00X.suffix", where "X" is number 172 * of previous QMC runs for the simulation and "suffix" 173 * is the suffix for the output file. 174 */ 175 void setStatus(const std::string& aname, const std::string& h5name, bool append) override; 176 add_H_and_Psi(QMCHamiltonian * h,TrialWaveFunction * psi)177 void add_H_and_Psi(QMCHamiltonian* h, TrialWaveFunction* psi) override{}; 178 179 void createRngsStepContexts(int num_crowds); 180 181 void putWalkers(std::vector<xmlNodePtr>& wset) override; 182 183 ///set global offsets of the walkers 184 void setWalkerOffsets(); 185 186 std::vector<RandomGenerator_t*> RngCompatibility; 187 getRng()188 inline std::vector<RandomGenerator_t*>& getRng() { return RngCompatibility; } 189 190 // ///return the random generators 191 // inline std::vector<std::unique_ptr RandomGenerator_t*>& getRng() { return Rng; } 192 193 ///return the i-th random generator getRng(int i)194 inline RandomGenerator_t& getRng(int i) override { return (*Rng[i]); } 195 getEngineName()196 std::string getEngineName() override { return QMCType; } getDriverMode()197 unsigned long getDriverMode() override { return qmc_driver_mode_.to_ulong(); } 198 get_num_living_walkers()199 IndexType get_num_living_walkers() const { return population_.get_walkers().size(); } get_num_dead_walkers()200 IndexType get_num_dead_walkers() const { return population_.get_dead_walkers().size(); } 201 202 /** @ingroup Legacy interface to be dropped 203 * @{ 204 */ put(xmlNodePtr cur)205 bool put(xmlNodePtr cur) override { return false; }; 206 207 /** QMCDriverNew driver second (3rd, 4th...) stage of constructing a valid driver 208 * 209 * This is the shared entry point with legacy, 210 * from QMCMain so the API cannot be updated yet 211 * 212 * \todo remove cur, the driver and all its child nodes should be completely processed before 213 * this stage of driver initialization is hit. 214 */ 215 virtual void process(xmlNodePtr cur) override = 0; 216 217 /** Do common section starting tasks 218 * 219 * \todo This should not take xmlNodePtr 220 * It should either take BranchEngineInput and EstimatorInput 221 * And these are the arguments to the branch_engine and estimator_manager 222 * Constructors or these objects should be created elsewhere. 223 */ 224 void startup(xmlNodePtr cur, QMCDriverNew::AdjustedWalkerCounts awc); 225 226 static void initialLogEvaluation(int crowd_id, UPtrVector<Crowd>& crowds, UPtrVector<ContextForSteps>& step_context); 227 228 229 /** should be set in input don't see a reason to set individually 230 * @param pbyp if true, use particle-by-particle update 231 */ setUpdateMode(bool pbyp)232 inline void setUpdateMode(bool pbyp) override { qmc_driver_mode_[QMC_UPDATE_MODE] = pbyp; } 233 putTraces(xmlNodePtr txml)234 void putTraces(xmlNodePtr txml) override {} requestTraces(bool allow_traces)235 void requestTraces(bool allow_traces) override {} 236 /** }@ */ 237 238 protected: 239 /** pure function returning AdjustedWalkerCounts data structure 240 * 241 * The logic is now walker counts is fairly simple. 242 * TotalWalkers trumps all other walker parameters 243 * If TotalWalkers is absent walkers_per_rank is used. 244 * if they are both absent then the default is one walker per crowd, 245 * each rank has crowds walkers. 246 * if crowds aren't specified you get one per main level thread. 247 * 248 * You can have crowds or ranks with no walkers. 249 * You cannot have more crowds than threads. 250 * 251 * passing num_ranks instead of internally querying comm->size() 252 * makes unit testing much quicker. 253 * 254 */ 255 static QMCDriverNew::AdjustedWalkerCounts adjustGlobalWalkerCount(int num_ranks, 256 int rank_id, 257 IndexType desired_count, 258 IndexType walkers_per_rank, 259 RealType reserve_walkers, 260 int num_crowds); 261 262 static void checkNumCrowdsLTNumThreads(const int num_crowds); 263 264 /// check logpsi and grad and lap against values computed from scratch 265 static bool checkLogAndGL(Crowd& crowd); 266 get_root_name()267 const std::string& get_root_name() const override { return project_data_.CurrentMainRoot(); } 268 269 /** The timers for the driver. 270 * 271 * This cleans up the driver constructor, and a reference to this structure 272 * Takes the timers into thread scope. We assume the timers are threadsafe. 273 */ 274 struct DriverTimers 275 { 276 NewTimer& checkpoint_timer; 277 NewTimer& run_steps_timer; 278 NewTimer& create_walkers_timer; 279 NewTimer& init_walkers_timer; 280 NewTimer& buffer_timer; 281 NewTimer& movepbyp_timer; 282 NewTimer& hamiltonian_timer; 283 NewTimer& collectables_timer; DriverTimersDriverTimers284 DriverTimers(const std::string& prefix) 285 : checkpoint_timer(*timer_manager.createTimer(prefix + "CheckPoint", timer_level_medium)), 286 run_steps_timer(*timer_manager.createTimer(prefix + "RunSteps", timer_level_medium)), 287 create_walkers_timer(*timer_manager.createTimer(prefix + "CreateWalkers", timer_level_medium)), 288 init_walkers_timer(*timer_manager.createTimer(prefix + "InitWalkers", timer_level_medium)), 289 buffer_timer(*timer_manager.createTimer(prefix + "Buffer", timer_level_medium)), 290 movepbyp_timer(*timer_manager.createTimer(prefix + "MovePbyP", timer_level_medium)), 291 hamiltonian_timer(*timer_manager.createTimer(prefix + "Hamiltonian", timer_level_medium)), 292 collectables_timer(*timer_manager.createTimer(prefix + "Collectables", timer_level_medium)) 293 {} 294 }; 295 296 const QMCDriverInput qmcdriver_input_; 297 298 /** @ingroup Driver mutable input values 299 * 300 * they should be limited to values that can be changed from input 301 * or are live state. 302 * @{ 303 */ 304 RealType max_disp_sq_; 305 ///the number of saved samples 306 IndexType target_samples_; 307 308 /// the number of blocks between recomptePsi 309 IndexType nBlocksBetweenRecompute; 310 311 /**}@*/ 312 313 std::vector<std::unique_ptr<Crowd>> crowds_; 314 315 std::string h5_file_root_; 316 317 ///drift modifer 318 std::unique_ptr<DriftModifierBase> drift_modifier_; 319 320 ///the number to delay updates by 321 int k_delay; 322 323 /** period of recording walker configurations 324 * 325 * Default is 0 indicating that only the last configuration will be saved. 326 */ 327 int walker_dump_period; 328 329 330 IndexType current_step_; 331 332 ///counter for number of moves accepted 333 IndexType nAccept; 334 335 ///counter for number of moves /rejected 336 IndexType nReject; 337 338 ///Time-step factor \f$ 1/(2\tau)\f$ 339 RealType m_oneover2tau; 340 ///Time-step factor \f$ \sqrt{\tau}\f$ 341 RealType m_sqrttau; 342 343 ///type of qmc: assigned by subclasses 344 const std::string QMCType; 345 ///root of all the output files 346 std::string root_name_; 347 348 /** the entire (on node) walker population 349 * it serves VMCBatch and DMCBatch right now but will be polymorphic 350 */ 351 MCPopulation population_; 352 353 /** the golden multi walker shared resource 354 * serves ParticleSet TrialWaveFunction right now but actually should be based on MCPopulation. 355 * per crowd resources are copied from this gold instance 356 * it should be activated when dispatchers don't serialize walkers 357 */ 358 struct DriverWalkerResourceCollection golden_resource_; 359 360 /// multi walker dispatchers 361 const MultiWalkerDispatchers dispatchers_; 362 363 /** Observables manager 364 * Has very problematic owner ship and life cycle. 365 * Can be transferred via branch manager one driver to the next indefinitely 366 * TODO: Modify Branch manager and others to clear this up. 367 */ 368 std::unique_ptr<EstimatorManagerNew> estimator_manager_; 369 370 ///record engine for walkers 371 HDFWalkerOutput* wOut; 372 373 /** Per crowd move contexts, this is where the DistanceTables etc. reside 374 */ 375 std::vector<std::unique_ptr<ContextForSteps>> step_contexts_; 376 377 ///Random number generators 378 std::vector<std::unique_ptr<RandomGenerator_t>> Rng; 379 380 ///a list of mcwalkerset element 381 std::vector<xmlNodePtr> mcwalkerNodePtr; 382 383 ///temporary storage for drift 384 ParticleSet::ParticlePos_t drift; 385 386 ///temporary storage for random displacement 387 ParticleSet::ParticlePos_t deltaR; 388 389 // ///alternate method of setting QMC run parameters 390 // IndexType nStepsBetweenSamples; 391 // ///samples per thread 392 // IndexType nSamplesPerThread; 393 394 // TODO: restart 395 // /** period of dumping walker configurations and everything else for restart 396 // * 397 // * The unit is a block. 398 // */ 399 // int check_point_period_; 400 401 /** }@ */ 402 403 DriverTimers timers_; 404 405 ///time the driver lifetime 406 ScopedTimer driver_scope_timer_; 407 ///profile the driver lifetime 408 ScopedProfiler driver_scope_profiler_; 409 410 /// project info for accessing global fileroot and series id 411 const ProjectData& project_data_; 412 413 private: 414 friend std::ostream& operator<<(std::ostream& o_stream, const QMCDriverNew& qmcd); 415 416 SetNonLocalMoveHandler setNonLocalMoveHandler_; 417 418 static void defaultSetNonLocalMoveHandler(QMCHamiltonian& gold_ham); 419 420 friend class qmcplusplus::testing::VMCBatchedTest; 421 friend class qmcplusplus::testing::DMCBatchedTest; 422 friend class qmcplusplus::testing::QMCDriverNewTestWrapper; 423 }; 424 } // namespace qmcplusplus 425 426 #endif 427