/****************************************** Copyright (c) 2016, Mate Soos Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ***********************************************/ #include "datasync.h" #include "varreplacer.h" #include "solver.h" #include "shareddata.h" #include using namespace CMSat; DataSync::DataSync(Solver* _solver, SharedData* _sharedData, bool _is_mpi) : solver(_solver) , sharedData(_sharedData) , is_mpi(_is_mpi) , seen(solver->seen) , toClear(solver->toClear) { #ifdef USE_MPI if (is_mpi) { int err; err = MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); assert(err == MPI_SUCCESS); err = MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); assert(err == MPI_SUCCESS); release_assert(!(mpiSize > 1 && mpiRank == 0)); assert(!(mpiSize > 1 && sharedData == NULL)); } #endif } void DataSync::set_shared_data(SharedData* _sharedData) { sharedData = _sharedData; } void DataSync::new_var(const bool bva) { if (!enabled()) return; if (!bva) { syncFinish.push_back(0); syncFinish.push_back(0); } assert(solver->nVarsOutside()*2 == syncFinish.size()); } void DataSync::new_vars(size_t n) { if (!enabled()) return; syncFinish.insert(syncFinish.end(), 2*n, 0); assert(solver->nVarsOutside()*2 == syncFinish.size()); } void DataSync::save_on_var_memory() { } void DataSync::rebuild_bva_map() { must_rebuild_bva_map = true; } void DataSync::updateVars( const vector& /*outerToInter*/ , const vector& /*interToOuter*/ ) { } bool DataSync::syncData() { if (!enabled() || lastSyncConf + solver->conf.sync_every_confl >= solver->sumConflicts ) { return true; } numCalls++; assert(sharedData != NULL); assert(solver->decisionLevel() == 0); if (must_rebuild_bva_map) { outer_to_without_bva_map = solver->build_outer_to_without_bva_map(); must_rebuild_bva_map = false; } bool ok; sharedData->unit_mutex.lock(); ok = shareUnitData(); sharedData->unit_mutex.unlock(); if (!ok) return false; sharedData->bin_mutex.lock(); extend_bins_if_needed(); clear_set_binary_values(); ok = shareBinData(); sharedData->bin_mutex.unlock(); if (!ok) return false; #ifdef USE_MPI if (is_mpi && mpiSize > 1 && solver->conf.thread_num == 0) { sharedData->unit_mutex.lock(); sharedData->bin_mutex.lock(); ok = syncFromMPI(); if (ok && numCalls % 2 == 1) { syncToMPI(); } if (!ok) return false; } if (is_mpi) { getNeedToInterruptFromMPI(); } #endif lastSyncConf = solver->sumConflicts; return true; } void DataSync::clear_set_binary_values() { for(size_t i = 0; i < solver->nVarsOutside()*2; i++) { Lit lit1 = Lit::toLit(i); lit1 = solver->map_to_with_bva(lit1); lit1 = solver->varReplacer->get_lit_replaced_with_outer(lit1); lit1 = solver->map_outer_to_inter(lit1); if (solver->value(lit1) != l_Undef) { sharedData->bins[i].clear(); } } } void DataSync::extend_bins_if_needed() { assert(sharedData->bins.size() <= (solver->nVarsOutside())*2); if (sharedData->bins.size() == (solver->nVarsOutside())*2) return; sharedData->bins.resize(solver->nVarsOutside()*2); } bool DataSync::shareBinData() { uint32_t oldRecvBinData = stats.recvBinData; uint32_t oldSentBinData = stats.sentBinData; syncBinFromOthers(); syncBinToOthers(); size_t mem = sharedData->calc_memory_use_bins(); if (solver->conf.verbosity >= 3) { cout << "c [sync] got bins " << (stats.recvBinData - oldRecvBinData) << " sent bins " << (stats.sentBinData - oldSentBinData) << " mem use: " << mem/(1024*1024) << " M" << endl; } return true; } bool DataSync::syncBinFromOthers() { for (uint32_t wsLit = 0; wsLit < sharedData->bins.size(); wsLit++) { if (sharedData->bins[wsLit].data == NULL) { continue; } Lit lit1 = Lit::toLit(wsLit); lit1 = solver->map_to_with_bva(lit1); lit1 = solver->varReplacer->get_lit_replaced_with_outer(lit1); lit1 = solver->map_outer_to_inter(lit1); if (solver->varData[lit1.var()].removed != Removed::none || solver->value(lit1.var()) != l_Undef ) { continue; } vector& bins = *sharedData->bins[wsLit].data; watch_subarray ws = solver->watches[lit1]; assert(syncFinish.size() > wsLit); if (bins.size() > syncFinish[wsLit] && !syncBinFromOthers(lit1, bins, syncFinish[wsLit], ws) ) { return false; } } return true; } bool DataSync::syncBinFromOthers( const Lit lit , const vector& bins , uint32_t& finished , watch_subarray ws ) { assert(solver->varReplacer->get_lit_replaced_with(lit) == lit); assert(solver->varData[lit.var()].removed == Removed::none); assert(toClear.empty()); for (const Watched& w: ws) { if (w.isBin()) { toClear.push_back(w.lit2()); assert(seen.size() > w.lit2().toInt()); seen[w.lit2().toInt()] = true; } } vector lits(2); for (uint32_t i = finished; i < bins.size(); i++) { Lit otherLit = bins[i]; otherLit = solver->map_to_with_bva(otherLit); otherLit = solver->varReplacer->get_lit_replaced_with_outer(otherLit); otherLit = solver->map_outer_to_inter(otherLit); if (solver->varData[otherLit.var()].removed != Removed::none || solver->value(otherLit) != l_Undef ) { continue; } assert(seen.size() > otherLit.toInt()); if (!seen[otherLit.toInt()]) { stats.recvBinData++; lits[0] = lit; lits[1] = otherLit; //Don't add DRAT: it would add to the thread data, too solver->add_clause_int(lits, true, ClauseStats(), true, NULL, false); if (!solver->ok) { goto end; } } } finished = bins.size(); end: for (const Lit l: toClear) { seen[l.toInt()] = false; } toClear.clear(); return solver->okay(); } void DataSync::syncBinToOthers() { for(const std::pair& bin: newBinClauses) { addOneBinToOthers(bin.first, bin.second); } newBinClauses.clear(); } void DataSync::addOneBinToOthers(Lit lit1, Lit lit2) { assert(lit1 < lit2); if (sharedData->bins[lit1.toInt()].data == NULL) { return; } vector& bins = *sharedData->bins[lit1.toInt()].data; for (const Lit lit : bins) { if (lit == lit2) return; } bins.push_back(lit2); stats.sentBinData++; } bool DataSync::shareUnitData() { uint32_t thisGotUnitData = 0; uint32_t thisSentUnitData = 0; SharedData& shared = *sharedData; if (shared.value.size() < solver->nVarsOutside()) { shared.value.resize(solver->nVarsOutside(), l_Undef); } for (uint32_t var = 0; var < solver->nVarsOutside(); var++) { Lit thisLit = Lit(var, false); thisLit = solver->map_to_with_bva(thisLit); thisLit = solver->varReplacer->get_lit_replaced_with_outer(thisLit); thisLit = solver->map_outer_to_inter(thisLit); const lbool thisVal = solver->value(thisLit); const lbool otherVal = shared.value[var]; if (thisVal == l_Undef && otherVal == l_Undef) { continue; } if (thisVal != l_Undef && otherVal != l_Undef) { if (thisVal != otherVal) { solver->ok = false; return false; } else { continue; } } if (otherVal != l_Undef) { assert(thisVal == l_Undef); Lit litToEnqueue = thisLit ^ (otherVal == l_False); if (solver->varData[litToEnqueue.var()].removed != Removed::none) { continue; } solver->enqueue(litToEnqueue); solver->ok = solver->propagate().isNULL(); if (!solver->ok) { return false; } thisGotUnitData++; continue; } if (thisVal != l_Undef) { assert(otherVal == l_Undef); shared.value[var] = thisVal; thisSentUnitData++; continue; } } if (solver->conf.verbosity >= 3 //&& (thisGotUnitData > 0 || thisSentUnitData > 0) ) { cout << "c [sync] got units " << thisGotUnitData << " sent units " << thisSentUnitData << endl; } stats.recvUnitData += thisGotUnitData; stats.sentUnitData += thisSentUnitData; return true; } void DataSync::signalNewBinClause(Lit lit1, Lit lit2) { if (!enabled()) { return; } if (must_rebuild_bva_map) { outer_to_without_bva_map = solver->build_outer_to_without_bva_map(); must_rebuild_bva_map = false; } if (solver->varData[lit1.var()].is_bva) return; if (solver->varData[lit2.var()].is_bva) return; lit1 = solver->map_inter_to_outer(lit1); lit1 = map_outside_without_bva(lit1); lit2 = solver->map_inter_to_outer(lit2); lit2 = map_outside_without_bva(lit2); if (lit1.toInt() > lit2.toInt()) { std::swap(lit1, lit2); } newBinClauses.push_back(std::make_pair(lit1, lit2)); } /////////////////////////////////////// // MPI /////////////////////////////////////// #ifdef USE_MPI void DataSync::getNeedToInterruptFromMPI() { int flag; MPI_Status status; int err = MPI_Iprobe(0, 1, MPI_COMM_WORLD, &flag, &status); assert(err == MPI_SUCCESS); if (flag == false) { return; } char* buf = NULL; err = MPI_Recv((unsigned*)buf, 0, MPI_UNSIGNED, 0, 1, MPI_COMM_WORLD, &status); assert(err == MPI_SUCCESS); solver->set_must_interrupt_asap(); } bool DataSync::syncFromMPI() { int err; MPI_Status status; int flag; int count; uint32_t tmp = 0; uint32_t thisMpiRecvUnitData = 0; uint32_t thisMpiRecvBinData = 0; err = MPI_Iprobe(0, 0, MPI_COMM_WORLD, &flag, &status); assert(err == MPI_SUCCESS); if (flag == false) return true; err = MPI_Get_count(&status, MPI_UNSIGNED, &count); assert(err == MPI_SUCCESS); #ifdef VERBOSE_DEBUG_MPI_SENDRCV std::cout << "-->> MPI " << mpiRank << " Received " << count << " uint32_t-s" << std::endl; #endif uint32_t* buf = new uint32_t[count]; err = MPI_Recv((unsigned*)buf, count, MPI_UNSIGNED, 0, 0, MPI_COMM_WORLD, &status); assert(err == MPI_SUCCESS); //Unit clauses int at = 0; assert(solver->nVars() == buf[at]); at++; for (uint32_t var = 0; var < solver->nVars(); var++, at++) { const lbool otherVal = toLbool(buf[at]); if (!sync_mpi_unit(otherVal, var, NULL, thisMpiRecvUnitData, tmp)) { #ifdef VERBOSE_DEBUG_MPI_SENDRCV std::cout << "-->> MPI " << mpiRank << " solver FALSE" << std::endl; #endif goto end; } } solver->ok = solver->propagate().isNULL(); if (!solver->ok) goto end; mpiRecvUnitData += thisMpiRecvUnitData; //Binary clauses assert(buf[at] == solver->nVars()*2); at++; for (uint32_t wsLit = 0; wsLit < solver->nVars()*2; wsLit++) { Lit lit = ~Lit::toLit(wsLit); uint32_t num = buf[at]; at++; for (uint32_t i = 0; i < num; i++, at++) { Lit otherLit = Lit::toLit(buf[at]); addOneBinToOthers(lit, otherLit); thisMpiRecvBinData++; } } mpiRecvBinData += thisMpiRecvBinData; end: #ifdef VERBOSE_DEBUG_MPI_SENDRCV std::cout << "-->> MPI " << mpiRank << " Received " << thisMpiRecvUnitData << " units" << std::endl; std::cout << "-->> MPI " << mpiRank << " Received " << thisMpiRecvBinData << " bins" << std::endl; std::cout << "-->> MPI " << mpiRank << " Received " << thisMpiRecvTriData << " tris" << std::endl; #endif delete[] buf; return solver->ok; } void DataSync::syncToMPI() { int err; if (mpiSendData != NULL) { MPI_Status status; err = MPI_Wait(&sendReq, &status); assert(err == MPI_SUCCESS); delete mpiSendData; mpiSendData = NULL; } vector data; data.push_back((uint32_t)solver->nVars()); for (uint32_t var = 0; var < solver->nVars(); var++) { data.push_back(toInt(solver->value(var))); } //Binary uint32_t thisMpiSentBinData = 0; data.push_back((uint32_t)solver->nVars()*2); uint32_t wsLit = 0; for(auto it = sharedData->bins.begin() , end = sharedData->bins.end(); it != end; it++, wsLit++ ) { //Lit lit1 = ~Lit::toLit(wsLit); assert(it->data->size() >= syncMPIFinish[wsLit]); uint32_t sizeToSend = it->data->size() - syncMPIFinish[wsLit]; data.push_back(sizeToSend); for (uint32_t i = syncMPIFinish[wsLit]; i < it->data->size(); i++) { data.push_back(it->data->at(i).toInt()); thisMpiSentBinData++; } syncMPIFinish[wsLit] = it->data->size(); } assert(wsLit == solver->nVars()*2); mpiSentBinData += thisMpiSentBinData; #ifdef VERBOSE_DEBUG_MPI_SENDRCV std::cout << "-->> MPI " << mpiRank << " Sent " << data.size() << " uint32_t -s" << std::endl; std::cout << "-->> MPI " << mpiRank << " Sent " << thisMpiSentBinData << " bins " << std::endl; std::cout << "-->> MPI " << mpiRank << " Sent " << thisMpiSentTriData << " tris " << std::endl; #endif mpiSendData = new uint32_t[data.size()]; std::copy(data.begin(), data.end(), mpiSendData); err = MPI_Isend(mpiSendData, data.size(), MPI_UNSIGNED, 0, 0, MPI_COMM_WORLD, &sendReq); assert(err == MPI_SUCCESS); } bool DataSync::sync_mpi_unit( const lbool otherVal, const uint32_t var, SharedData* shared, uint32_t& thisGotUnitData, uint32_t& thisSentUnitData ) { Lit l = Lit(var, false); Lit lit1 = solver->map_to_with_bva(l); lit1 = solver->varReplacer->get_lit_replaced_with_outer(lit1); lit1 = solver->map_outer_to_inter(lit1); const lbool thisVal = solver->value(lit1); if (thisVal == l_Undef && otherVal == l_Undef) { return true; } if (thisVal != l_Undef && otherVal != l_Undef) { if (thisVal != otherVal) { solver->ok = false; return false; } else { return true; } } if (otherVal != l_Undef) { assert(thisVal == l_Undef); Lit litToEnqueue = lit1 ^ (otherVal == l_False); if (solver->varData[litToEnqueue.var()].removed != Removed::none) { return true; } solver->enqueue(litToEnqueue); solver->ok = solver->propagate().isNULL(); if (!solver->ok) { return false; } thisGotUnitData++; return true; } if (shared != NULL && thisVal != l_Undef) { assert(otherVal == l_Undef); shared->value[var] = thisVal; thisSentUnitData++; return true; } return true; } #endif