1 /** 2 * UGENE - Integrated Bioinformatics Tools. 3 * Copyright (C) 2008-2021 UniPro <ugene@unipro.ru> 4 * http://ugene.net 5 * 6 * This program is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU General Public License 8 * as published by the Free Software Foundation; either version 2 9 * of the License, or (at your option) any later version. 10 * 11 * This program is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program; if not, write to the Free Software 18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 19 * MA 02110-1301, USA. 20 */ 21 22 #ifndef _U2_MYSQL_ASSEMBLY_MULTI_TABLE_DBI_H_ 23 #define _U2_MYSQL_ASSEMBLY_MULTI_TABLE_DBI_H_ 24 25 #include <QReadWriteLock> 26 27 #include <U2Core/U2SqlHelpers.h> 28 29 #include "MysqlAssemblyAdapter.h" 30 #include "util/AssemblyPackAlgorithm.h" 31 32 namespace U2 { 33 34 class MysqlDbi; 35 class MysqlSingleTableAssemblyAdapter; 36 class MysqlSingleTablePackAlgorithmAdapter; 37 38 class MysqlMtaSingleTableAdapter { 39 public: 40 /** Wrapper over 1 table in database. If singleTableAdapter == NULL the table was not created yet */ 41 MysqlMtaSingleTableAdapter(MysqlSingleTableAssemblyAdapter *adapter, int rowPos, int elenPos, const QByteArray &extra); 42 43 MysqlSingleTableAssemblyAdapter *singleTableAdapter; 44 int rowPos; 45 int elenPos; 46 QByteArray idExtra; 47 }; 48 49 class MysqlMultiTableAssemblyAdapter : public MysqlAssemblyAdapter { 50 public: 51 MysqlMultiTableAssemblyAdapter(MysqlDbi *dbi, const U2DataId &assemblyId, const AssemblyCompressor *compressor, MysqlDbRef *ref, U2OpStatus &os); 52 ~MysqlMultiTableAssemblyAdapter(); 53 54 virtual qint64 countReads(const U2Region ®ion, U2OpStatus &os); 55 56 virtual qint64 getMaxPackedRow(const U2Region ®ion, U2OpStatus &os); 57 virtual qint64 getMaxEndPos(U2OpStatus &os); 58 59 virtual U2DbiIterator<U2AssemblyRead> *getReads(const U2Region ®ion, U2OpStatus &os, bool sortedHint = false); 60 virtual U2DbiIterator<U2AssemblyRead> *getReadsByRow(const U2Region ®ion, qint64 minRow, qint64 maxRow, U2OpStatus &os); 61 virtual U2DbiIterator<U2AssemblyRead> *getReadsByName(const QByteArray &name, U2OpStatus &os); 62 63 virtual void addReads(U2DbiIterator<U2AssemblyRead> *it, U2AssemblyReadsImportInfo &ii, U2OpStatus &os); 64 virtual void removeReads(const QList<U2DataId> &readIds, U2OpStatus &os); 65 virtual void dropReadsTables(U2OpStatus &os); 66 67 virtual void pack(U2AssemblyPackStat &stat, U2OpStatus &os); 68 virtual void calculateCoverage(const U2Region ®ion, U2AssemblyCoverageStat &coverage, U2OpStatus &os); 69 70 virtual void createReadsIndexes(U2OpStatus &os); 71 72 int getElenRangePosByLength(qint64 readLength) const; 73 int getElenRangePosById(const U2DataId &id) const; 74 int getNumberOfElenRanges() const; 75 76 int getRowRangePosByRow(quint64 row) const; 77 int getRowRangePosById(const U2DataId &id) const; 78 int getRowsPerRange() const; 79 80 const QVector<MysqlMtaSingleTableAdapter *> &getAdapters() const; 81 const QVector<QByteArray> &getIdExtrasPerRange() const; 82 83 MysqlDbRef *getDbRef() const; 84 85 MysqlMtaSingleTableAdapter *getAdapterByRowAndElenRange(int rowRange, int elenRange, bool createIfNotExits, U2OpStatus &os); 86 87 private: 88 QString getTableSuffix(int rowRange, int elenRange); 89 static QByteArray getIdExtra(int rowRange, int elenRange); 90 91 /** Checks if table info must be re-read from DB and calls re-read if needed */ 92 void syncTables(U2OpStatus &os); 93 94 /** For a new and empty assembly analyzes reads data and calculate ranges */ 95 void initTables(const QList<U2AssemblyRead> &reads, U2OpStatus &os); 96 97 /** Re-reads table info from DB */ 98 void rereadTables(const QByteArray &idata, U2OpStatus &os); 99 100 /** Flushes tables info into database */ 101 void flushTables(U2OpStatus &os); 102 103 void clearTableAdaptersInfo(); 104 105 MysqlMtaSingleTableAdapter *createAdapter(int rowRange, int elenRange, U2OpStatus &os); 106 107 void initAdaptersGrid(int nRows, int nRanges); 108 109 MysqlDbi *dbi; 110 111 /** All non-NUL adapters */ 112 QVector<MysqlMtaSingleTableAdapter *> adapters; 113 114 /** outer dim -> prow, inner dim -> elen */ 115 QVector<QVector<MysqlMtaSingleTableAdapter *>> adaptersGrid; 116 117 /** id extras for every table, same hierarchy with tableAdapters */ 118 QVector<QByteArray> idExtras; 119 120 /** assembly object version adapters are used for */ 121 qint32 version; 122 123 /** effective length ranges */ 124 QVector<U2Region> elenRanges; 125 126 /** prow range per table */ 127 qint32 rowsPerRange; 128 129 // TODO: add read-locks into all methods 130 QReadWriteLock tablesSyncLock; 131 132 static const int DEFAULT_ROWS_PER_TABLE = 5000; 133 static const int N_READS_TO_FLUSH_TOTAL = 100000; 134 static const int N_READS_TO_FLUSH_PER_RANGE = 10000; 135 }; 136 137 class MysqlReadTableMigrationData { 138 public: 139 MysqlReadTableMigrationData(); 140 MysqlReadTableMigrationData(qint64 oldId, MysqlMtaSingleTableAdapter *oldT, int newP); 141 142 qint64 readId; 143 MysqlMtaSingleTableAdapter *oldTable; 144 int newProw; 145 }; 146 147 class MysqlMultiTablePackAlgorithmAdapter : public PackAlgorithmAdapter { 148 public: 149 MysqlMultiTablePackAlgorithmAdapter(MysqlMultiTableAssemblyAdapter *a); 150 ~MysqlMultiTablePackAlgorithmAdapter(); 151 152 virtual U2DbiIterator<PackAlgorithmData> *selectAllReads(U2OpStatus &os); 153 virtual void assignProw(const U2DataId &readId, qint64 prow, U2OpStatus &os); 154 155 void releaseDbResources(); 156 void migrateAll(U2OpStatus &os); 157 158 private: 159 void ensureGridSize(int nRows); 160 void migrate(MysqlMtaSingleTableAdapter *newA, const QVector<MysqlReadTableMigrationData> &data, qint64 migratedBefore, qint64 totalMigrationCount, U2OpStatus &os); 161 162 MysqlMultiTableAssemblyAdapter *multiTableAdapter; 163 QVector<MysqlSingleTablePackAlgorithmAdapter *> packAdapters; 164 QVector<QVector<MysqlSingleTablePackAlgorithmAdapter *>> packAdaptersGrid; 165 QHash<MysqlMtaSingleTableAdapter *, QVector<MysqlReadTableMigrationData>> migrations; 166 167 static const int MAX_PERCENT_TO_REINDEX = 20; 168 }; 169 170 // Class that multiplexes multiple read iterators into 1 171 class MysqlMtaReadsIterator : public U2DbiIterator<U2AssemblyRead> { 172 public: 173 MysqlMtaReadsIterator(QVector<U2DbiIterator<U2AssemblyRead> *> &iterators, const QVector<QByteArray> &idExtras, bool sortedHint); 174 175 virtual ~MysqlMtaReadsIterator(); 176 177 virtual bool hasNext(); 178 179 virtual U2AssemblyRead next(); 180 181 virtual U2AssemblyRead peek(); 182 183 private: 184 QVector<U2DbiIterator<U2AssemblyRead> *> iterators; 185 int currentRange; 186 QVector<QByteArray> idExtras; 187 bool sortedHint; 188 }; 189 190 // Class that multiplexes multiple read packed data iterators into 1 and supports ordering 191 class MysqlMTAPackAlgorithmDataIterator : public U2DbiIterator<PackAlgorithmData> { 192 public: 193 MysqlMTAPackAlgorithmDataIterator(QVector<U2DbiIterator<PackAlgorithmData> *> &iterators, const QVector<QByteArray> &idExtras); 194 195 virtual ~MysqlMTAPackAlgorithmDataIterator(); 196 197 virtual bool hasNext(); 198 199 virtual PackAlgorithmData next(); 200 201 virtual PackAlgorithmData peek(); 202 203 private: 204 void fetchNextData(); 205 206 QVector<U2DbiIterator<PackAlgorithmData> *> iterators; 207 PackAlgorithmData nextData; 208 QVector<QByteArray> idExtras; 209 }; 210 211 } // namespace U2 212 213 #endif // _U2_MYSQL_ASSEMBLY_MULTI_TABLE_DBI_H_ 214