1 /**
2 * UGENE - Integrated Bioinformatics Tools.
3 * Copyright (C) 2008-2021 UniPro <ugene@unipro.ru>
4 * http://ugene.net
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version 2
9 * of the License, or (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
19 * MA 02110-1301, USA.
20 */
21
22 #include "BAMDbiPlugin.h"
23
24 #include <QAction>
25 #include <QDir>
26 #include <QMainWindow>
27 #include <QMenu>
28 #include <QMessageBox>
29 #include <QTemporaryFile>
30
31 #include <U2Core/AddDocumentTask.h>
32 #include <U2Core/AppContext.h>
33 #include <U2Core/AppSettings.h>
34 #include <U2Core/AssemblyObject.h>
35 #include <U2Core/CloneObjectTask.h>
36 #include <U2Core/DbiDocumentFormat.h>
37 #include <U2Core/DocumentUtils.h>
38 #include <U2Core/FileAndDirectoryUtils.h>
39 #include <U2Core/GUrlUtils.h>
40 #include <U2Core/IOAdapter.h>
41 #include <U2Core/IOAdapterUtils.h>
42 #include <U2Core/LoadDocumentTask.h>
43 #include <U2Core/ProjectModel.h>
44 #include <U2Core/QObjectScopedPointer.h>
45 #include <U2Core/TaskSignalMapper.h>
46 #include <U2Core/TextUtils.h>
47 #include <U2Core/U2DbiRegistry.h>
48 #include <U2Core/U2OpStatusUtils.h>
49 #include <U2Core/U2SafePoints.h>
50 #include <U2Core/UserApplicationsSettings.h>
51
52 #include <U2Formats/SAMFormat.h>
53
54 #include <U2Gui/LastUsedDirHelper.h>
55 #include <U2Gui/MainWindow.h>
56 #include <U2Gui/OpenViewTask.h>
57
58 #include "BAMFormat.h"
59 #include "ConvertToSQLiteDialog.h"
60 #include "ConvertToSQLiteTask.h"
61 #include "Dbi.h"
62 #include "Exception.h"
63 #include "LoadBamInfoTask.h"
64 #include "SamtoolsBasedDbi.h"
65
66 namespace U2 {
67 namespace BAM {
68
U2_PLUGIN_INIT_FUNC()69 extern "C" Q_DECL_EXPORT Plugin *U2_PLUGIN_INIT_FUNC() {
70 BAMDbiPlugin *plug = new BAMDbiPlugin();
71 return plug;
72 }
73
BAMDbiPlugin()74 BAMDbiPlugin::BAMDbiPlugin()
75 : Plugin(tr("BAM format support"), tr("Interface for indexed read-only access to BAM files")) {
76 DocumentFormat *bamDbi = new BAMFormat();
77 AppContext::getDocumentFormatRegistry()->registerFormat(bamDbi);
78 AppContext::getDbiRegistry()->registerDbiFactory(new SamtoolsBasedDbiFactory());
79
80 AppContext::getDocumentFormatRegistry()->getImportSupport()->addDocumentImporter(new BAMImporter());
81 }
82
83 //////////////////////////////////////////////////////////////////////////
84 // BAM importer
BAMImporter()85 BAMImporter::BAMImporter()
86 : DocumentImporter("bam-importer", tr("BAM/SAM file import")) {
87 // prepare sorted extensions list
88 QSet<QString> extsSet;
89 BAMFormatUtils bam;
90 SAMFormat sam;
91 extsSet.unite(bam.getSupportedDocumentFileExtensions().toSet()).unite(sam.getSupportedDocumentFileExtensions().toSet());
92 QStringList exts = extsSet.toList();
93 std::sort(exts.begin(), exts.end());
94
95 formatIds << BaseDocumentFormats::BAM << BaseDocumentFormats::SAM;
96 extensions << exts;
97 importerDescription = tr("BAM files importer is used to convert conventional BAM and SAM files into UGENE database format. Having BAM or SAM file converted into UGENE DB format you get an fast and efficient interface to your data with an option to change the content");
98 supportedObjectTypes << GObjectTypes::ASSEMBLY;
99 }
100
101 #define SAM_HINT "bam-importer-sam-hint"
102
checkRawData(const QByteArray & rawData,const GUrl & url)103 FormatCheckResult BAMImporter::checkRawData(const QByteArray &rawData, const GUrl &url) {
104 BAMFormatUtils bamFormatUtils;
105 FormatCheckResult bamScore = bamFormatUtils.checkRawData(rawData, url);
106
107 SAMFormat samFormat;
108 FormatCheckResult samScore = samFormat.checkRawData(rawData, url);
109
110 if (bamScore.score > samScore.score) {
111 return bamScore;
112 }
113 samScore.properties[SAM_HINT] = true;
114 return samScore;
115 }
116
createImportTask(const FormatDetectionResult & res,bool showGui,const QVariantMap & hints)117 DocumentProviderTask *BAMImporter::createImportTask(const FormatDetectionResult &res, bool showGui, const QVariantMap &hints) {
118 bool sam = res.rawDataCheckResult.properties[SAM_HINT].toBool();
119 QVariantMap fullHints(hints);
120 fullHints[SAM_HINT] = sam;
121 return new BAMImporterTask(res.url, showGui, fullHints);
122 }
123
BAMImporterTask(const GUrl & url,bool _useGui,const QVariantMap & hints)124 BAMImporterTask::BAMImporterTask(const GUrl &url, bool _useGui, const QVariantMap &hints)
125 : DocumentProviderTask(tr("BAM/SAM file import: %1").arg(url.fileName()), TaskFlags_NR_FOSCOE),
126 loadInfoTask(nullptr),
127 loadBamInfoTask(nullptr),
128 prepareToImportTask(nullptr),
129 convertTask(nullptr),
130 loadDocTask(nullptr),
131 isSqliteDbTransit(false),
132 useGui(_useGui),
133 sam(hints.value(SAM_HINT, false).toBool()),
134 hints(hints),
135 hintedDbiRef(hints.value(DocumentFormat::DBI_REF_HINT).value<U2DbiRef>()),
136 startTime(0) {
137 documentDescription = url.fileName();
138 loadInfoTask = new LoadInfoTask(url, sam);
139 addSubTask(loadInfoTask);
140 }
141
prepare()142 void BAMImporterTask::prepare() {
143 startTime = time(0);
144 }
145
146 namespace {
getDirUrl(const GUrl & fileUrl)147 QString getDirUrl(const GUrl &fileUrl) {
148 return QFileInfo(fileUrl.getURLString()).dir().absolutePath();
149 }
150 } // namespace
151
onSubTaskFinished(Task * subTask)152 QList<Task *> BAMImporterTask::onSubTaskFinished(Task *subTask) {
153 QList<Task *> res;
154
155 if (subTask->hasError()) {
156 propagateSubtaskError();
157 return res;
158 }
159
160 if (loadInfoTask == subTask) {
161 initPrepareToImportTask();
162 CHECK(nullptr != prepareToImportTask, res);
163 res << prepareToImportTask;
164 }
165
166 else if (prepareToImportTask == subTask && prepareToImportTask->isNewURL()) {
167 initLoadBamInfoTask();
168 CHECK(nullptr != loadBamInfoTask, res);
169 res << loadBamInfoTask;
170 }
171
172 else if (loadBamInfoTask == subTask || prepareToImportTask == subTask) {
173 initConvertToSqliteTask();
174 CHECK(nullptr != convertTask, res);
175 res << convertTask;
176 }
177
178 else if (isSqliteDbTransit && convertTask == subTask) {
179 initCloneObjectTasks();
180 CHECK(!cloneTasks.isEmpty(), res);
181 res << cloneTasks;
182 }
183
184 else if (!isSqliteDbTransit && convertTask == subTask) {
185 initLoadDocumentTask();
186 CHECK(nullptr != loadDocTask, res);
187 res << loadDocTask;
188 }
189
190 else if ((isSqliteDbTransit && cloneTasks.contains(subTask))) {
191 cloneTasks.removeOne(subTask);
192 CloneObjectTask *cloneTask = qobject_cast<CloneObjectTask *>(subTask);
193 SAFE_POINT_EXT(nullptr != cloneTask, setError("Unexpected task type: CloneObjectTask expected"), res);
194 delete cloneTask->getSourceObject();
195
196 if (cloneTasks.isEmpty()) {
197 initLoadDocumentTask();
198 CHECK(nullptr != loadDocTask, res);
199 res << loadDocTask;
200 }
201 }
202
203 else if (subTask == loadDocTask) {
204 resultDocument = loadDocTask->takeDocument();
205 }
206
207 return res;
208 }
209
report()210 Task::ReportResult BAMImporterTask::report() {
211 time_t totalTime = time(0) - startTime;
212 taskLog.info(QString("BAMImporter task total time is %1 sec").arg(totalTime));
213 return ReportResult_Finished;
214 }
215
initPrepareToImportTask()216 void BAMImporterTask::initPrepareToImportTask() {
217 GUrl srcUrl = loadInfoTask->getSourceUrl();
218
219 isSqliteDbTransit = hintedDbiRef.isValid() && SQLITE_DBI_ID != hintedDbiRef.dbiFactoryId;
220 if (!isSqliteDbTransit) {
221 localDbiRef = U2DbiRef(SQLITE_DBI_ID, srcUrl.dirPath() + QDir::separator() + srcUrl.fileName() + ".ugenedb");
222 } else {
223 const QString tmpDir = AppContext::getAppSettings()->getUserAppsSettings()->getCurrentProcessTemporaryDirPath("assembly_conversion") + QDir::separator();
224 QDir().mkpath(tmpDir);
225
226 const QString pattern = tmpDir + "XXXXXX.ugenedb";
227 QTemporaryFile *tempLocalDb = new QTemporaryFile(pattern, this);
228
229 tempLocalDb->open();
230 const QString filePath = tempLocalDb->fileName();
231 tempLocalDb->close();
232
233 SAFE_POINT_EXT(QFile::exists(filePath), setError(tr("Can't create a temporary database")), );
234
235 localDbiRef = U2DbiRef(SQLITE_DBI_ID, filePath);
236 }
237
238 QString refUrl;
239 bool convert = true;
240 if (useGui) {
241 QObjectScopedPointer<ConvertToSQLiteDialog> convertDialog = new ConvertToSQLiteDialog(loadInfoTask->getSourceUrl(), loadInfoTask->getInfo(), loadInfoTask->isSam());
242 convertDialog->hideAddToProjectOption();
243 const int rc = convertDialog->exec();
244 CHECK_EXT(!convertDialog.isNull(), setError("NULL dialog"), );
245
246 if (rc == QDialog::Accepted) {
247 localDbiRef = U2DbiRef(SQLITE_DBI_ID, convertDialog->getDestinationUrl().getURLString());
248 refUrl = convertDialog->getReferenceUrl();
249
250 } else {
251 convert = false;
252 stateInfo.setCanceled(true);
253 }
254 } else if (loadInfoTask->isSam() && loadInfoTask->getInfo().getHeader().getReferences().isEmpty()) {
255 convert = false;
256 setError(tr("SAM cannot be converted to BAM: neither reference nor header in SAM file is present"));
257 }
258
259 if (convert) {
260 QString dirUrl = getDirUrl(loadInfoTask->getSourceUrl());
261 if (!FileAndDirectoryUtils::isDirectoryWritable(dirUrl)) {
262 const GUrl url(U2DbiUtils::ref2Url(localDbiRef));
263 if (url.isLocalFile()) {
264 dirUrl = getDirUrl(url);
265 } else {
266 dirUrl = getDirUrl(AppContext::getAppSettings()->getUserAppsSettings()->getUserTemporaryDirPath());
267 }
268 }
269 prepareToImportTask = new PrepareToImportTask(loadInfoTask->getSourceUrl(), loadInfoTask->isSam(), refUrl, dirUrl);
270 }
271 }
272
initLoadBamInfoTask()273 void BAMImporterTask::initLoadBamInfoTask() {
274 bool samFormat = false;
275 loadBamInfoTask = new LoadInfoTask(prepareToImportTask->getSourceUrl(), samFormat);
276 }
277
initConvertToSqliteTask()278 void BAMImporterTask::initConvertToSqliteTask() {
279 bool samFormat = false;
280 GUrl sourceURL;
281 BAMInfo bamInfo;
282 if (prepareToImportTask->isNewURL()) {
283 sourceURL = loadBamInfoTask->getSourceUrl();
284 bamInfo = loadBamInfoTask->getInfo();
285 } else {
286 sourceURL = prepareToImportTask->getSourceUrl();
287 bamInfo = loadInfoTask->getInfo();
288 }
289 convertTask = new ConvertToSQLiteTask(sourceURL, localDbiRef, bamInfo, samFormat);
290 }
291
initCloneObjectTasks()292 void BAMImporterTask::initCloneObjectTasks() {
293 QList<U2Assembly> assemblies = convertTask->getAssemblies();
294 foreach (const U2Assembly &assembly, assemblies) {
295 AssemblyObject *object = new AssemblyObject(assembly.visualName, U2EntityRef(localDbiRef, assembly.id));
296 cloneTasks << new CloneObjectTask(object, hintedDbiRef, hints.value(DocumentFormat::DBI_FOLDER_HINT, U2ObjectDbi::ROOT_FOLDER).toString());
297 }
298 }
299
initLoadDocumentTask()300 void BAMImporterTask::initLoadDocumentTask() {
301 if (hints.value(BAMImporter::LOAD_RESULT_DOCUMENT, true).toBool()) {
302 loadDocTask = LoadDocumentTask::getDefaultLoadDocTask(convertTask->getDestinationUrl());
303 if (loadDocTask == nullptr) {
304 setError(tr("Failed to get load task for : %1").arg(convertTask->getDestinationUrl().getURLString()));
305 }
306 }
307 }
308
309 } // namespace BAM
310 } // namespace U2
311