1 /**
2  * UGENE - Integrated Bioinformatics Tools.
3  * Copyright (C) 2008-2021 UniPro <ugene@unipro.ru>
4  * http://ugene.net
5  *
6  * This program is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; either version 2
9  * of the License, or (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
19  * MA 02110-1301, USA.
20  */
21 
22 #include "BAMDbiPlugin.h"
23 
24 #include <QAction>
25 #include <QDir>
26 #include <QMainWindow>
27 #include <QMenu>
28 #include <QMessageBox>
29 #include <QTemporaryFile>
30 
31 #include <U2Core/AddDocumentTask.h>
32 #include <U2Core/AppContext.h>
33 #include <U2Core/AppSettings.h>
34 #include <U2Core/AssemblyObject.h>
35 #include <U2Core/CloneObjectTask.h>
36 #include <U2Core/DbiDocumentFormat.h>
37 #include <U2Core/DocumentUtils.h>
38 #include <U2Core/FileAndDirectoryUtils.h>
39 #include <U2Core/GUrlUtils.h>
40 #include <U2Core/IOAdapter.h>
41 #include <U2Core/IOAdapterUtils.h>
42 #include <U2Core/LoadDocumentTask.h>
43 #include <U2Core/ProjectModel.h>
44 #include <U2Core/QObjectScopedPointer.h>
45 #include <U2Core/TaskSignalMapper.h>
46 #include <U2Core/TextUtils.h>
47 #include <U2Core/U2DbiRegistry.h>
48 #include <U2Core/U2OpStatusUtils.h>
49 #include <U2Core/U2SafePoints.h>
50 #include <U2Core/UserApplicationsSettings.h>
51 
52 #include <U2Formats/SAMFormat.h>
53 
54 #include <U2Gui/LastUsedDirHelper.h>
55 #include <U2Gui/MainWindow.h>
56 #include <U2Gui/OpenViewTask.h>
57 
58 #include "BAMFormat.h"
59 #include "ConvertToSQLiteDialog.h"
60 #include "ConvertToSQLiteTask.h"
61 #include "Dbi.h"
62 #include "Exception.h"
63 #include "LoadBamInfoTask.h"
64 #include "SamtoolsBasedDbi.h"
65 
66 namespace U2 {
67 namespace BAM {
68 
U2_PLUGIN_INIT_FUNC()69 extern "C" Q_DECL_EXPORT Plugin *U2_PLUGIN_INIT_FUNC() {
70     BAMDbiPlugin *plug = new BAMDbiPlugin();
71     return plug;
72 }
73 
BAMDbiPlugin()74 BAMDbiPlugin::BAMDbiPlugin()
75     : Plugin(tr("BAM format support"), tr("Interface for indexed read-only access to BAM files")) {
76     DocumentFormat *bamDbi = new BAMFormat();
77     AppContext::getDocumentFormatRegistry()->registerFormat(bamDbi);
78     AppContext::getDbiRegistry()->registerDbiFactory(new SamtoolsBasedDbiFactory());
79 
80     AppContext::getDocumentFormatRegistry()->getImportSupport()->addDocumentImporter(new BAMImporter());
81 }
82 
83 //////////////////////////////////////////////////////////////////////////
84 // BAM importer
BAMImporter()85 BAMImporter::BAMImporter()
86     : DocumentImporter("bam-importer", tr("BAM/SAM file import")) {
87     // prepare sorted extensions list
88     QSet<QString> extsSet;
89     BAMFormatUtils bam;
90     SAMFormat sam;
91     extsSet.unite(bam.getSupportedDocumentFileExtensions().toSet()).unite(sam.getSupportedDocumentFileExtensions().toSet());
92     QStringList exts = extsSet.toList();
93     std::sort(exts.begin(), exts.end());
94 
95     formatIds << BaseDocumentFormats::BAM << BaseDocumentFormats::SAM;
96     extensions << exts;
97     importerDescription = tr("BAM files importer is used to convert conventional BAM and SAM files into UGENE database format. Having BAM or SAM file converted into UGENE DB format you get an fast and efficient interface to your data with an option to change the content");
98     supportedObjectTypes << GObjectTypes::ASSEMBLY;
99 }
100 
101 #define SAM_HINT "bam-importer-sam-hint"
102 
checkRawData(const QByteArray & rawData,const GUrl & url)103 FormatCheckResult BAMImporter::checkRawData(const QByteArray &rawData, const GUrl &url) {
104     BAMFormatUtils bamFormatUtils;
105     FormatCheckResult bamScore = bamFormatUtils.checkRawData(rawData, url);
106 
107     SAMFormat samFormat;
108     FormatCheckResult samScore = samFormat.checkRawData(rawData, url);
109 
110     if (bamScore.score > samScore.score) {
111         return bamScore;
112     }
113     samScore.properties[SAM_HINT] = true;
114     return samScore;
115 }
116 
createImportTask(const FormatDetectionResult & res,bool showGui,const QVariantMap & hints)117 DocumentProviderTask *BAMImporter::createImportTask(const FormatDetectionResult &res, bool showGui, const QVariantMap &hints) {
118     bool sam = res.rawDataCheckResult.properties[SAM_HINT].toBool();
119     QVariantMap fullHints(hints);
120     fullHints[SAM_HINT] = sam;
121     return new BAMImporterTask(res.url, showGui, fullHints);
122 }
123 
BAMImporterTask(const GUrl & url,bool _useGui,const QVariantMap & hints)124 BAMImporterTask::BAMImporterTask(const GUrl &url, bool _useGui, const QVariantMap &hints)
125     : DocumentProviderTask(tr("BAM/SAM file import: %1").arg(url.fileName()), TaskFlags_NR_FOSCOE),
126       loadInfoTask(nullptr),
127       loadBamInfoTask(nullptr),
128       prepareToImportTask(nullptr),
129       convertTask(nullptr),
130       loadDocTask(nullptr),
131       isSqliteDbTransit(false),
132       useGui(_useGui),
133       sam(hints.value(SAM_HINT, false).toBool()),
134       hints(hints),
135       hintedDbiRef(hints.value(DocumentFormat::DBI_REF_HINT).value<U2DbiRef>()),
136       startTime(0) {
137     documentDescription = url.fileName();
138     loadInfoTask = new LoadInfoTask(url, sam);
139     addSubTask(loadInfoTask);
140 }
141 
prepare()142 void BAMImporterTask::prepare() {
143     startTime = time(0);
144 }
145 
146 namespace {
getDirUrl(const GUrl & fileUrl)147 QString getDirUrl(const GUrl &fileUrl) {
148     return QFileInfo(fileUrl.getURLString()).dir().absolutePath();
149 }
150 }  // namespace
151 
onSubTaskFinished(Task * subTask)152 QList<Task *> BAMImporterTask::onSubTaskFinished(Task *subTask) {
153     QList<Task *> res;
154 
155     if (subTask->hasError()) {
156         propagateSubtaskError();
157         return res;
158     }
159 
160     if (loadInfoTask == subTask) {
161         initPrepareToImportTask();
162         CHECK(nullptr != prepareToImportTask, res);
163         res << prepareToImportTask;
164     }
165 
166     else if (prepareToImportTask == subTask && prepareToImportTask->isNewURL()) {
167         initLoadBamInfoTask();
168         CHECK(nullptr != loadBamInfoTask, res);
169         res << loadBamInfoTask;
170     }
171 
172     else if (loadBamInfoTask == subTask || prepareToImportTask == subTask) {
173         initConvertToSqliteTask();
174         CHECK(nullptr != convertTask, res);
175         res << convertTask;
176     }
177 
178     else if (isSqliteDbTransit && convertTask == subTask) {
179         initCloneObjectTasks();
180         CHECK(!cloneTasks.isEmpty(), res);
181         res << cloneTasks;
182     }
183 
184     else if (!isSqliteDbTransit && convertTask == subTask) {
185         initLoadDocumentTask();
186         CHECK(nullptr != loadDocTask, res);
187         res << loadDocTask;
188     }
189 
190     else if ((isSqliteDbTransit && cloneTasks.contains(subTask))) {
191         cloneTasks.removeOne(subTask);
192         CloneObjectTask *cloneTask = qobject_cast<CloneObjectTask *>(subTask);
193         SAFE_POINT_EXT(nullptr != cloneTask, setError("Unexpected task type: CloneObjectTask expected"), res);
194         delete cloneTask->getSourceObject();
195 
196         if (cloneTasks.isEmpty()) {
197             initLoadDocumentTask();
198             CHECK(nullptr != loadDocTask, res);
199             res << loadDocTask;
200         }
201     }
202 
203     else if (subTask == loadDocTask) {
204         resultDocument = loadDocTask->takeDocument();
205     }
206 
207     return res;
208 }
209 
report()210 Task::ReportResult BAMImporterTask::report() {
211     time_t totalTime = time(0) - startTime;
212     taskLog.info(QString("BAMImporter task total time is %1 sec").arg(totalTime));
213     return ReportResult_Finished;
214 }
215 
initPrepareToImportTask()216 void BAMImporterTask::initPrepareToImportTask() {
217     GUrl srcUrl = loadInfoTask->getSourceUrl();
218 
219     isSqliteDbTransit = hintedDbiRef.isValid() && SQLITE_DBI_ID != hintedDbiRef.dbiFactoryId;
220     if (!isSqliteDbTransit) {
221         localDbiRef = U2DbiRef(SQLITE_DBI_ID, srcUrl.dirPath() + QDir::separator() + srcUrl.fileName() + ".ugenedb");
222     } else {
223         const QString tmpDir = AppContext::getAppSettings()->getUserAppsSettings()->getCurrentProcessTemporaryDirPath("assembly_conversion") + QDir::separator();
224         QDir().mkpath(tmpDir);
225 
226         const QString pattern = tmpDir + "XXXXXX.ugenedb";
227         QTemporaryFile *tempLocalDb = new QTemporaryFile(pattern, this);
228 
229         tempLocalDb->open();
230         const QString filePath = tempLocalDb->fileName();
231         tempLocalDb->close();
232 
233         SAFE_POINT_EXT(QFile::exists(filePath), setError(tr("Can't create a temporary database")), );
234 
235         localDbiRef = U2DbiRef(SQLITE_DBI_ID, filePath);
236     }
237 
238     QString refUrl;
239     bool convert = true;
240     if (useGui) {
241         QObjectScopedPointer<ConvertToSQLiteDialog> convertDialog = new ConvertToSQLiteDialog(loadInfoTask->getSourceUrl(), loadInfoTask->getInfo(), loadInfoTask->isSam());
242         convertDialog->hideAddToProjectOption();
243         const int rc = convertDialog->exec();
244         CHECK_EXT(!convertDialog.isNull(), setError("NULL dialog"), );
245 
246         if (rc == QDialog::Accepted) {
247             localDbiRef = U2DbiRef(SQLITE_DBI_ID, convertDialog->getDestinationUrl().getURLString());
248             refUrl = convertDialog->getReferenceUrl();
249 
250         } else {
251             convert = false;
252             stateInfo.setCanceled(true);
253         }
254     } else if (loadInfoTask->isSam() && loadInfoTask->getInfo().getHeader().getReferences().isEmpty()) {
255         convert = false;
256         setError(tr("SAM cannot be converted to BAM: neither reference nor header in SAM file is present"));
257     }
258 
259     if (convert) {
260         QString dirUrl = getDirUrl(loadInfoTask->getSourceUrl());
261         if (!FileAndDirectoryUtils::isDirectoryWritable(dirUrl)) {
262             const GUrl url(U2DbiUtils::ref2Url(localDbiRef));
263             if (url.isLocalFile()) {
264                 dirUrl = getDirUrl(url);
265             } else {
266                 dirUrl = getDirUrl(AppContext::getAppSettings()->getUserAppsSettings()->getUserTemporaryDirPath());
267             }
268         }
269         prepareToImportTask = new PrepareToImportTask(loadInfoTask->getSourceUrl(), loadInfoTask->isSam(), refUrl, dirUrl);
270     }
271 }
272 
initLoadBamInfoTask()273 void BAMImporterTask::initLoadBamInfoTask() {
274     bool samFormat = false;
275     loadBamInfoTask = new LoadInfoTask(prepareToImportTask->getSourceUrl(), samFormat);
276 }
277 
initConvertToSqliteTask()278 void BAMImporterTask::initConvertToSqliteTask() {
279     bool samFormat = false;
280     GUrl sourceURL;
281     BAMInfo bamInfo;
282     if (prepareToImportTask->isNewURL()) {
283         sourceURL = loadBamInfoTask->getSourceUrl();
284         bamInfo = loadBamInfoTask->getInfo();
285     } else {
286         sourceURL = prepareToImportTask->getSourceUrl();
287         bamInfo = loadInfoTask->getInfo();
288     }
289     convertTask = new ConvertToSQLiteTask(sourceURL, localDbiRef, bamInfo, samFormat);
290 }
291 
initCloneObjectTasks()292 void BAMImporterTask::initCloneObjectTasks() {
293     QList<U2Assembly> assemblies = convertTask->getAssemblies();
294     foreach (const U2Assembly &assembly, assemblies) {
295         AssemblyObject *object = new AssemblyObject(assembly.visualName, U2EntityRef(localDbiRef, assembly.id));
296         cloneTasks << new CloneObjectTask(object, hintedDbiRef, hints.value(DocumentFormat::DBI_FOLDER_HINT, U2ObjectDbi::ROOT_FOLDER).toString());
297     }
298 }
299 
initLoadDocumentTask()300 void BAMImporterTask::initLoadDocumentTask() {
301     if (hints.value(BAMImporter::LOAD_RESULT_DOCUMENT, true).toBool()) {
302         loadDocTask = LoadDocumentTask::getDefaultLoadDocTask(convertTask->getDestinationUrl());
303         if (loadDocTask == nullptr) {
304             setError(tr("Failed to get load task for : %1").arg(convertTask->getDestinationUrl().getURLString()));
305         }
306     }
307 }
308 
309 }  // namespace BAM
310 }  // namespace U2
311