1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
2
3 /*
4 Sonic Visualiser
5 An audio file viewer and annotation editor.
6 Centre for Digital Music, Queen Mary, University of London.
7 This file copyright 2008-2012 QMUL.
8
9 This program is free software; you can redistribute it and/or
10 modify it under the terms of the GNU General Public License as
11 published by the Free Software Foundation; either version 2 of the
12 License, or (at your option) any later version. See the file
13 COPYING included with this distribution for more information.
14 */
15
16 #include "PluginRDFIndexer.h"
17
18 #include "data/fileio/CachedFile.h"
19 #include "data/fileio/FileSource.h"
20 #include "data/fileio/PlaylistFileReader.h"
21 #include "plugin/PluginIdentifier.h"
22
23 #include "base/Profiler.h"
24 #include "base/Debug.h"
25
26 #include <vamp-hostsdk/PluginHostAdapter.h>
27
28 #include <dataquay/BasicStore.h>
29 #include <dataquay/RDFException.h>
30
31 #include <QFileInfo>
32 #include <QDir>
33 #include <QUrl>
34 #include <QDateTime>
35 #include <QSettings>
36 #include <QFile>
37
38 #include <iostream>
39
40 using std::vector;
41 using std::string;
42 using Vamp::PluginHostAdapter;
43
44 using Dataquay::Uri;
45 using Dataquay::Node;
46 using Dataquay::Nodes;
47 using Dataquay::Triple;
48 using Dataquay::Triples;
49 using Dataquay::BasicStore;
50 using Dataquay::RDFException;
51 using Dataquay::RDFDuplicateImportException;
52
53 PluginRDFIndexer *
54 PluginRDFIndexer::m_instance = nullptr;
55
56 PluginRDFIndexer *
getInstance()57 PluginRDFIndexer::getInstance()
58 {
59 if (!m_instance) m_instance = new PluginRDFIndexer();
60 return m_instance;
61 }
62
PluginRDFIndexer()63 PluginRDFIndexer::PluginRDFIndexer() :
64 m_index(new Dataquay::BasicStore)
65 {
66 m_index->addPrefix("vamp", Uri("http://purl.org/ontology/vamp/"));
67 m_index->addPrefix("foaf", Uri("http://xmlns.com/foaf/0.1/"));
68 m_index->addPrefix("dc", Uri("http://purl.org/dc/elements/1.1/"));
69 indexInstalledURLs();
70 }
71
72 const BasicStore *
getIndex()73 PluginRDFIndexer::getIndex()
74 {
75 return m_index;
76 }
77
~PluginRDFIndexer()78 PluginRDFIndexer::~PluginRDFIndexer()
79 {
80 QMutexLocker locker(&m_mutex);
81 }
82
83 void
indexInstalledURLs()84 PluginRDFIndexer::indexInstalledURLs()
85 {
86 vector<string> paths = PluginHostAdapter::getPluginPath();
87
88 // SVDEBUG << "\nPluginRDFIndexer::indexInstalledURLs: pid is " << getpid() << endl;
89
90 QStringList filters;
91 filters << "*.ttl";
92 filters << "*.TTL";
93 filters << "*.n3";
94 filters << "*.N3";
95 filters << "*.rdf";
96 filters << "*.RDF";
97
98 // Search each Vamp plugin path for an RDF file that either has
99 // name "soname", "soname:label" or "soname/label" plus RDF
100 // extension. Use that order of preference, and prefer ttl over
101 // n3 over rdf extension.
102
103 for (vector<string>::const_iterator i = paths.begin(); i != paths.end(); ++i) {
104
105 QDir dir(i->c_str());
106 if (!dir.exists()) continue;
107
108 QStringList entries = dir.entryList
109 (filters, QDir::Files | QDir::Readable);
110
111 for (QStringList::const_iterator j = entries.begin();
112 j != entries.end(); ++j) {
113
114 QFileInfo fi(dir.filePath(*j));
115 pullFile(fi.absoluteFilePath());
116 }
117
118 QStringList subdirs = dir.entryList
119 (QDir::AllDirs | QDir::NoDotAndDotDot | QDir::Readable);
120
121 for (QStringList::const_iterator j = subdirs.begin();
122 j != subdirs.end(); ++j) {
123
124 QDir subdir(dir.filePath(*j));
125 if (subdir.exists()) {
126 entries = subdir.entryList
127 (filters, QDir::Files | QDir::Readable);
128 for (QStringList::const_iterator k = entries.begin();
129 k != entries.end(); ++k) {
130 QFileInfo fi(subdir.filePath(*k));
131 pullFile(fi.absoluteFilePath());
132 }
133 }
134 }
135 }
136
137 reindex();
138 }
139
140 bool
indexConfiguredURLs()141 PluginRDFIndexer::indexConfiguredURLs()
142 {
143 SVDEBUG << "PluginRDFIndexer::indexConfiguredURLs" << endl;
144
145 QSettings settings;
146 settings.beginGroup("RDF");
147
148 QString indexKey("rdf-indices");
149 QStringList indices = settings.value(indexKey).toStringList();
150
151 for (int i = 0; i < indices.size(); ++i) {
152
153 QString index = indices[i];
154
155 SVDEBUG << "PluginRDFIndexer::indexConfiguredURLs: index url is "
156 << index << endl;
157
158 CachedFile cf(index);
159 if (!cf.isOK()) continue;
160
161 FileSource indexSource(cf.getLocalFilename());
162
163 PlaylistFileReader reader(indexSource);
164 if (!reader.isOK()) continue;
165
166 PlaylistFileReader::Playlist list = reader.load();
167 for (PlaylistFileReader::Playlist::const_iterator j = list.begin();
168 j != list.end(); ++j) {
169 SVDEBUG << "PluginRDFIndexer::indexConfiguredURLs: url is "
170 << *j << endl;
171 pullURL(*j);
172 }
173 }
174
175 QString urlListKey("rdf-urls");
176 QStringList urls = settings.value(urlListKey).toStringList();
177
178 for (int i = 0; i < urls.size(); ++i) {
179 pullURL(urls[i]);
180 }
181
182 settings.endGroup();
183 reindex();
184 return true;
185 }
186
187 QString
getURIForPluginId(QString pluginId)188 PluginRDFIndexer::getURIForPluginId(QString pluginId)
189 {
190 QMutexLocker locker(&m_mutex);
191
192 if (m_idToUriMap.find(pluginId) == m_idToUriMap.end()) return "";
193 return m_idToUriMap[pluginId];
194 }
195
196 QString
getIdForPluginURI(QString uri)197 PluginRDFIndexer::getIdForPluginURI(QString uri)
198 {
199 m_mutex.lock();
200
201 if (m_uriToIdMap.find(uri) == m_uriToIdMap.end()) {
202
203 m_mutex.unlock();
204
205 // Haven't found this uri referenced in any document on the
206 // local filesystem; try resolving the pre-fragment part of
207 // the uri as a document URL and reading that if possible.
208
209 // Because we may want to refer to this document again, we
210 // cache it locally if it turns out to exist.
211
212 SVDEBUG << "PluginRDFIndexer::getIdForPluginURI: NOTE: Failed to find a local RDF document describing plugin <" << uri << ">: attempting to retrieve one remotely by guesswork" << endl;
213
214 QString baseUrl = QUrl(uri).toString(QUrl::RemoveFragment);
215
216 indexURL(baseUrl);
217
218 m_mutex.lock();
219
220 if (m_uriToIdMap.find(uri) == m_uriToIdMap.end()) {
221 m_uriToIdMap[uri] = "";
222 }
223 }
224
225 QString id = m_uriToIdMap[uri];
226 m_mutex.unlock();
227 return id;
228 }
229
230 QStringList
getIndexedPluginIds()231 PluginRDFIndexer::getIndexedPluginIds()
232 {
233 QMutexLocker locker(&m_mutex);
234
235 QStringList ids;
236 for (StringMap::const_iterator i = m_idToUriMap.begin();
237 i != m_idToUriMap.end(); ++i) {
238 ids.push_back(i->first);
239 }
240 return ids;
241 }
242
243 bool
pullFile(QString filepath)244 PluginRDFIndexer::pullFile(QString filepath)
245 {
246 QUrl url = QUrl::fromLocalFile(filepath);
247 QString urlString = url.toString();
248 return pullURL(urlString);
249 }
250
251 bool
indexURL(QString urlString)252 PluginRDFIndexer::indexURL(QString urlString)
253 {
254 bool pulled = pullURL(urlString);
255 if (!pulled) return false;
256 reindex();
257 return true;
258 }
259
260 bool
pullURL(QString urlString)261 PluginRDFIndexer::pullURL(QString urlString)
262 {
263 Profiler profiler("PluginRDFIndexer::indexURL");
264
265 // SVDEBUG << "PluginRDFIndexer::indexURL(" << urlString << ")" << endl;
266
267 QMutexLocker locker(&m_mutex);
268
269 QUrl local = urlString;
270
271 if (FileSource::isRemote(urlString) &&
272 FileSource::canHandleScheme(urlString)) {
273
274 CachedFile cf(urlString, nullptr, "application/rdf+xml");
275 if (!cf.isOK()) {
276 return false;
277 }
278
279 local = QUrl::fromLocalFile(cf.getLocalFilename());
280
281 } else if (urlString.startsWith("file:")) {
282
283 local = QUrl(urlString);
284
285 } else {
286
287 local = QUrl::fromLocalFile(urlString);
288 }
289
290 try {
291 m_index->import(local, BasicStore::ImportFailOnDuplicates);
292 } catch (RDFDuplicateImportException &e) {
293 SVDEBUG << e.what() << endl;
294 SVDEBUG << "PluginRDFIndexer::pullURL: Document at " << urlString
295 << " duplicates triples found in earlier loaded document -- skipping it" << endl;
296 return false;
297 } catch (RDFException &e) {
298 SVDEBUG << e.what() << endl;
299 SVDEBUG << "PluginRDFIndexer::pullURL: Failed to import document from "
300 << urlString << ": " << e.what() << endl;
301 return false;
302 }
303 return true;
304 }
305
306 bool
reindex()307 PluginRDFIndexer::reindex()
308 {
309 Triples tt = m_index->match
310 (Triple(Node(), Uri("a"), m_index->expand("vamp:Plugin")));
311 Nodes plugins = tt.subjects();
312
313 bool foundSomething = false;
314 bool addedSomething = false;
315
316 foreach (Node plugin, plugins) {
317
318 if (plugin.type != Node::URI) {
319 SVDEBUG << "PluginRDFIndexer::reindex: Plugin has no URI: node is "
320 << plugin << endl;
321 continue;
322 }
323
324 Node idn = m_index->complete
325 (Triple(plugin, m_index->expand("vamp:identifier"), Node()));
326
327 if (idn.type != Node::Literal) {
328 SVDEBUG << "PluginRDFIndexer::reindex: Plugin " << plugin
329 << " lacks vamp:identifier literal" << endl;
330 continue;
331 }
332
333 Node libn = m_index->complete
334 (Triple(Node(), m_index->expand("vamp:available_plugin"), plugin));
335
336 if (libn.type != Node::URI) {
337 SVDEBUG << "PluginRDFIndexer::reindex: Plugin " << plugin
338 << " is not vamp:available_plugin in any library" << endl;
339 continue;
340 }
341
342 Node son = m_index->complete
343 (Triple(libn, m_index->expand("vamp:identifier"), Node()));
344
345 if (son.type != Node::Literal) {
346 SVDEBUG << "PluginRDFIndexer::reindex: Library " << libn
347 << " lacks vamp:identifier for soname" << endl;
348 continue;
349 }
350
351 QString pluginUri = plugin.value;
352 QString identifier = idn.value;
353 QString soname = son.value;
354
355 QString pluginId = PluginIdentifier::createIdentifier
356 ("vamp", soname, identifier);
357
358 foundSomething = true;
359
360 if (m_idToUriMap.find(pluginId) != m_idToUriMap.end()) {
361 continue;
362 }
363
364 m_idToUriMap[pluginId] = pluginUri;
365
366 addedSomething = true;
367
368 if (pluginUri != "") {
369 if (m_uriToIdMap.find(pluginUri) != m_uriToIdMap.end()) {
370 SVDEBUG << "PluginRDFIndexer::reindex: WARNING: Found multiple plugins with the same URI:" << endl;
371 SVDEBUG << " 1. Plugin id \"" << m_uriToIdMap[pluginUri] << "\"" << endl;
372 SVDEBUG << " 2. Plugin id \"" << pluginId << "\"" << endl;
373 SVDEBUG << "both claim URI <" << pluginUri << ">" << endl;
374 } else {
375 m_uriToIdMap[pluginUri] = pluginId;
376 }
377 }
378 }
379
380 if (!foundSomething) {
381 SVDEBUG << "PluginRDFIndexer::reindex: NOTE: Plugins found, but none sufficiently described" << endl;
382 }
383
384 return addedSomething;
385 }
386