1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
2 
3 /*
4     Sonic Visualiser
5     An audio file viewer and annotation editor.
6     Centre for Digital Music, Queen Mary, University of London.
7     This file copyright 2008-2012 QMUL.
8 
9     This program is free software; you can redistribute it and/or
10     modify it under the terms of the GNU General Public License as
11     published by the Free Software Foundation; either version 2 of the
12     License, or (at your option) any later version.  See the file
13     COPYING included with this distribution for more information.
14 */
15 
16 #include "PluginRDFIndexer.h"
17 
18 #include "data/fileio/CachedFile.h"
19 #include "data/fileio/FileSource.h"
20 #include "data/fileio/PlaylistFileReader.h"
21 #include "plugin/PluginIdentifier.h"
22 
23 #include "base/Profiler.h"
24 #include "base/Debug.h"
25 
26 #include <vamp-hostsdk/PluginHostAdapter.h>
27 
28 #include <dataquay/BasicStore.h>
29 #include <dataquay/RDFException.h>
30 
31 #include <QFileInfo>
32 #include <QDir>
33 #include <QUrl>
34 #include <QDateTime>
35 #include <QSettings>
36 #include <QFile>
37 
38 #include <iostream>
39 
40 using std::vector;
41 using std::string;
42 using Vamp::PluginHostAdapter;
43 
44 using Dataquay::Uri;
45 using Dataquay::Node;
46 using Dataquay::Nodes;
47 using Dataquay::Triple;
48 using Dataquay::Triples;
49 using Dataquay::BasicStore;
50 using Dataquay::RDFException;
51 using Dataquay::RDFDuplicateImportException;
52 
53 PluginRDFIndexer *
54 PluginRDFIndexer::m_instance = nullptr;
55 
56 PluginRDFIndexer *
getInstance()57 PluginRDFIndexer::getInstance()
58 {
59     if (!m_instance) m_instance = new PluginRDFIndexer();
60     return m_instance;
61 }
62 
PluginRDFIndexer()63 PluginRDFIndexer::PluginRDFIndexer() :
64     m_index(new Dataquay::BasicStore)
65 {
66     m_index->addPrefix("vamp", Uri("http://purl.org/ontology/vamp/"));
67     m_index->addPrefix("foaf", Uri("http://xmlns.com/foaf/0.1/"));
68     m_index->addPrefix("dc", Uri("http://purl.org/dc/elements/1.1/"));
69     indexInstalledURLs();
70 }
71 
72 const BasicStore *
getIndex()73 PluginRDFIndexer::getIndex()
74 {
75     return m_index;
76 }
77 
~PluginRDFIndexer()78 PluginRDFIndexer::~PluginRDFIndexer()
79 {
80     QMutexLocker locker(&m_mutex);
81 }
82 
83 void
indexInstalledURLs()84 PluginRDFIndexer::indexInstalledURLs()
85 {
86     vector<string> paths = PluginHostAdapter::getPluginPath();
87 
88 //    SVDEBUG << "\nPluginRDFIndexer::indexInstalledURLs: pid is " << getpid() << endl;
89 
90     QStringList filters;
91     filters << "*.ttl";
92     filters << "*.TTL";
93     filters << "*.n3";
94     filters << "*.N3";
95     filters << "*.rdf";
96     filters << "*.RDF";
97 
98     // Search each Vamp plugin path for an RDF file that either has
99     // name "soname", "soname:label" or "soname/label" plus RDF
100     // extension.  Use that order of preference, and prefer ttl over
101     // n3 over rdf extension.
102 
103     for (vector<string>::const_iterator i = paths.begin(); i != paths.end(); ++i) {
104 
105         QDir dir(i->c_str());
106         if (!dir.exists()) continue;
107 
108         QStringList entries = dir.entryList
109             (filters, QDir::Files | QDir::Readable);
110 
111         for (QStringList::const_iterator j = entries.begin();
112              j != entries.end(); ++j) {
113 
114             QFileInfo fi(dir.filePath(*j));
115             pullFile(fi.absoluteFilePath());
116         }
117 
118         QStringList subdirs = dir.entryList
119             (QDir::AllDirs | QDir::NoDotAndDotDot | QDir::Readable);
120 
121         for (QStringList::const_iterator j = subdirs.begin();
122              j != subdirs.end(); ++j) {
123 
124             QDir subdir(dir.filePath(*j));
125             if (subdir.exists()) {
126                 entries = subdir.entryList
127                     (filters, QDir::Files | QDir::Readable);
128                 for (QStringList::const_iterator k = entries.begin();
129                      k != entries.end(); ++k) {
130                     QFileInfo fi(subdir.filePath(*k));
131                     pullFile(fi.absoluteFilePath());
132                 }
133             }
134         }
135     }
136 
137     reindex();
138 }
139 
140 bool
indexConfiguredURLs()141 PluginRDFIndexer::indexConfiguredURLs()
142 {
143     SVDEBUG << "PluginRDFIndexer::indexConfiguredURLs" << endl;
144 
145     QSettings settings;
146     settings.beginGroup("RDF");
147 
148     QString indexKey("rdf-indices");
149     QStringList indices = settings.value(indexKey).toStringList();
150 
151     for (int i = 0; i < indices.size(); ++i) {
152 
153         QString index = indices[i];
154 
155         SVDEBUG << "PluginRDFIndexer::indexConfiguredURLs: index url is "
156                   << index << endl;
157 
158         CachedFile cf(index);
159         if (!cf.isOK()) continue;
160 
161         FileSource indexSource(cf.getLocalFilename());
162 
163         PlaylistFileReader reader(indexSource);
164         if (!reader.isOK()) continue;
165 
166         PlaylistFileReader::Playlist list = reader.load();
167         for (PlaylistFileReader::Playlist::const_iterator j = list.begin();
168              j != list.end(); ++j) {
169             SVDEBUG << "PluginRDFIndexer::indexConfiguredURLs: url is "
170                   << *j << endl;
171             pullURL(*j);
172         }
173     }
174 
175     QString urlListKey("rdf-urls");
176     QStringList urls = settings.value(urlListKey).toStringList();
177 
178     for (int i = 0; i < urls.size(); ++i) {
179         pullURL(urls[i]);
180     }
181 
182     settings.endGroup();
183     reindex();
184     return true;
185 }
186 
187 QString
getURIForPluginId(QString pluginId)188 PluginRDFIndexer::getURIForPluginId(QString pluginId)
189 {
190     QMutexLocker locker(&m_mutex);
191 
192     if (m_idToUriMap.find(pluginId) == m_idToUriMap.end()) return "";
193     return m_idToUriMap[pluginId];
194 }
195 
196 QString
getIdForPluginURI(QString uri)197 PluginRDFIndexer::getIdForPluginURI(QString uri)
198 {
199     m_mutex.lock();
200 
201     if (m_uriToIdMap.find(uri) == m_uriToIdMap.end()) {
202 
203         m_mutex.unlock();
204 
205         // Haven't found this uri referenced in any document on the
206         // local filesystem; try resolving the pre-fragment part of
207         // the uri as a document URL and reading that if possible.
208 
209         // Because we may want to refer to this document again, we
210         // cache it locally if it turns out to exist.
211 
212         SVDEBUG << "PluginRDFIndexer::getIdForPluginURI: NOTE: Failed to find a local RDF document describing plugin <" << uri << ">: attempting to retrieve one remotely by guesswork" << endl;
213 
214         QString baseUrl = QUrl(uri).toString(QUrl::RemoveFragment);
215 
216         indexURL(baseUrl);
217 
218         m_mutex.lock();
219 
220         if (m_uriToIdMap.find(uri) == m_uriToIdMap.end()) {
221             m_uriToIdMap[uri] = "";
222         }
223     }
224 
225     QString id = m_uriToIdMap[uri];
226     m_mutex.unlock();
227     return id;
228 }
229 
230 QStringList
getIndexedPluginIds()231 PluginRDFIndexer::getIndexedPluginIds()
232 {
233     QMutexLocker locker(&m_mutex);
234 
235     QStringList ids;
236     for (StringMap::const_iterator i = m_idToUriMap.begin();
237          i != m_idToUriMap.end(); ++i) {
238         ids.push_back(i->first);
239     }
240     return ids;
241 }
242 
243 bool
pullFile(QString filepath)244 PluginRDFIndexer::pullFile(QString filepath)
245 {
246     QUrl url = QUrl::fromLocalFile(filepath);
247     QString urlString = url.toString();
248     return pullURL(urlString);
249 }
250 
251 bool
indexURL(QString urlString)252 PluginRDFIndexer::indexURL(QString urlString)
253 {
254     bool pulled = pullURL(urlString);
255     if (!pulled) return false;
256     reindex();
257     return true;
258 }
259 
260 bool
pullURL(QString urlString)261 PluginRDFIndexer::pullURL(QString urlString)
262 {
263     Profiler profiler("PluginRDFIndexer::indexURL");
264 
265 //    SVDEBUG << "PluginRDFIndexer::indexURL(" << urlString << ")" << endl;
266 
267     QMutexLocker locker(&m_mutex);
268 
269     QUrl local = urlString;
270 
271     if (FileSource::isRemote(urlString) &&
272         FileSource::canHandleScheme(urlString)) {
273 
274         CachedFile cf(urlString, nullptr, "application/rdf+xml");
275         if (!cf.isOK()) {
276             return false;
277         }
278 
279         local = QUrl::fromLocalFile(cf.getLocalFilename());
280 
281     } else if (urlString.startsWith("file:")) {
282 
283         local = QUrl(urlString);
284 
285     } else {
286 
287         local = QUrl::fromLocalFile(urlString);
288     }
289 
290     try {
291         m_index->import(local, BasicStore::ImportFailOnDuplicates);
292     } catch (RDFDuplicateImportException &e) {
293         SVDEBUG << e.what() << endl;
294         SVDEBUG << "PluginRDFIndexer::pullURL: Document at " << urlString
295                  << " duplicates triples found in earlier loaded document -- skipping it" << endl;
296         return false;
297     } catch (RDFException &e) {
298         SVDEBUG << e.what() << endl;
299         SVDEBUG << "PluginRDFIndexer::pullURL: Failed to import document from "
300                  << urlString << ": " << e.what() << endl;
301         return false;
302     }
303     return true;
304 }
305 
306 bool
reindex()307 PluginRDFIndexer::reindex()
308 {
309     Triples tt = m_index->match
310         (Triple(Node(), Uri("a"), m_index->expand("vamp:Plugin")));
311     Nodes plugins = tt.subjects();
312 
313     bool foundSomething = false;
314     bool addedSomething = false;
315 
316     foreach (Node plugin, plugins) {
317 
318         if (plugin.type != Node::URI) {
319             SVDEBUG << "PluginRDFIndexer::reindex: Plugin has no URI: node is "
320                  << plugin << endl;
321             continue;
322         }
323 
324         Node idn = m_index->complete
325             (Triple(plugin, m_index->expand("vamp:identifier"), Node()));
326 
327         if (idn.type != Node::Literal) {
328             SVDEBUG << "PluginRDFIndexer::reindex: Plugin " << plugin
329                  << " lacks vamp:identifier literal" << endl;
330             continue;
331         }
332 
333         Node libn = m_index->complete
334             (Triple(Node(), m_index->expand("vamp:available_plugin"), plugin));
335 
336         if (libn.type != Node::URI) {
337             SVDEBUG << "PluginRDFIndexer::reindex: Plugin " << plugin
338                  << " is not vamp:available_plugin in any library" << endl;
339             continue;
340         }
341 
342         Node son = m_index->complete
343             (Triple(libn, m_index->expand("vamp:identifier"), Node()));
344 
345         if (son.type != Node::Literal) {
346             SVDEBUG << "PluginRDFIndexer::reindex: Library " << libn
347                  << " lacks vamp:identifier for soname" << endl;
348             continue;
349         }
350 
351         QString pluginUri = plugin.value;
352         QString identifier = idn.value;
353         QString soname = son.value;
354 
355         QString pluginId = PluginIdentifier::createIdentifier
356             ("vamp", soname, identifier);
357 
358         foundSomething = true;
359 
360         if (m_idToUriMap.find(pluginId) != m_idToUriMap.end()) {
361             continue;
362         }
363 
364         m_idToUriMap[pluginId] = pluginUri;
365 
366         addedSomething = true;
367 
368         if (pluginUri != "") {
369             if (m_uriToIdMap.find(pluginUri) != m_uriToIdMap.end()) {
370                 SVDEBUG << "PluginRDFIndexer::reindex: WARNING: Found multiple plugins with the same URI:" << endl;
371                 SVDEBUG << "  1. Plugin id \"" << m_uriToIdMap[pluginUri] << "\"" << endl;
372                 SVDEBUG << "  2. Plugin id \"" << pluginId << "\"" << endl;
373                 SVDEBUG << "both claim URI <" << pluginUri << ">" << endl;
374             } else {
375                 m_uriToIdMap[pluginUri] = pluginId;
376             }
377         }
378     }
379 
380     if (!foundSomething) {
381         SVDEBUG << "PluginRDFIndexer::reindex: NOTE: Plugins found, but none sufficiently described" << endl;
382     }
383 
384     return addedSomething;
385 }
386