1 /******************************************************************************
2  *
3  * Copyright (C) 1997-2020 by Dimitri van Heesch.
4  *
5  * Permission to use, copy, modify, and distribute this software and its
6  * documentation under the terms of the GNU General Public License is hereby
7  * granted. No representations are made about the suitability of this software
8  * for any purpose. It is provided "as is" without express or implied warranty.
9  * See the GNU General Public License for more details.
10  *
11  * Documents produced by Doxygen are derivative works derived from the
12  * input used in their production; they are not affected by this license.
13  *
14  */
15 
16 
17 #include <ctype.h>
18 #include <assert.h>
19 #include <sstream>
20 
21 #include "searchindex.h"
22 #include "config.h"
23 #include "util.h"
24 #include "doxygen.h"
25 #include "language.h"
26 #include "pagedef.h"
27 #include "growbuf.h"
28 #include "message.h"
29 #include "version.h"
30 #include "groupdef.h"
31 #include "filedef.h"
32 #include "memberdef.h"
33 #include "filename.h"
34 #include "membername.h"
35 #include "resourcemgr.h"
36 #include "namespacedef.h"
37 #include "classdef.h"
38 #include "utf8.h"
39 #include "classlist.h"
40 
41 //---------------------------------------------------------------------------------------------
42 // the following part is for the server based search engine
43 //---------------------------------------------------------------------------------------------
44 
45 // file format: (all multi-byte values are stored in big endian format)
46 //   4 byte header
47 //   256*256*4 byte index (4 bytes)
48 //   for each index entry: a zero terminated list of words
49 //   for each word: a \0 terminated string + 4 byte offset to the stats info
50 //   padding bytes to align at 4 byte boundary
51 //   for each word: the number of urls (4 bytes)
52 //               + for each url containing the word 8 bytes statistics
53 //                 (4 bytes index to url string + 4 bytes frequency counter)
54 //   for each url: a \0 terminated string
55 
56 const size_t numIndexEntries = 256*256;
57 
58 //--------------------------------------------------------------------
59 
IndexWord(QCString word)60 IndexWord::IndexWord(QCString word) : m_word(word)
61 {
62   //printf("IndexWord::IndexWord(%s)\n",word);
63 }
64 
addUrlIndex(int idx,bool hiPriority)65 void IndexWord::addUrlIndex(int idx,bool hiPriority)
66 {
67   //printf("IndexWord::addUrlIndex(%d,%d)\n",idx,hiPriority);
68   auto it = m_urls.find(idx);
69   if (it==m_urls.end())
70   {
71     //printf("URLInfo::URLInfo(%d)\n",idx);
72     it = m_urls.insert(std::make_pair(idx,URLInfo(idx,0))).first;
73   }
74   it->second.freq+=2;
75   if (hiPriority) it->second.freq|=1; // mark as high priority document
76 }
77 
78 //--------------------------------------------------------------------
79 
SearchIndex()80 SearchIndex::SearchIndex() : SearchIndexIntf(Internal)
81 {
82   m_index.resize(numIndexEntries);
83 }
84 
setCurrentDoc(const Definition * ctx,const QCString & anchor,bool isSourceFile)85 void SearchIndex::setCurrentDoc(const Definition *ctx,const QCString &anchor,bool isSourceFile)
86 {
87   if (ctx==0) return;
88   assert(!isSourceFile || ctx->definitionType()==Definition::TypeFile);
89   //printf("SearchIndex::setCurrentDoc(%s,%s,%s)\n",name,baseName,anchor);
90   QCString url=isSourceFile ? (toFileDef(ctx))->getSourceFileBase() : ctx->getOutputFileBase();
91   url+=Config_getString(HTML_FILE_EXTENSION);
92   QCString baseUrl = url;
93   if (!anchor.isEmpty()) url+=QCString("#")+anchor;
94   if (!isSourceFile) baseUrl=url;
95   QCString name=ctx->qualifiedName();
96   if (ctx->definitionType()==Definition::TypeMember)
97   {
98     const MemberDef *md = toMemberDef(ctx);
99     name.prepend((md->getLanguage()==SrcLangExt_Fortran  ?
100                  theTranslator->trSubprogram(TRUE,TRUE) :
101                  theTranslator->trMember(TRUE,TRUE))+" ");
102   }
103   else // compound type
104   {
105     SrcLangExt lang = ctx->getLanguage();
106     QCString sep = getLanguageSpecificSeparator(lang);
107     if (sep!="::")
108     {
109       name = substitute(name,"::",sep);
110     }
111     switch (ctx->definitionType())
112     {
113       case Definition::TypePage:
114         {
115           const PageDef *pd = toPageDef(ctx);
116           if (pd->hasTitle())
117           {
118             name = theTranslator->trPage(TRUE,TRUE)+" "+pd->title();
119           }
120           else
121           {
122             name = theTranslator->trPage(TRUE,TRUE)+" "+pd->name();
123           }
124         }
125         break;
126       case Definition::TypeClass:
127         {
128           const ClassDef *cd = toClassDef(ctx);
129           name.prepend(cd->compoundTypeString()+" ");
130         }
131         break;
132       case Definition::TypeNamespace:
133         {
134           if (lang==SrcLangExt_Java || lang==SrcLangExt_CSharp)
135           {
136             name = theTranslator->trPackage(name);
137           }
138           else if (lang==SrcLangExt_Fortran)
139           {
140             name.prepend(theTranslator->trModule(TRUE,TRUE)+" ");
141           }
142           else
143           {
144             name.prepend(theTranslator->trNamespace(TRUE,TRUE)+" ");
145           }
146         }
147         break;
148       case Definition::TypeGroup:
149         {
150           const GroupDef *gd = toGroupDef(ctx);
151           if (!gd->groupTitle().isEmpty())
152           {
153             name = theTranslator->trGroup(TRUE,TRUE)+" "+gd->groupTitle();
154           }
155           else
156           {
157             name.prepend(theTranslator->trGroup(TRUE,TRUE)+" ");
158           }
159         }
160         break;
161       default:
162         break;
163     }
164   }
165 
166   auto it = m_url2IdMap.find(baseUrl.str());
167   if (it == m_url2IdMap.end())
168   {
169     ++m_urlIndex;
170     m_url2IdMap.insert(std::make_pair(baseUrl.str(),m_urlIndex));
171     m_urls.insert(std::make_pair(m_urlIndex,URL(name,url)));
172   }
173   else
174   {
175     m_urls.insert(std::make_pair(it->second,URL(name,url)));
176   }
177 }
178 
charsToIndex(const QCString & word)179 static int charsToIndex(const QCString &word)
180 {
181   if (word.length()<2) return -1;
182 
183   // Fast string hashing algorithm
184   //register ushort h=0;
185   //const char *k = word;
186   //ushort mask=0xfc00;
187   //while ( *k )
188   //{
189   //  h = (h&mask)^(h<<6)^(*k++);
190   //}
191   //return h;
192 
193   // Simple hashing that allows for substring searching
194   uint c1=(uchar)word[0];
195   uint c2=(uchar)word[1];
196   return c1*256+c2;
197 }
198 
addWord(const QCString & word,bool hiPriority,bool recurse)199 void SearchIndex::addWord(const QCString &word,bool hiPriority,bool recurse)
200 {
201   if (word.isEmpty()) return;
202   QCString wStr = QCString(word).lower();
203   //printf("SearchIndex::addWord(%s,%d) wStr=%s\n",word,hiPriority,qPrint(wStr));
204   int idx=charsToIndex(wStr);
205   if (idx<0 || idx>=static_cast<int>(m_index.size())) return;
206   auto it = m_words.find(wStr.str());
207   if (it==m_words.end())
208   {
209     //fprintf(stderr,"addWord(%s) at index %d\n",word,idx);
210     m_index[idx].push_back(IndexWord(wStr));
211     it = m_words.insert({ wStr.str(), static_cast<int>(m_index[idx].size())-1 }).first;
212   }
213   m_index[idx][it->second].addUrlIndex(m_urlIndex,hiPriority);
214   int i;
215   bool found=FALSE;
216   if (!recurse) // the first time we check if we can strip the prefix
217   {
218     i=getPrefixIndex(word);
219     if (i>0)
220     {
221       addWord(word.data()+i,hiPriority,TRUE);
222       found=TRUE;
223     }
224   }
225   if (!found) // no prefix stripped
226   {
227     i=0;
228     while (word[i]!=0 &&
229            !((word[i]=='_' || word[i]==':' || (word[i]>='a' && word[i]<='z')) &&  // [_a-z:]
230              (word[i+1]>='A' && word[i+1]<='Z')))                                 // [A-Z]
231     {
232       i++;
233     }
234     if (word[i]!=0 && i>=1)
235     {
236       addWord(word.data()+i+1,hiPriority,TRUE);
237     }
238   }
239 }
240 
addWord(const QCString & word,bool hiPriority)241 void SearchIndex::addWord(const QCString &word,bool hiPriority)
242 {
243   addWord(word,hiPriority,FALSE);
244 }
245 
writeInt(std::ostream & f,size_t index)246 static void writeInt(std::ostream &f,size_t index)
247 {
248   f.put(static_cast<int>(index>>24));
249   f.put(static_cast<int>((index>>16)&0xff));
250   f.put(static_cast<int>((index>>8)&0xff));
251   f.put(static_cast<int>(index&0xff));
252 }
253 
writeString(std::ostream & f,const QCString & s)254 static void writeString(std::ostream &f,const QCString &s)
255 {
256   uint l = s.length();
257   for (uint i=0;i<l;i++) f.put(s[i]);
258   f.put(0);
259 }
260 
write(const QCString & fileName)261 void SearchIndex::write(const QCString &fileName)
262 {
263   size_t i;
264   size_t size=4; // for the header
265   size+=4*numIndexEntries; // for the index
266   size_t wordsOffset = size;
267   // first pass: compute the size of the wordlist
268   for (i=0;i<numIndexEntries;i++)
269   {
270     const auto &wlist = m_index[i];
271     if (!wlist.empty())
272     {
273       for (const auto &iw : wlist)
274       {
275         int ws = iw.word().length()+1;
276         size+=ws+4; // word + url info list offset
277       }
278       size+=1; // zero list terminator
279     }
280   }
281 
282   // second pass: compute the offsets in the index
283   size_t indexOffsets[numIndexEntries];
284   size_t offset=wordsOffset;
285   for (i=0;i<numIndexEntries;i++)
286   {
287     const auto &wlist = m_index[i];
288     if (!wlist.empty())
289     {
290       indexOffsets[i]=offset;
291       for (const auto &iw : wlist)
292       {
293         offset+= iw.word().length()+1;
294         offset+=4; // word + offset to url info array
295       }
296       offset+=1; // zero list terminator
297     }
298     else
299     {
300       indexOffsets[i]=0;
301     }
302   }
303   size_t padding = size;
304   size = (size+3)&~3; // round up to 4 byte boundary
305   padding = size - padding;
306 
307   std::vector<size_t> wordStatOffsets(m_words.size());
308 
309   int count=0;
310 
311   // third pass: compute offset to stats info for each word
312   for (i=0;i<numIndexEntries;i++)
313   {
314     const auto &wlist = m_index[i];
315     if (!wlist.empty())
316     {
317       for (const auto &iw : wlist)
318       {
319         //printf("wordStatOffsets[%d]=%d\n",count,size);
320         wordStatOffsets[count++] = size;
321         size+=4 + iw.urls().size() * 8; // count + (url_index,freq) per url
322       }
323     }
324   }
325   std::vector<size_t> urlOffsets(m_urls.size());
326   for (const auto &udi : m_urls)
327   {
328     urlOffsets[udi.first]=size;
329     size+=udi.second.name.length()+1+
330           udi.second.url.length()+1;
331   }
332 
333   //printf("Total size %x bytes (word=%x stats=%x urls=%x)\n",size,wordsOffset,statsOffset,urlsOffset);
334   std::ofstream f(fileName.str(),std::ofstream::out | std::ofstream::binary);
335   if (f.is_open())
336   {
337     // write header
338     f.put('D'); f.put('O'); f.put('X'); f.put('S');
339     // write index
340     for (i=0;i<numIndexEntries;i++)
341     {
342       writeInt(f,indexOffsets[i]);
343     }
344     // write word lists
345     count=0;
346     for (i=0;i<numIndexEntries;i++)
347     {
348       const auto &wlist = m_index[i];
349       if (!wlist.empty())
350       {
351         for (const auto &iw : wlist)
352         {
353           writeString(f,iw.word());
354           writeInt(f,wordStatOffsets[count++]);
355         }
356         f.put(0);
357       }
358     }
359     // write extra padding bytes
360     for (i=0;i<padding;i++) f.put(0);
361     // write word statistics
362     for (i=0;i<numIndexEntries;i++)
363     {
364       const auto &wlist = m_index[i];
365       if (!wlist.empty())
366       {
367         for (const auto &iw : wlist)
368         {
369           size_t numUrls = iw.urls().size();
370           writeInt(f,numUrls);
371           for (const auto &ui : iw.urls())
372           {
373             writeInt(f,urlOffsets[ui.second.urlIdx]);
374             writeInt(f,ui.second.freq);
375           }
376         }
377       }
378     }
379     // write urls
380     for (const auto &udi : m_urls)
381     {
382       writeString(f,udi.second.name);
383       writeString(f,udi.second.url);
384     }
385   }
386 
387 }
388 
389 
390 //---------------------------------------------------------------------------
391 // the following part is for writing an external search index
392 
393 struct SearchDocEntry
394 {
395   QCString type;
396   QCString name;
397   QCString args;
398   QCString extId;
399   QCString url;
400   GrowBuf  importantText;
401   GrowBuf  normalText;
402 };
403 
404 struct SearchIndexExternal::Private
405 {
406   std::map<std::string,SearchDocEntry> docEntries;
407   SearchDocEntry *current = 0;
408 };
409 
SearchIndexExternal()410 SearchIndexExternal::SearchIndexExternal() : SearchIndexIntf(External), p(std::make_unique<Private>())
411 {
412 }
413 
definitionToName(const Definition * ctx)414 static QCString definitionToName(const Definition *ctx)
415 {
416   if (ctx && ctx->definitionType()==Definition::TypeMember)
417   {
418     const MemberDef *md = toMemberDef(ctx);
419     if (md->isFunction())
420       return "function";
421     else if (md->isSlot())
422       return "slot";
423     else if (md->isSignal())
424       return "signal";
425     else if (md->isVariable())
426       return "variable";
427     else if (md->isTypedef())
428       return "typedef";
429     else if (md->isEnumerate())
430       return "enum";
431     else if (md->isEnumValue())
432       return "enumvalue";
433     else if (md->isProperty())
434       return "property";
435     else if (md->isEvent())
436       return "event";
437     else if (md->isRelated() || md->isForeign())
438       return "related";
439     else if (md->isFriend())
440       return "friend";
441     else if (md->isDefine())
442       return "define";
443   }
444   else if (ctx)
445   {
446     switch(ctx->definitionType())
447     {
448       case Definition::TypeClass:
449         return (toClassDef(ctx))->compoundTypeString();
450       case Definition::TypeFile:
451         return "file";
452       case Definition::TypeNamespace:
453         return "namespace";
454       case Definition::TypeConcept:
455         return "concept";
456       case Definition::TypeGroup:
457         return "group";
458       case Definition::TypePackage:
459         return "package";
460       case Definition::TypePage:
461         return "page";
462       case Definition::TypeDir:
463         return "dir";
464       default:
465         break;
466     }
467   }
468   return "unknown";
469 }
470 
setCurrentDoc(const Definition * ctx,const QCString & anchor,bool isSourceFile)471 void SearchIndexExternal::setCurrentDoc(const Definition *ctx,const QCString &anchor,bool isSourceFile)
472 {
473   static QCString extId = stripPath(Config_getString(EXTERNAL_SEARCH_ID));
474   QCString baseName = isSourceFile ? (toFileDef(ctx))->getSourceFileBase() : ctx->getOutputFileBase();
475   QCString url = addHtmlExtensionIfMissing(baseName);
476   if (!anchor.isEmpty()) url+=QCString("#")+anchor;
477   QCString key = extId+";"+url;
478 
479   auto it = p->docEntries.find(key.str());
480   if (it == p->docEntries.end())
481   {
482     SearchDocEntry e;
483     e.type = isSourceFile ? QCString("source") : definitionToName(ctx);
484     e.name = ctx->qualifiedName();
485     if (ctx->definitionType()==Definition::TypeMember)
486     {
487       e.args = (toMemberDef(ctx))->argsString();
488     }
489     e.extId = extId;
490     e.url  = url;
491     it = p->docEntries.insert({key.str(),e}).first;
492     //printf("searchIndexExt %s : %s\n",qPrint(e->name),qPrint(e->url));
493   }
494   p->current = &it->second;
495 }
496 
addWord(const QCString & word,bool hiPriority)497 void SearchIndexExternal::addWord(const QCString &word,bool hiPriority)
498 {
499   if (word.isEmpty() || !isId(word[0]) || p->current==0) return;
500   GrowBuf *pText = hiPriority ? &p->current->importantText : &p->current->normalText;
501   if (pText->getPos()>0) pText->addChar(' ');
502   pText->addStr(word);
503   //printf("addWord %s\n",word);
504 }
505 
write(const QCString & fileName)506 void SearchIndexExternal::write(const QCString &fileName)
507 {
508   std::ofstream t(fileName.str(),std::ofstream::out | std::ofstream::binary);
509   if (t.is_open())
510   {
511     t << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
512     t << "<add>\n";
513     for (auto &kv : p->docEntries)
514     {
515       SearchDocEntry &doc = kv.second;
516       doc.normalText.addChar(0);    // make sure buffer ends with a 0 terminator
517       doc.importantText.addChar(0); // make sure buffer ends with a 0 terminator
518       t << "  <doc>\n";
519       t << "    <field name=\"type\">"     << doc.type << "</field>\n";
520       t << "    <field name=\"name\">"     << convertToXML(doc.name) << "</field>\n";
521       if (!doc.args.isEmpty())
522       {
523         t << "    <field name=\"args\">"     << convertToXML(doc.args) << "</field>\n";
524       }
525       if (!doc.extId.isEmpty())
526       {
527         t << "    <field name=\"tag\">"      << convertToXML(doc.extId)  << "</field>\n";
528       }
529       t << "    <field name=\"url\">"      << convertToXML(doc.url)  << "</field>\n";
530       t << "    <field name=\"keywords\">" << convertToXML(doc.importantText.get())  << "</field>\n";
531       t << "    <field name=\"text\">"     << convertToXML(doc.normalText.get())     << "</field>\n";
532       t << "  </doc>\n";
533     }
534     t << "</add>\n";
535   }
536   else
537   {
538     err("Failed to open file %s for writing!\n",qPrint(fileName));
539   }
540 }
541 
542 //---------------------------------------------------------------------------------------------
543 // the following part is for the javascript based search engine
544 //---------------------------------------------------------------------------------------------
545 
searchName(const Definition * d)546 QCString searchName(const Definition *d)
547 {
548   return d->definitionType()==Definition::TypeGroup ?  QCString(toGroupDef(d)->groupTitle()) :
549          d->definitionType()==Definition::TypePage  ?  toPageDef(d)->title() :
550                                                        d->localName();
551 }
552 
searchId(const Definition * d)553 QCString searchId(const Definition *d)
554 {
555   std::string s = searchName(d).str();
556   TextStream t;
557   for (size_t i=0;i<s.length();i++)
558   {
559     if (isIdJS(s[i]))
560     {
561       t << s[i];
562     }
563     else // escape non-identifier characters
564     {
565       static const char *hex = "0123456789ABCDEF";
566       unsigned char uc = static_cast<unsigned char>(s[i]);
567       t << '_';
568       t << hex[uc>>4];
569       t << hex[uc&0xF];
570     }
571   }
572 
573   return convertUTF8ToLower(t.str());
574 }
575 
576 
577 #define SEARCH_INDEX_ALL           0
578 #define SEARCH_INDEX_CLASSES       1
579 #define SEARCH_INDEX_INTERFACES    2
580 #define SEARCH_INDEX_STRUCTS       3
581 #define SEARCH_INDEX_EXCEPTIONS    4
582 #define SEARCH_INDEX_NAMESPACES    5
583 #define SEARCH_INDEX_FILES         6
584 #define SEARCH_INDEX_FUNCTIONS     7
585 #define SEARCH_INDEX_VARIABLES     8
586 #define SEARCH_INDEX_TYPEDEFS      9
587 #define SEARCH_INDEX_SEQUENCES    10
588 #define SEARCH_INDEX_DICTIONARIES 11
589 #define SEARCH_INDEX_ENUMS        12
590 #define SEARCH_INDEX_ENUMVALUES   13
591 #define SEARCH_INDEX_PROPERTIES   14
592 #define SEARCH_INDEX_EVENTS       15
593 #define SEARCH_INDEX_RELATED      16
594 #define SEARCH_INDEX_DEFINES      17
595 #define SEARCH_INDEX_GROUPS       18
596 #define SEARCH_INDEX_PAGES        19
597 #define SEARCH_INDEX_CONCEPTS     20
598 
599 static std::array<SearchIndexInfo,NUM_SEARCH_INDICES> g_searchIndexInfo =
600 { {
601   //  index                         name            getText                                                  symbolList
__anon664f801f0102() 602   { /* SEARCH_INDEX_ALL */          "all"         , []() { return theTranslator->trAll();                 }, {} },
__anon664f801f0302() 603   { /* SEARCH_INDEX_CLASSES */      "classes"     , []() { return theTranslator->trClasses();             }, {} },
__anon664f801f0502() 604   { /* SEARCH_INDEX_INTERFACES */   "interfaces"  , []() { return theTranslator->trSliceInterfaces();     }, {} },
__anon664f801f0702() 605   { /* SEARCH_INDEX_STRUCTS */      "structs"     , []() { return theTranslator->trStructs();             }, {} },
__anon664f801f0902() 606   { /* SEARCH_INDEX_EXCEPTIONS */   "exceptions"  , []() { return theTranslator->trExceptions();          }, {} },
__anon664f801f0b02() 607   { /* SEARCH_INDEX_NAMESPACES */   "namespaces"  , []() { return Config_getBool(OPTIMIZE_OUTPUT_SLICE) ?
608                                                                   theTranslator->trModules() :
609                                                                   theTranslator->trNamespace(TRUE,FALSE); }, {} },
__anon664f801f0d02() 610   { /* SEARCH_INDEX_FILES */        "files"       , []() { return theTranslator->trFile(TRUE,FALSE);      }, {} },
__anon664f801f0f02() 611   { /* SEARCH_INDEX_FUNCTIONS */    "functions"   , []() { return Config_getBool(OPTIMIZE_OUTPUT_SLICE) ?
612                                                                   theTranslator->trOperations() :
613                                                                   theTranslator->trFunctions();           }, {} },
__anon664f801f1102() 614   { /* SEARCH_INDEX_VARIABLES */    "variables"   , []() { return Config_getBool(OPTIMIZE_OUTPUT_SLICE) ?
615                                                                   theTranslator->trConstants() :
616                                                                   theTranslator->trVariables();           }, {} },
__anon664f801f1402() 617   { /* SEARCH_INDEX_TYPEDEFS */     "typedefs"    , []() { return theTranslator->trTypedefs();            }, {} },
__anon664f801f1502() 618   { /* SEARCH_INDEX_SEQUENCES */    "sequences"   , []() { return theTranslator->trSequences();           }, {} },
__anon664f801f1802() 619   { /* SEARCH_INDEX_DICTIONARIES */ "dictionaries", []() { return theTranslator->trDictionaries();        }, {} },
__anon664f801f1902() 620   { /* SEARCH_INDEX_ENUMS */        "enums"       , []() { return theTranslator->trEnumerations();        }, {} },
__anon664f801f1c02() 621   { /* SEARCH_INDEX_ENUMVALUES */   "enumvalues"  , []() { return theTranslator->trEnumerationValues();   }, {} },
__anon664f801f1e02() 622   { /* SEARCH_INDEX_PROPERTIES */   "properties"  , []() { return theTranslator->trProperties();          }, {} },
__anon664f801f1f02() 623   { /* SEARCH_INDEX_EVENTS */       "events"      , []() { return theTranslator->trEvents();              }, {} },
__anon664f801f2202() 624   { /* SEARCH_INDEX_RELATED */      "related"     , []() { return theTranslator->trFriends();             }, {} },
__anon664f801f2402() 625   { /* SEARCH_INDEX_DEFINES */      "defines"     , []() { return theTranslator->trDefines();             }, {} },
__anon664f801f2602() 626   { /* SEARCH_INDEX_GROUPS */       "groups"      , []() { return theTranslator->trGroup(TRUE,FALSE);     }, {} },
__anon664f801f2802() 627   { /* SEARCH_INDEX_PAGES */        "pages"       , []() { return theTranslator->trPage(TRUE,FALSE);      }, {} },
__anon664f801f2902() 628   { /* SEARCH_INDEX_CONCEPTS */     "concepts"    , []() { return theTranslator->trConcept(true,false);   }, {} }
629 } };
630 
addMemberToSearchIndex(const MemberDef * md)631 static void addMemberToSearchIndex(const MemberDef *md)
632 {
633   static bool hideFriendCompounds = Config_getBool(HIDE_FRIEND_COMPOUNDS);
634   bool isLinkable = md->isLinkable();
635   const ClassDef *cd=0;
636   const NamespaceDef *nd=0;
637   const FileDef *fd=0;
638   const GroupDef *gd=0;
639   if (isLinkable &&
640       (
641        ((cd=md->getClassDef()) && cd->isLinkable() && cd->templateMaster()==0) ||
642        ((gd=md->getGroupDef()) && gd->isLinkable())
643       )
644      )
645   {
646     std::string n = md->name().str();
647     if (!n.empty())
648     {
649       std::string letter = convertUTF8ToLower(getUTF8CharAt(n,0));
650       bool isFriendToHide = hideFriendCompounds &&
651         (QCString(md->typeString())=="friend class" ||
652          QCString(md->typeString())=="friend struct" ||
653          QCString(md->typeString())=="friend union");
654       if (!(md->isFriend() && isFriendToHide))
655       {
656         g_searchIndexInfo[SEARCH_INDEX_ALL].add(letter,md);
657       }
658       if (md->isFunction() || md->isSlot() || md->isSignal())
659       {
660         g_searchIndexInfo[SEARCH_INDEX_FUNCTIONS].add(letter,md);
661       }
662       else if (md->isVariable())
663       {
664         g_searchIndexInfo[SEARCH_INDEX_VARIABLES].add(letter,md);
665       }
666       else if (md->isSequence())
667       {
668         g_searchIndexInfo[SEARCH_INDEX_SEQUENCES].add(letter,md);
669       }
670       else if (md->isDictionary())
671       {
672         g_searchIndexInfo[SEARCH_INDEX_DICTIONARIES].add(letter,md);
673       }
674       else if (md->isTypedef())
675       {
676         g_searchIndexInfo[SEARCH_INDEX_TYPEDEFS].add(letter,md);
677       }
678       else if (md->isEnumerate())
679       {
680         g_searchIndexInfo[SEARCH_INDEX_ENUMS].add(letter,md);
681       }
682       else if (md->isEnumValue())
683       {
684         g_searchIndexInfo[SEARCH_INDEX_ENUMVALUES].add(letter,md);
685       }
686       else if (md->isProperty())
687       {
688         g_searchIndexInfo[SEARCH_INDEX_PROPERTIES].add(letter,md);
689       }
690       else if (md->isEvent())
691       {
692         g_searchIndexInfo[SEARCH_INDEX_EVENTS].add(letter,md);
693       }
694       else if (md->isRelated() || md->isForeign() ||
695                (md->isFriend() && !isFriendToHide))
696       {
697         g_searchIndexInfo[SEARCH_INDEX_RELATED].add(letter,md);
698       }
699     }
700   }
701   else if (isLinkable &&
702       (((nd=md->getNamespaceDef()) && nd->isLinkable()) ||
703        ((fd=md->getFileDef())      && fd->isLinkable())
704       )
705      )
706   {
707     std::string n = md->name().str();
708     if (!n.empty())
709     {
710       std::string letter = convertUTF8ToLower(getUTF8CharAt(n,0));
711       g_searchIndexInfo[SEARCH_INDEX_ALL].add(letter,md);
712 
713       if (md->isFunction())
714       {
715         g_searchIndexInfo[SEARCH_INDEX_FUNCTIONS].add(letter,md);
716       }
717       else if (md->isVariable())
718       {
719         g_searchIndexInfo[SEARCH_INDEX_VARIABLES].add(letter,md);
720       }
721       else if (md->isSequence())
722       {
723         g_searchIndexInfo[SEARCH_INDEX_SEQUENCES].add(letter,md);
724       }
725       else if (md->isDictionary())
726       {
727         g_searchIndexInfo[SEARCH_INDEX_DICTIONARIES].add(letter,md);
728       }
729       else if (md->isTypedef())
730       {
731         g_searchIndexInfo[SEARCH_INDEX_TYPEDEFS].add(letter,md);
732       }
733       else if (md->isEnumerate())
734       {
735         g_searchIndexInfo[SEARCH_INDEX_ENUMS].add(letter,md);
736       }
737       else if (md->isEnumValue())
738       {
739         g_searchIndexInfo[SEARCH_INDEX_ENUMVALUES].add(letter,md);
740       }
741       else if (md->isDefine())
742       {
743         g_searchIndexInfo[SEARCH_INDEX_DEFINES].add(letter,md);
744       }
745     }
746   }
747 }
748 
749 //---------------------------------------------------------------------------------------------
750 
createJavaScriptSearchIndex()751 void createJavaScriptSearchIndex()
752 {
753   // index classes
754   for (const auto &cd : *Doxygen::classLinkedMap)
755   {
756     std::string letter = convertUTF8ToLower(getUTF8CharAt(cd->localName().str(),0));
757     if (cd->isLinkable())
758     {
759       g_searchIndexInfo[SEARCH_INDEX_ALL].add(letter,cd.get());
760       if (Config_getBool(OPTIMIZE_OUTPUT_SLICE))
761       {
762         if (cd->compoundType()==ClassDef::Interface)
763         {
764           g_searchIndexInfo[SEARCH_INDEX_INTERFACES].add(letter,cd.get());
765         }
766         else if (cd->compoundType()==ClassDef::Struct)
767         {
768           g_searchIndexInfo[SEARCH_INDEX_STRUCTS].add(letter,cd.get());
769         }
770         else if (cd->compoundType()==ClassDef::Exception)
771         {
772           g_searchIndexInfo[SEARCH_INDEX_EXCEPTIONS].add(letter,cd.get());
773         }
774         else // cd->compoundType()==ClassDef::Class
775         {
776           g_searchIndexInfo[SEARCH_INDEX_CLASSES].add(letter,cd.get());
777         }
778       }
779       else // non slice optimisation: group all types under classes
780       {
781         g_searchIndexInfo[SEARCH_INDEX_CLASSES].add(letter,cd.get());
782       }
783     }
784   }
785 
786   // index namespaces
787   for (const auto &nd : *Doxygen::namespaceLinkedMap)
788   {
789     std::string letter = convertUTF8ToLower(getUTF8CharAt(nd->name().str(),0));
790     if (nd->isLinkable())
791     {
792       g_searchIndexInfo[SEARCH_INDEX_ALL].add(letter,nd.get());
793       g_searchIndexInfo[SEARCH_INDEX_NAMESPACES].add(letter,nd.get());
794     }
795   }
796 
797   // index concepts
798   for (const auto &cd : *Doxygen::conceptLinkedMap)
799   {
800     std::string letter = convertUTF8ToLower(getUTF8CharAt(cd->name().str(),0));
801     if (cd->isLinkable())
802     {
803       g_searchIndexInfo[SEARCH_INDEX_ALL].add(letter,cd.get());
804       g_searchIndexInfo[SEARCH_INDEX_CONCEPTS].add(letter,cd.get());
805     }
806   }
807 
808   // index files
809   for (const auto &fn : *Doxygen::inputNameLinkedMap)
810   {
811     for (const auto &fd : *fn)
812     {
813       std::string letter = convertUTF8ToLower(getUTF8CharAt(fd->name().str(),0));
814       if (fd->isLinkable())
815       {
816         g_searchIndexInfo[SEARCH_INDEX_ALL].add(letter,fd.get());
817         g_searchIndexInfo[SEARCH_INDEX_FILES].add(letter,fd.get());
818       }
819     }
820   }
821 
822   // index class members
823   {
824     // for each member name
825     for (const auto &mn : *Doxygen::memberNameLinkedMap)
826     {
827       // for each member definition
828       for (const auto &md : *mn)
829       {
830         addMemberToSearchIndex(md.get());
831       }
832     }
833   }
834 
835   // index file/namespace members
836   {
837     // for each member name
838     for (const auto &mn : *Doxygen::functionNameLinkedMap)
839     {
840       // for each member definition
841       for (const auto &md : *mn)
842       {
843         addMemberToSearchIndex(md.get());
844       }
845     }
846   }
847 
848   // index groups
849   for (const auto &gd : *Doxygen::groupLinkedMap)
850   {
851     if (gd->isLinkable())
852     {
853       std::string title = gd->groupTitle().str();
854       if (!title.empty()) // TODO: able searching for all word in the title
855       {
856         std::string letter = convertUTF8ToLower(getUTF8CharAt(title,0));
857         g_searchIndexInfo[SEARCH_INDEX_ALL].add(letter,gd.get());
858         g_searchIndexInfo[SEARCH_INDEX_GROUPS].add(letter,gd.get());
859       }
860     }
861   }
862 
863   // index pages
864   for (const auto &pd : *Doxygen::pageLinkedMap)
865   {
866     if (pd->isLinkable())
867     {
868       std::string title = pd->title().str();
869       if (!title.empty())
870       {
871         std::string letter = convertUTF8ToLower(getUTF8CharAt(title,0));
872         g_searchIndexInfo[SEARCH_INDEX_ALL].add(letter,pd.get());
873         g_searchIndexInfo[SEARCH_INDEX_PAGES].add(letter,pd.get());
874       }
875     }
876   }
877   if (Doxygen::mainPage)
878   {
879     std::string title = Doxygen::mainPage->title().str();
880     if (!title.empty())
881     {
882       std::string letter = convertUTF8ToLower(getUTF8CharAt(title,0));
883       g_searchIndexInfo[SEARCH_INDEX_ALL].add(letter,Doxygen::mainPage.get());
884       g_searchIndexInfo[SEARCH_INDEX_PAGES].add(letter,Doxygen::mainPage.get());
885     }
886   }
887 
888   // sort all lists
889   for (auto &sii : g_searchIndexInfo) // for each index
890   {
891     for (auto &kv : sii.symbolMap) // for each symbol in the index
892     {
893       // sort the symbols (first on "search" name, and then on full name)
894       std::sort(kv.second.begin(),
895                 kv.second.end(),
896                 [](const Definition *d1,const Definition *d2)
897                 {
898                   int eq = qstricmp(searchName(d1),searchName(d2));         // search name first
899                   return eq==0 ? qstricmp(d1->name(),d2->name())<0 : eq<0;  // then full name
900                 });
901     }
902   }
903 }
904 
writeJavaScriptSearchIndex()905 void writeJavaScriptSearchIndex()
906 {
907   // write index files
908   QCString searchDirName = Config_getString(HTML_OUTPUT)+"/search";
909 
910   for (auto &sii : g_searchIndexInfo)
911   {
912     int p=0;
913     for (const auto &kv : sii.symbolMap)
914     {
915       int cnt = 0;
916       QCString baseName;
917       baseName.sprintf("%s_%x",sii.name.data(),p);
918 
919       QCString fileName = searchDirName + "/"+baseName+Doxygen::htmlFileExtension;
920       QCString dataFileName = searchDirName + "/"+baseName+".js";
921 
922       std::ofstream t(fileName.str(), std::ofstream::out | std::ofstream::binary);
923       std::ofstream ti(dataFileName.str(), std::ofstream::out | std::ofstream::binary);
924       if (t.is_open() && ti.is_open())
925       {
926         {
927           t << "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\""
928             " \"https://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">\n";
929           t << "<html xmlns=\"http://www.w3.org/1999/xhtml\">\n";
930           t << "<head><title></title>\n";
931           t << "<meta http-equiv=\"Content-Type\" content=\"text/xhtml;charset=UTF-8\"/>\n";
932           t << "<meta name=\"generator\" content=\"Doxygen " << getDoxygenVersion() << "\"/>\n";
933           t << "<link rel=\"stylesheet\" type=\"text/css\" href=\"search.css\"/>\n";
934           t << "<script type=\"text/javascript\" src=\"" << baseName << ".js\"></script>\n";
935           t << "<script type=\"text/javascript\" src=\"search.js\"></script>\n";
936           t << "</head>\n";
937           t << "<body class=\"SRPage\">\n";
938           t << "<div id=\"SRIndex\">\n";
939           t << "<div class=\"SRStatus\" id=\"Loading\">" << theTranslator->trLoading() << "</div>\n";
940           t << "<div id=\"SRResults\"></div>\n"; // here the results will be inserted
941           t << "<script type=\"text/javascript\">\n";
942           t << "/* @license magnet:?xt=urn:btih:d3d9a9a6595521f9666a5e94cc830dab83b65699&amp;dn=expat.txt MIT */\n";
943           t << "createResults();\n"; // this function will insert the results
944           t << "/* @license-end */\n";
945           t << "</script>\n";
946           t << "<div class=\"SRStatus\" id=\"Searching\">"
947             << theTranslator->trSearching() << "</div>\n";
948           t << "<div class=\"SRStatus\" id=\"NoMatches\">"
949             << theTranslator->trNoMatches() << "</div>\n";
950 
951           t << "<script type=\"text/javascript\">\n";
952           t << "/* @license magnet:?xt=urn:btih:d3d9a9a6595521f9666a5e94cc830dab83b65699&amp;dn=expat.txt MIT */\n";
953           t << "document.getElementById(\"Loading\").style.display=\"none\";\n";
954           t << "document.getElementById(\"NoMatches\").style.display=\"none\";\n";
955           t << "var searchResults = new SearchResults(\"searchResults\");\n";
956           t << "searchResults.Search();\n";
957           t << "window.addEventListener(\"message\", function(event) {\n";
958           t << "  if (event.data == \"take_focus\") {\n";
959           t << "    var elem = searchResults.NavNext(0);\n";
960           t << "    if (elem) elem.focus();\n";
961           t << "  }\n";
962           t << "});\n";
963           t << "/* @license-end */\n";
964           t << "</script>\n";
965           t << "</div>\n"; // SRIndex
966           t << "</body>\n";
967           t << "</html>\n";
968         }
969 
970         ti << "var searchData=\n";
971         // format
972         // searchData[] = array of items
973         // searchData[x][0] = id
974         // searchData[x][1] = [ name + child1 + child2 + .. ]
975         // searchData[x][1][0] = name as shown
976         // searchData[x][1][y+1] = info for child y
977         // searchData[x][1][y+1][0] = url
978         // searchData[x][1][y+1][1] = 1 => target="_parent"
979         // searchData[x][1][y+1][2] = scope
980 
981         ti << "[\n";
982         bool firstEntry=TRUE;
983 
984         int childCount=0;
985         QCString lastName;
986         const Definition *prevScope = 0;
987         for (auto it = kv.second.begin(); it!=kv.second.end();)
988         {
989           const Definition *d = *it;
990           QCString sname = searchName(d);
991           QCString id    = searchId(d);
992 
993           if (sname!=lastName) // this item has a different search word
994           {
995             if (!firstEntry)
996             {
997               ti << "]]]";
998               ti << ",\n";
999             }
1000             firstEntry=FALSE;
1001 
1002             ti << "  ['" << id << "_" << cnt++ << "',['" << convertToXML(sname) << "',[";
1003             childCount=0;
1004             prevScope=0;
1005           }
1006 
1007           ++it;
1008           const Definition *scope     = d->getOuterScope();
1009           const Definition *next      = it!=kv.second.end() ? *it : 0;
1010           const Definition *nextScope = 0;
1011           const MemberDef  *md        = toMemberDef(d);
1012           if (next) nextScope = next->getOuterScope();
1013           QCString anchor = d->anchor();
1014 
1015           if (childCount>0)
1016           {
1017             ti << "],[";
1018           }
1019           ti << "'" << externalRef("../",d->getReference(),TRUE)
1020             << addHtmlExtensionIfMissing(d->getOutputFileBase());
1021           if (!anchor.isEmpty())
1022           {
1023             ti << "#" << anchor;
1024           }
1025           ti << "',";
1026 
1027           static bool extLinksInWindow = Config_getBool(EXT_LINKS_IN_WINDOW);
1028           if (!extLinksInWindow || d->getReference().isEmpty())
1029           {
1030             ti << "1,";
1031           }
1032           else
1033           {
1034             ti << "0,";
1035           }
1036 
1037           if (lastName!=sname && (next==0 || searchName(next)!=sname)) // unique name
1038           {
1039             if (d->getOuterScope()!=Doxygen::globalScope)
1040             {
1041               ti << "'" << convertToXML(d->getOuterScope()->name()) << "'";
1042             }
1043             else if (md)
1044             {
1045               const FileDef *fd = md->getBodyDef();
1046               if (fd==0) fd = md->getFileDef();
1047               if (fd)
1048               {
1049                 ti << "'" << convertToXML(fd->localName()) << "'";
1050               }
1051             }
1052             else
1053             {
1054               ti << "''";
1055             }
1056           }
1057           else // multiple entries with the same name
1058           {
1059             bool found=FALSE;
1060             bool overloadedFunction = ((prevScope!=0 && scope==prevScope) ||
1061                 (scope && scope==nextScope)) && md && (md->isFunction() || md->isSlot());
1062             QCString prefix;
1063             if (md) prefix=convertToXML(md->localName());
1064             if (overloadedFunction) // overloaded member function
1065             {
1066               prefix+=convertToXML(md->argsString());
1067               // show argument list to disambiguate overloaded functions
1068             }
1069             else if (md) // unique member function
1070             {
1071               prefix+="()"; // only to show it is a function
1072             }
1073             QCString name;
1074             if (d->definitionType()==Definition::TypeClass)
1075             {
1076               name = convertToXML((toClassDef(d))->displayName());
1077               found = TRUE;
1078             }
1079             else if (d->definitionType()==Definition::TypeNamespace)
1080             {
1081               name = convertToXML((toNamespaceDef(d))->displayName());
1082               found = TRUE;
1083             }
1084             else if (scope==0 || scope==Doxygen::globalScope) // in global scope
1085             {
1086               if (md)
1087               {
1088                 const FileDef *fd = md->getBodyDef();
1089                 if (fd==0) fd = md->resolveAlias()->getFileDef();
1090                 if (fd)
1091                 {
1092                   if (!prefix.isEmpty()) prefix+=":&#160;";
1093                   name = prefix + convertToXML(fd->localName());
1094                   found = TRUE;
1095                 }
1096               }
1097             }
1098             else if (md && (md->resolveAlias()->getClassDef() || md->resolveAlias()->getNamespaceDef()))
1099               // member in class or namespace scope
1100             {
1101               SrcLangExt lang = md->getLanguage();
1102               name = convertToXML(d->getOuterScope()->qualifiedName())
1103                 + getLanguageSpecificSeparator(lang) + prefix;
1104               found = TRUE;
1105             }
1106             else if (scope) // some thing else? -> show scope
1107             {
1108               name = prefix + convertToXML(scope->name());
1109               found = TRUE;
1110             }
1111             if (!found) // fallback
1112             {
1113               name = prefix + "("+theTranslator->trGlobalNamespace()+")";
1114             }
1115 
1116             ti << "'" << name << "'";
1117 
1118             prevScope = scope;
1119             childCount++;
1120           }
1121           lastName = sname;
1122         }
1123         if (!firstEntry)
1124         {
1125           ti << "]]]\n";
1126         }
1127         ti << "];\n";
1128       }
1129       else
1130       {
1131         err("Failed to open file '%s' for writing...\n",qPrint(fileName));
1132       }
1133       p++;
1134     }
1135   }
1136 
1137   {
1138     std::ofstream t(searchDirName.str()+"/searchdata.js",
1139                     std::ofstream::out | std::ofstream::binary);
1140     if (t.is_open())
1141     {
1142       t << "var indexSectionsWithContent =\n";
1143       t << "{\n";
1144       int j=0;
1145       for (const auto &sii : g_searchIndexInfo)
1146       {
1147         if (!sii.symbolMap.empty())
1148         {
1149           if (j>0) t << ",\n";
1150           t << "  " << j << ": \"";
1151 
1152           for (const auto &kv : sii.symbolMap)
1153           {
1154             if ( kv.first == "\"" ) t << "\\";
1155             t << kv.first;
1156           }
1157           t << "\"";
1158           j++;
1159         }
1160       }
1161       if (j>0) t << "\n";
1162       t << "};\n\n";
1163       t << "var indexSectionNames =\n";
1164       t << "{\n";
1165       j=0;
1166       for (const auto &sii : g_searchIndexInfo)
1167       {
1168         if (!sii.symbolMap.empty())
1169         {
1170           if (j>0) t << ",\n";
1171           t << "  " << j << ": \"" << sii.name << "\"";
1172           j++;
1173         }
1174       }
1175       if (j>0) t << "\n";
1176       t << "};\n\n";
1177       t << "var indexSectionLabels =\n";
1178       t << "{\n";
1179       j=0;
1180       for (const auto &sii : g_searchIndexInfo)
1181       {
1182         if (!sii.symbolMap.empty())
1183         {
1184           if (j>0) t << ",\n";
1185           t << "  " << j << ": \"" << convertToXML(sii.getText()) << "\"";
1186           j++;
1187         }
1188       }
1189       if (j>0) t << "\n";
1190       t << "};\n\n";
1191     }
1192     ResourceMgr::instance().copyResource("search.js",searchDirName);
1193   }
1194 
1195   {
1196     QCString noMatchesFileName =searchDirName+"/nomatches"+Doxygen::htmlFileExtension;
1197     std::ofstream t(noMatchesFileName.str(), std::ofstream::out | std::ofstream::binary);
1198     if (t.is_open())
1199     {
1200       t << "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" "
1201            "\"https://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">\n";
1202       t << "<html xmlns=\"http://www.w3.org/1999/xhtml\">\n";
1203       t << "<head><title></title>\n";
1204       t << "<meta http-equiv=\"Content-Type\" content=\"text/xhtml;charset=UTF-8\"/>\n";
1205       t << "<link rel=\"stylesheet\" type=\"text/css\" href=\"search.css\"/>\n";
1206       t << "<script type=\"text/javascript\" src=\"search.js\"></script>\n";
1207       t << "</head>\n";
1208       t << "<body class=\"SRPage\">\n";
1209       t << "<div id=\"SRIndex\">\n";
1210       t << "<div class=\"SRStatus\" id=\"NoMatches\">"
1211         << theTranslator->trNoMatches() << "</div>\n";
1212       t << "</div>\n";
1213       t << "</body>\n";
1214       t << "</html>\n";
1215     }
1216   }
1217 
1218   Doxygen::indexList->addStyleSheetFile("search/search.js");
1219 }
1220 
add(const std::string & letter,const Definition * def)1221 void SearchIndexInfo::add(const std::string &letter,const Definition *def)
1222 {
1223   //printf("%p: %s->%s (full=%s)\n",this,qPrint(letter),qPrint(searchName(def)),qPrint(def->name()));
1224   auto it = symbolMap.find(letter);
1225   if (it!=symbolMap.end())
1226   {
1227     it->second.push_back(def);
1228   }
1229   else
1230   {
1231     symbolMap.insert(std::make_pair(letter,std::vector<const Definition*>({def})));
1232   }
1233 }
1234 
getSearchIndices()1235 const std::array<SearchIndexInfo,NUM_SEARCH_INDICES> &getSearchIndices()
1236 {
1237   return g_searchIndexInfo;
1238 }
1239 
1240 //---------------------------------------------------------------------------------------------
1241 
initSearchIndexer()1242 void initSearchIndexer()
1243 {
1244   static bool searchEngine      = Config_getBool(SEARCHENGINE);
1245   static bool serverBasedSearch = Config_getBool(SERVER_BASED_SEARCH);
1246   static bool externalSearch    = Config_getBool(EXTERNAL_SEARCH);
1247   if (searchEngine && serverBasedSearch)
1248   {
1249     if (externalSearch) // external tools produce search index and engine
1250     {
1251       Doxygen::searchIndex = new SearchIndexExternal;
1252     }
1253     else // doxygen produces search index and engine
1254     {
1255       Doxygen::searchIndex = new SearchIndex;
1256     }
1257   }
1258   else // no search engine or pure javascript based search function
1259   {
1260     Doxygen::searchIndex = 0;
1261   }
1262 }
1263 
finalizeSearchIndexer()1264 void finalizeSearchIndexer()
1265 {
1266   delete Doxygen::searchIndex;
1267 }
1268