1 /* This file is (c) 2008-2012 Konstantin Isakov <ikm@goldendict.org>
2 * Part of GoldenDict. Licensed under GPLv3 or later, see the LICENSE file */
3
4 #include "dictdfiles.hh"
5 #include "btreeidx.hh"
6 #include "folding.hh"
7 #include "utf8.hh"
8 #include "dictzip.h"
9 #include "htmlescape.hh"
10 #include "fsencoding.hh"
11 #include "langcoder.hh"
12 #include <map>
13 #include <set>
14 #include <string>
15 #include <vector>
16 #include <list>
17 #include <wctype.h>
18 #include <stdlib.h>
19 #include "gddebug.hh"
20 #include "ftshelpers.hh"
21 #include <QUrl>
22
23 #include <QDebug>
24
25 #if QT_VERSION >= QT_VERSION_CHECK( 5, 0, 0 )
26 #include <QRegularExpression>
27 #endif
28
29 #ifdef _MSC_VER
30 #include <stub_msvc.h>
31 #endif
32
33 namespace DictdFiles {
34
35 using std::map;
36 using std::multimap;
37 using std::pair;
38 using std::set;
39 using std::string;
40 using gd::wstring;
41 using std::vector;
42 using std::list;
43
44 using BtreeIndexing::WordArticleLink;
45 using BtreeIndexing::IndexedWords;
46 using BtreeIndexing::IndexInfo;
47
48 namespace {
49
50 DEF_EX_STR( exCantReadFile, "Can't read file", Dictionary::Ex )
51 DEF_EX( exFailedToReadLineFromIndex, "Failed to read line from index file", Dictionary::Ex )
52 DEF_EX( exMalformedIndexFileLine, "Malformed index file line encountered", Dictionary::Ex )
53 DEF_EX( exInvalidBase64, "Invalid base64 sequence encountered", Dictionary::Ex )
54 DEF_EX_STR( exDictzipError, "DICTZIP error", Dictionary::Ex )
55
56 enum
57 {
58 Signature = 0x58444344, // DCDX on little-endian, XDCD on big-endian
59 CurrentFormatVersion = 5 + BtreeIndexing::FormatVersion + Folding::Version
60 };
61
62 struct IdxHeader
63 {
64 uint32_t signature; // First comes the signature, DCDX
65 uint32_t formatVersion; // File format version (CurrentFormatVersion)
66 uint32_t wordCount; // Total number of words
67 uint32_t articleCount; // Total number of articles
68 uint32_t indexBtreeMaxElements; // Two fields from IndexInfo
69 uint32_t indexRootOffset;
70 uint32_t langFrom; // Source language
71 uint32_t langTo; // Target language
72 }
73 #ifndef _MSC_VER
74 __attribute__((packed))
75 #endif
76 ;
77
indexIsOldOrBad(string const & indexFile)78 bool indexIsOldOrBad( string const & indexFile )
79 {
80 File::Class idx( indexFile, "rb" );
81
82 IdxHeader header;
83
84 return idx.readRecords( &header, sizeof( header ), 1 ) != 1 ||
85 header.signature != Signature ||
86 header.formatVersion != CurrentFormatVersion;
87 }
88
89 class DictdDictionary: public BtreeIndexing::BtreeDictionary
90 {
91 Mutex idxMutex;
92 File::Class idx, indexFile; // The later is .index file
93 IdxHeader idxHeader;
94 dictData * dz;
95 string dictionaryName;
96 Mutex indexFileMutex, dzMutex;
97
98 public:
99
100 DictdDictionary( string const & id, string const & indexFile,
101 vector< string > const & dictionaryFiles );
102
103 ~DictdDictionary();
104
getName()105 virtual string getName() throw()
106 { return dictionaryName; }
107
getProperties()108 virtual map< Dictionary::Property, string > getProperties() throw()
109 { return map< Dictionary::Property, string >(); }
110
getArticleCount()111 virtual unsigned long getArticleCount() throw()
112 { return idxHeader.articleCount; }
113
getWordCount()114 virtual unsigned long getWordCount() throw()
115 { return idxHeader.wordCount; }
116
117 virtual void loadIcon() throw();
118
getLangFrom() const119 inline virtual quint32 getLangFrom() const
120 { return idxHeader.langFrom; }
121
getLangTo() const122 inline virtual quint32 getLangTo() const
123 { return idxHeader.langTo; }
124
125 virtual sptr< Dictionary::DataRequest > getArticle( wstring const &,
126 vector< wstring > const & alts,
127 wstring const &,
128 bool ignoreDiacritics )
129 THROW_SPEC( std::exception );
130
131 virtual QString const& getDescription();
132
133 virtual sptr< Dictionary::DataRequest > getSearchResults( QString const & searchString,
134 int searchMode, bool matchCase,
135 int distanceBetweenWords,
136 int maxResults,
137 bool ignoreWordsOrder,
138 bool ignoreDiacritics );
139 void getArticleText( uint32_t articleAddress, QString & headword, QString & text );
140
141 virtual void makeFTSIndex(QAtomicInt & isCancelled, bool firstIteration );
142
setFTSParameters(Config::FullTextSearch const & fts)143 virtual void setFTSParameters( Config::FullTextSearch const & fts )
144 {
145 can_FTS = fts.enabled
146 && !fts.disabledTypes.contains( "DICTD", Qt::CaseInsensitive )
147 && ( fts.maxDictionarySize == 0 || getArticleCount() <= fts.maxDictionarySize );
148 }
149 };
150
DictdDictionary(string const & id,string const & indexFile,vector<string> const & dictionaryFiles)151 DictdDictionary::DictdDictionary( string const & id,
152 string const & indexFile,
153 vector< string > const & dictionaryFiles ):
154 BtreeDictionary( id, dictionaryFiles ),
155 idx( indexFile, "rb" ),
156 indexFile( dictionaryFiles[ 0 ], "rb" ),
157 idxHeader( idx.read< IdxHeader >() )
158 {
159
160 // Read the dictionary name
161 idx.seek( sizeof( idxHeader ) );
162
163 vector< char > dName( idx.read< uint32_t >() );
164 if( dName.size() > 0 )
165 {
166 idx.read( &dName.front(), dName.size() );
167 dictionaryName = string( &dName.front(), dName.size() );
168 }
169
170 // Open the .dict file
171
172 DZ_ERRORS error;
173 dz = dict_data_open( dictionaryFiles[ 1 ].c_str(), &error, 0 );
174
175 if ( !dz )
176 throw exDictzipError( string( dz_error_str( error ) )
177 + "(" + getDictionaryFilenames()[ 1 ] + ")" );
178
179 // Initialize the index
180
181 openIndex( IndexInfo( idxHeader.indexBtreeMaxElements,
182 idxHeader.indexRootOffset ),
183 idx, idxMutex );
184
185 // Full-text search parameters
186
187 can_FTS = true;
188
189 ftsIdxName = indexFile + "_FTS";
190
191 if( !Dictionary::needToRebuildIndex( dictionaryFiles, ftsIdxName )
192 && !FtsHelpers::ftsIndexIsOldOrBad( ftsIdxName, this ) )
193 FTS_index_completed.ref();
194 }
195
~DictdDictionary()196 DictdDictionary::~DictdDictionary()
197 {
198 if ( dz )
199 dict_data_close( dz );
200 }
201
nameFromFileName(string const & indexFileName)202 string nameFromFileName( string const & indexFileName )
203 {
204 if ( indexFileName.empty() )
205 return string();
206
207 char const * sep = strrchr( indexFileName.c_str(), FsEncoding::separator() );
208
209 if ( !sep )
210 sep = indexFileName.c_str();
211
212 char const * dot = strrchr( sep, '.' );
213
214 if ( !dot )
215 dot = indexFileName.c_str() + indexFileName.size();
216
217 return Utf8::encode( FsEncoding::decode( string( sep + 1, dot - sep - 1 ) ) );
218 }
219
loadIcon()220 void DictdDictionary::loadIcon() throw()
221 {
222 if ( dictionaryIconLoaded )
223 return;
224
225 QString fileName =
226 QDir::fromNativeSeparators( FsEncoding::decode( getDictionaryFilenames()[ 0 ].c_str() ) );
227
228 // Remove the extension
229 fileName.chop( 5 );
230
231 if( !loadIconFromFile( fileName ) )
232 {
233 // Load failed -- use default icons
234 dictionaryNativeIcon = dictionaryIcon = QIcon(":/icons/icon32_dictd.png");
235 }
236
237 dictionaryIconLoaded = true;
238 }
239
decodeBase64(string const & str)240 uint32_t decodeBase64( string const & str )
241 {
242 static char const digits[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
243
244 uint32_t number = 0;
245
246 for( char const * next = str.c_str(); *next; ++next )
247 {
248 char const * d = strchr( digits, *next );
249
250 if ( !d )
251 throw exInvalidBase64();
252
253 number = number * 64 + ( d - digits );
254 }
255
256 return number;
257 }
258
getArticle(wstring const & word,vector<wstring> const & alts,wstring const &,bool ignoreDiacritics)259 sptr< Dictionary::DataRequest > DictdDictionary::getArticle( wstring const & word,
260 vector< wstring > const & alts,
261 wstring const &,
262 bool ignoreDiacritics )
263 THROW_SPEC( std::exception )
264 {
265 try
266 {
267 vector< WordArticleLink > chain = findArticles( word, ignoreDiacritics );
268
269 for( unsigned x = 0; x < alts.size(); ++x )
270 {
271 /// Make an additional query for each alt
272
273 vector< WordArticleLink > altChain = findArticles( alts[ x ], ignoreDiacritics );
274
275 chain.insert( chain.end(), altChain.begin(), altChain.end() );
276 }
277
278 multimap< wstring, string > mainArticles, alternateArticles;
279
280 set< uint32_t > articlesIncluded; // Some synonyms make it that the articles
281 // appear several times. We combat this
282 // by only allowing them to appear once.
283
284 wstring wordCaseFolded = Folding::applySimpleCaseOnly( word );
285 if( ignoreDiacritics )
286 wordCaseFolded = Folding::applyDiacriticsOnly( wordCaseFolded );
287
288 char buf[ 16384 ];
289
290 for( unsigned x = 0; x < chain.size(); ++x )
291 {
292 if ( articlesIncluded.find( chain[ x ].articleOffset ) != articlesIncluded.end() )
293 continue; // We already have this article in the body.
294
295 // Now load that article
296
297 {
298 Mutex::Lock _( indexFileMutex );
299 indexFile.seek( chain[ x ].articleOffset );
300
301 if ( !indexFile.gets( buf, sizeof( buf ), true ) )
302 throw exFailedToReadLineFromIndex();
303 }
304
305 char * tab1 = strchr( buf, '\t' );
306
307 if ( !tab1 )
308 throw exMalformedIndexFileLine();
309
310 char * tab2 = strchr( tab1 + 1, '\t' );
311
312 if ( !tab2 )
313 throw exMalformedIndexFileLine();
314
315 // After tab1 should be article offset, after tab2 -- article size
316
317 uint32_t articleOffset = decodeBase64( string( tab1 + 1, tab2 - tab1 - 1 ) );
318
319 char * tab3 = strchr( tab2 + 1, '\t');
320
321 uint32_t articleSize;
322 if ( tab3 )
323 {
324 articleSize = decodeBase64( string( tab2 + 1, tab3 - tab2 - 1 ) );
325 }
326 else
327 {
328 articleSize = decodeBase64( tab2 + 1 );
329 }
330
331 string articleText;
332
333 char * articleBody;
334 {
335 Mutex::Lock _( dzMutex );
336 articleBody = dict_data_read_( dz, articleOffset, articleSize, 0, 0 );
337 }
338
339 if ( !articleBody )
340 {
341 articleText = string( "<div class=\"dictd_article\">DICTZIP error: " )
342 + dict_error_str( dz ) + "</div>";
343 }
344 else
345 {
346 #if QT_VERSION >= QT_VERSION_CHECK( 5, 0, 0 )
347 static QRegularExpression phonetic( "\\\\([^\\\\]+)\\\\",
348 QRegularExpression::CaseInsensitiveOption ); // phonetics: \stuff\ ...
349 static QRegularExpression refs( "\\{([^\\{\\}]+)\\}",
350 QRegularExpression::CaseInsensitiveOption ); // links: {stuff}
351 static QRegularExpression links( "<a href=\"gdlookup://localhost/([^\"]*)\">",
352 QRegularExpression::CaseInsensitiveOption );
353 static QRegularExpression tags( "<[^>]*>",
354 QRegularExpression::CaseInsensitiveOption );
355 #else
356 static QRegExp phonetic( "\\\\([^\\\\]+)\\\\", Qt::CaseInsensitive ); // phonetics: \stuff\ ...
357 static QRegExp refs( "\\{([^\\{\\}]+)\\}", Qt::CaseInsensitive ); // links: {stuff}
358 static QRegExp links( "<a href=\"gdlookup://localhost/([^\"]*)\">", Qt::CaseInsensitive );
359 static QRegExp tags( "<[^>]*>", Qt::CaseInsensitive );
360 #endif
361
362 articleText = string( "<div class=\"dictd_article\"" );
363 if( isToLanguageRTL() )
364 articleText += " dir=\"rtl\"";
365 articleText += ">";
366
367 string convertedText = Html::preformat( articleBody, isToLanguageRTL() );
368 free( articleBody );
369
370 QString articleString = QString::fromUtf8( convertedText.c_str() )
371 .replace(phonetic, "<span class=\"dictd_phonetic\">\\1</span>")
372 .replace(refs, "<a href=\"gdlookup://localhost/\\1\">\\1</a>");
373 convertedText.erase();
374
375 int pos = 0;
376 #if QT_VERSION >= QT_VERSION_CHECK( 5, 0, 0 )
377 QString articleNewString;
378 QRegularExpressionMatchIterator it = links.globalMatch( articleString );
379 while( it.hasNext() )
380 {
381 QRegularExpressionMatch match = it.next();
382 articleNewString += articleString.midRef( pos, match.capturedStart() - pos );
383 pos = match.capturedEnd();
384
385 QString link = match.captured( 1 );
386 link.replace( tags, " " );
387 link.replace( " ", " " );
388
389 QString newLink = match.captured();
390 newLink.replace( 30, match.capturedLength( 1 ),
391 QString::fromUtf8( QUrl::toPercentEncoding( link.simplified() ) ) );
392 articleNewString += newLink;
393 }
394 if( pos )
395 {
396 articleNewString += articleString.midRef( pos );
397 articleString = articleNewString;
398 articleNewString.clear();
399 }
400 #else
401 for( ; ; )
402 {
403 pos = articleString.indexOf( links, pos );
404 if( pos < 0 )
405 break;
406
407 QString link = links.cap( 1 );
408 link.replace( tags, " " );
409 link.replace( " ", " " );
410 articleString.replace( pos + 30, links.cap( 1 ).length(),
411 QString::fromUtf8( QUrl::toPercentEncoding( link.simplified() ) ) );
412 pos += 30;
413 }
414 #endif
415
416 articleString += "</div>";
417
418 articleText += articleString.toUtf8().data();
419 }
420
421 // Ok. Now, does it go to main articles, or to alternate ones? We list
422 // main ones first, and alternates after.
423
424 // We do the case-folded comparison here.
425
426 wstring headwordStripped =
427 Folding::applySimpleCaseOnly( Utf8::decode( chain[ x ].word ) );
428 if( ignoreDiacritics )
429 headwordStripped = Folding::applyDiacriticsOnly( headwordStripped );
430
431 multimap< wstring, string > & mapToUse =
432 ( wordCaseFolded == headwordStripped ) ?
433 mainArticles : alternateArticles;
434
435 mapToUse.insert( pair< wstring, string >(
436 Folding::applySimpleCaseOnly( Utf8::decode( chain[ x ].word ) ),
437 articleText ) );
438
439 articlesIncluded.insert( chain[ x ].articleOffset );
440 }
441
442 if ( mainArticles.empty() && alternateArticles.empty() )
443 return new Dictionary::DataRequestInstant( false );
444
445 string result;
446
447 multimap< wstring, string >::const_iterator i;
448
449 for( i = mainArticles.begin(); i != mainArticles.end(); ++i )
450 result += i->second;
451
452 for( i = alternateArticles.begin(); i != alternateArticles.end(); ++i )
453 result += i->second;
454
455 sptr< Dictionary::DataRequestInstant > ret =
456 new Dictionary::DataRequestInstant( true );
457
458 ret->getData().resize( result.size() );
459
460 memcpy( &(ret->getData().front()), result.data(), result.size() );
461
462 return ret;
463 }
464 catch( std::exception & e )
465 {
466 return new Dictionary::DataRequestInstant( QString( e.what() ) );
467 }
468 }
469
getDescription()470 QString const& DictdDictionary::getDescription()
471 {
472 if( !dictionaryDescription.isEmpty() )
473 return dictionaryDescription;
474
475 sptr< Dictionary::DataRequest > req = getArticle( GD_NATIVE_TO_WS( L"00databaseinfo" ),
476 vector< wstring >(), wstring(), false );
477
478 if( req->dataSize() > 0 )
479 dictionaryDescription = Html::unescape( QString::fromUtf8( req->getFullData().data(), req->getFullData().size() ), true );
480 else
481 dictionaryDescription = "NONE";
482
483 return dictionaryDescription;
484 }
485
makeFTSIndex(QAtomicInt & isCancelled,bool firstIteration)486 void DictdDictionary::makeFTSIndex( QAtomicInt & isCancelled, bool firstIteration )
487 {
488 if( !( Dictionary::needToRebuildIndex( getDictionaryFilenames(), ftsIdxName )
489 || FtsHelpers::ftsIndexIsOldOrBad( ftsIdxName, this ) ) )
490 FTS_index_completed.ref();
491
492 if( haveFTSIndex() )
493 return;
494
495 if( ensureInitDone().size() )
496 return;
497
498 if( firstIteration && getArticleCount() > FTS::MaxDictionarySizeForFastSearch )
499 return;
500
501 gdDebug( "DictD: Building the full-text index for dictionary: %s\n",
502 getName().c_str() );
503
504 try
505 {
506 FtsHelpers::makeFTSIndex( this, isCancelled );
507 FTS_index_completed.ref();
508 }
509 catch( std::exception &ex )
510 {
511 gdWarning( "DictD: Failed building full-text search index for \"%s\", reason: %s\n", getName().c_str(), ex.what() );
512 QFile::remove( FsEncoding::decode( ftsIdxName.c_str() ) );
513 }
514 }
515
getArticleText(uint32_t articleAddress,QString & headword,QString & text)516 void DictdDictionary::getArticleText( uint32_t articleAddress, QString & headword, QString & text )
517 {
518 try
519 {
520 char buf[ 16384 ];
521 {
522 Mutex::Lock _( indexFileMutex );
523 indexFile.seek( articleAddress );
524
525 if ( !indexFile.gets( buf, sizeof( buf ), true ) )
526 throw exFailedToReadLineFromIndex();
527 }
528
529 char * tab1 = strchr( buf, '\t' );
530
531 if ( !tab1 )
532 throw exMalformedIndexFileLine();
533
534 headword = QString::fromUtf8( buf, tab1 - buf );
535
536 char * tab2 = strchr( tab1 + 1, '\t' );
537
538 if ( !tab2 )
539 throw exMalformedIndexFileLine();
540
541 // After tab1 should be article offset, after tab2 -- article size
542
543 uint32_t articleOffset = decodeBase64( string( tab1 + 1, tab2 - tab1 - 1 ) );
544
545 char * tab3 = strchr( tab2 + 1, '\t');
546
547 uint32_t articleSize;
548 if ( tab3 )
549 {
550 articleSize = decodeBase64( string( tab2 + 1, tab3 - tab2 - 1 ) );
551 }
552 else
553 {
554 articleSize = decodeBase64( tab2 + 1 );
555 }
556
557 string articleText;
558
559 char * articleBody;
560 {
561 Mutex::Lock _( dzMutex );
562 articleBody = dict_data_read_( dz, articleOffset, articleSize, 0, 0 );
563 }
564
565 if ( !articleBody )
566 {
567 articleText = dict_error_str( dz );
568 }
569 else
570 {
571 #if QT_VERSION >= QT_VERSION_CHECK( 5, 0, 0 )
572 static QRegularExpression phonetic( "\\\\([^\\\\]+)\\\\",
573 QRegularExpression::CaseInsensitiveOption ); // phonetics: \stuff\ ...
574 static QRegularExpression refs( "\\{([^\\{\\}]+)\\}",
575 QRegularExpression::CaseInsensitiveOption ); // links: {stuff}
576 #else
577 static QRegExp phonetic( "\\\\([^\\\\]+)\\\\", Qt::CaseInsensitive ); // phonetics: \stuff\ ...
578 static QRegExp refs( "\\{([^\\{\\}]+)\\}", Qt::CaseInsensitive ); // links: {stuff}
579 #endif
580
581 string convertedText = Html::preformat( articleBody, isToLanguageRTL() );
582 free( articleBody );
583
584 text = QString::fromUtf8( convertedText.data(), convertedText.size() )
585 .replace(phonetic, "<span class=\"dictd_phonetic\">\\1</span>")
586 .replace(refs, "<a href=\"gdlookup://localhost/\\1\">\\1</a>");
587
588 text = Html::unescape( text );
589 }
590 }
591 catch( std::exception &ex )
592 {
593 gdWarning( "DictD: Failed retrieving article from \"%s\", reason: %s\n", getName().c_str(), ex.what() );
594 }
595 }
596
getSearchResults(QString const & searchString,int searchMode,bool matchCase,int distanceBetweenWords,int maxResults,bool ignoreWordsOrder,bool ignoreDiacritics)597 sptr< Dictionary::DataRequest > DictdDictionary::getSearchResults( QString const & searchString,
598 int searchMode, bool matchCase,
599 int distanceBetweenWords,
600 int maxResults,
601 bool ignoreWordsOrder,
602 bool ignoreDiacritics )
603 {
604 return new FtsHelpers::FTSResultsRequest( *this, searchString,searchMode, matchCase, distanceBetweenWords, maxResults, ignoreWordsOrder, ignoreDiacritics );
605 }
606
607 } // anonymous namespace
608
makeDictionaries(vector<string> const & fileNames,string const & indicesDir,Dictionary::Initializing & initializing)609 vector< sptr< Dictionary::Class > > makeDictionaries(
610 vector< string > const & fileNames,
611 string const & indicesDir,
612 Dictionary::Initializing & initializing )
613 THROW_SPEC( std::exception )
614 {
615 vector< sptr< Dictionary::Class > > dictionaries;
616
617 for( vector< string >::const_iterator i = fileNames.begin(); i != fileNames.end();
618 ++i )
619 {
620 // Only allow .index suffixes
621
622 if ( i->size() < 6 ||
623 strcasecmp( i->c_str() + ( i->size() - 6 ), ".index" ) != 0 )
624 continue;
625
626 try
627 {
628 vector< string > dictFiles( 1, *i );
629
630 // Check if there is an 'abrv' file present
631 string baseName( *i, 0, i->size() - 5 );
632
633 dictFiles.push_back( string() );
634
635 if ( !File::tryPossibleName( baseName + "dict", dictFiles[ 1 ] ) &&
636 !File::tryPossibleName( baseName + "dict.dz", dictFiles[ 1 ] ) )
637 {
638 // No corresponding .dict file, skipping
639 continue;
640 }
641
642 string dictId = Dictionary::makeDictionaryId( dictFiles );
643
644 string indexFile = indicesDir + dictId;
645
646 if ( Dictionary::needToRebuildIndex( dictFiles, indexFile ) ||
647 indexIsOldOrBad( indexFile ) )
648 {
649 // Building the index
650 string dictionaryName = nameFromFileName( dictFiles[ 0 ] );
651
652 gdDebug( "DictD: Building the index for dictionary: %s\n", dictionaryName.c_str() );
653
654 initializing.indexingDictionary( dictionaryName );
655
656 File::Class idx( indexFile, "wb" );
657
658 IdxHeader idxHeader;
659
660 memset( &idxHeader, 0, sizeof( idxHeader ) );
661
662 // We write a dummy header first. At the end of the process the header
663 // will be rewritten with the right values.
664
665 idx.write( idxHeader );
666
667 IndexedWords indexedWords;
668
669 File::Class indexFile( dictFiles[ 0 ], "rb" );
670
671 // Read words from index until none's left.
672
673 char buf[ 16384 ];
674
675 do
676 {
677 uint32_t curOffset = indexFile.tell();
678
679 if ( !indexFile.gets( buf, sizeof( buf ), true ) )
680 break;
681
682 // Check that there are exactly two or three tabs in the record.
683 char * tab1 = strchr( buf, '\t' );
684 if ( tab1 )
685 {
686 char * tab2 = strchr( tab1 + 1, '\t' );
687 if ( tab2 )
688 {
689 char * tab3 = strchr( tab2 + 1, '\t');
690 if ( tab3 )
691 {
692 char * tab4 = strchr( tab3 + 1, '\t');
693 if ( tab4 )
694 {
695 GD_DPRINTF( "Warning: too many tabs present, skipping: %s\n", buf );
696 continue;
697 }
698
699 // Handle the forth entry, if it exists. From dictfmt man:
700 // When --index-keep-orig option is used fourth column is created
701 // (if necessary) in .index file.
702 indexedWords.addWord( Utf8::decode( string( tab3 + 1, strlen ( tab3 + 1 ) ) ), curOffset );
703 ++idxHeader.wordCount;
704 }
705 indexedWords.addWord( Utf8::decode( string( buf, strchr( buf, '\t' ) - buf ) ), curOffset );
706 ++idxHeader.wordCount;
707 ++idxHeader.articleCount;
708
709 // Check for proper dictionary name
710 if ( !strncmp( buf, "00databaseshort", 15 ) || !strncmp( buf, "00-database-short", 17 ) )
711 {
712 // After tab1 should be article offset, after tab2 -- article size
713 uint32_t articleOffset = decodeBase64( string( tab1 + 1, tab2 - tab1 - 1 ) );
714 uint32_t articleSize = decodeBase64( tab2 + 1 );
715
716 DZ_ERRORS error;
717 dictData * dz = dict_data_open( dictFiles[ 1 ].c_str(), &error, 0 );
718
719 if ( dz )
720 {
721 char * articleBody = dict_data_read_( dz, articleOffset, articleSize, 0, 0 );
722 if ( articleBody )
723 {
724 char * eol;
725 if ( !strncmp( articleBody, "00databaseshort", 15 ) || !strncmp( articleBody, "00-database-short", 17 ) )
726 eol = strchr( articleBody, '\n' ); // skip the first line (headword itself)
727 else
728 eol = articleBody; // No headword itself
729 if ( eol )
730 {
731 while( *eol && Utf8::isspace( *eol ) ) ++eol; // skip spaces
732
733 // use only the single line for the dictionary title
734 char * endEol = strchr( eol, '\n' );
735 if ( endEol )
736 *endEol = 0;
737
738 GD_DPRINTF( "DICT NAME: '%s'\n", eol );
739 dictionaryName = eol;
740 }
741 }
742 dict_data_close( dz );
743 }
744 else
745 throw exDictzipError( string( dz_error_str( error ) )
746 + "(" + dictFiles[ 1 ] + ")" );
747 }
748 }
749 else
750 {
751 GD_DPRINTF( "Warning: only a single tab present, skipping: %s\n", buf );
752 continue;
753 }
754 }
755 else
756 {
757 GD_DPRINTF( "Warning: no tabs present, skipping: %s\n", buf );
758 continue;
759 }
760
761
762 } while( !indexFile.eof() );
763
764
765 // Write dictionary name
766
767 idx.write( (uint32_t) dictionaryName.size() );
768 idx.write( dictionaryName.data(), dictionaryName.size() );
769
770 // Build index
771
772 IndexInfo idxInfo = BtreeIndexing::buildIndex( indexedWords, idx );
773
774 idxHeader.indexBtreeMaxElements = idxInfo.btreeMaxElements;
775 idxHeader.indexRootOffset = idxInfo.rootOffset;
776
777 // That concludes it. Update the header.
778
779 idxHeader.signature = Signature;
780 idxHeader.formatVersion = CurrentFormatVersion;
781
782 // read languages
783 QPair<quint32,quint32> langs =
784 LangCoder::findIdsForFilename( QString::fromStdString( dictFiles[ 0 ] ) );
785
786 // if no languages found, try dictionary's name
787 if ( langs.first == 0 || langs.second == 0 )
788 {
789 langs =
790 LangCoder::findIdsForFilename( QString::fromStdString( nameFromFileName( dictFiles[ 0 ] ) ) );
791 }
792
793 idxHeader.langFrom = langs.first;
794 idxHeader.langTo = langs.second;
795
796 idx.rewind();
797
798 idx.write( &idxHeader, sizeof( idxHeader ) );
799 }
800
801 dictionaries.push_back( new DictdDictionary( dictId,
802 indexFile,
803 dictFiles ) );
804 }
805 catch( std::exception & e )
806 {
807 gdWarning( "Dictd dictionary \"%s\" reading failed, error: %s\n",
808 i->c_str(), e.what() );
809 }
810 }
811
812 return dictionaries;
813 }
814
815 }
816
