1 //
2 // $Id: searchd.cpp 4113 2013-08-26 07:43:28Z deogar $
3 //
4
5 //
6 // Copyright (c) 2001-2013, Andrew Aksyonoff
7 // Copyright (c) 2008-2013, Sphinx Technologies Inc
8 // All rights reserved
9 //
10 // This program is free software; you can redistribute it and/or modify
11 // it under the terms of the GNU General Public License. You should have
12 // received a copy of the GPL license along with this program; if you
13 // did not, you can find it at http://www.gnu.org/
14 //
15
16 #include "sphinx.h"
17 #include "sphinxutils.h"
18 #include "sphinxexcerpt.h"
19 #include "sphinxrt.h"
20 #include "sphinxint.h"
21 #include "sphinxquery.h"
22
23 #include <errno.h>
24 #include <fcntl.h>
25 #include <signal.h>
26 #include <stdio.h>
27 #include <string.h>
28 #include <sys/stat.h>
29 #include <sys/types.h>
30 #include <time.h>
31 #include <stdarg.h>
32 #include <limits.h>
33 #include <locale.h>
34
35 #define SEARCHD_BACKLOG 5
36 #define SPHINXAPI_PORT 9312
37 #define SPHINXQL_PORT 9306
38 #define SPH_ADDRESS_SIZE sizeof("000.000.000.000")
39 #define SPH_ADDRPORT_SIZE sizeof("000.000.000.000:00000")
40 #define MVA_UPDATES_POOL 1048576
41 #define NETOUTBUF 8192
42
43
44 // don't shutdown on SIGKILL (debug purposes)
45 // 1 - SIGKILL will shut down the whole daemon; 0 - watchdog will reincarnate the daemon
46 #define WATCHDOG_SIGKILL 1
47
48
49 /////////////////////////////////////////////////////////////////////////////
50
51 #if USE_WINDOWS
52 // Win-specific headers and calls
53 #include <io.h>
54 #include <winsock2.h>
55 #include <tlhelp32.h>
56
57 #define sphSockRecv(_sock,_buf,_len) ::recv(_sock,_buf,_len,0)
58 #define sphSockSend(_sock,_buf,_len) ::send(_sock,_buf,_len,0)
59 #define sphSockClose(_sock) ::closesocket(_sock)
60
61 #define stat _stat
62
63 #else
64 // UNIX-specific headers and calls
65 #include <unistd.h>
66 #include <netinet/in.h>
67 #include <netinet/tcp.h>
68 #include <sys/file.h>
69 #include <sys/socket.h>
70 #include <sys/time.h>
71 #include <sys/wait.h>
72 #include <sys/un.h>
73 #include <netdb.h>
74
75 // there's no MSG_NOSIGNAL on OS X
76 #ifndef MSG_NOSIGNAL
77 #define MSG_NOSIGNAL 0
78 #endif
79
80 #define sphSockRecv(_sock,_buf,_len) ::recv(_sock,_buf,_len,MSG_NOSIGNAL)
81 #define sphSockSend(_sock,_buf,_len) ::send(_sock,_buf,_len,MSG_NOSIGNAL)
82 #define sphSockClose(_sock) ::close(_sock)
83
84 #endif
85
86 #if USE_SYSLOG
87 #include <syslog.h>
88 #endif
89
90 /////////////////////////////////////////////////////////////////////////////
91 // MISC GLOBALS
92 /////////////////////////////////////////////////////////////////////////////
93
94 struct ServedDesc_t
95 {
96 CSphIndex * m_pIndex;
97 CSphString m_sIndexPath;
98 bool m_bEnabled; ///< to disable index in cases when rotation fails
99 bool m_bMlock;
100 bool m_bPreopen;
101 bool m_bOnDiskDict;
102 bool m_bStar;
103 bool m_bExpand;
104 bool m_bToDelete;
105 bool m_bOnlyNew;
106 bool m_bRT;
107
108 ServedDesc_t ();
109 ~ServedDesc_t ();
110 };
111
112 struct ServedIndex_t : public ISphNoncopyable, public ServedDesc_t
113 {
114 public:
ServedIndex_tServedIndex_t115 ServedIndex_t () {}
116 ~ServedIndex_t ();
117
118 void ReadLock () const;
119 void WriteLock () const;
120 void Unlock () const;
121
122 bool InitLock () const;
123
124 private:
125 mutable CSphRwlock m_tLock;
126 };
127
128 /////////////////////////////////////////////////////////////////////////////
129
130 enum ESphAddIndex
131 {
132 ADD_ERROR = 0,
133 ADD_LOCAL = 1,
134 ADD_DISTR = 2,
135 ADD_RT = 3
136 };
137
138
139 enum ProtocolType_e
140 {
141 PROTO_SPHINX = 0,
142 PROTO_MYSQL41,
143
144 PROTO_TOTAL
145 };
146
147
148 static const char * g_dProtoNames[PROTO_TOTAL] =
149 {
150 "sphinxapi", "sphinxql"
151 };
152
153
154 static bool g_bService = false;
155 #if USE_WINDOWS
156 static bool g_bServiceStop = false;
157 static const char * g_sServiceName = "searchd";
158 HANDLE g_hPipe = INVALID_HANDLE_VALUE;
159 #endif
160
161 static CSphVector<CSphString> g_dArgs;
162
163 static bool g_bHeadDaemon = false;
164 static bool g_bLogStdout = true;
165
166 struct CrashQuery_t
167 {
168 const BYTE * m_pQuery; // last query
169 int m_iSize; // last query size
170 WORD m_uCMD; // last command (header)
171 WORD m_uVer; // last command's version (header)
172 bool m_bMySQL; // is query from MySQL or API
173
CrashQuery_tCrashQuery_t174 CrashQuery_t ()
175 : m_pQuery ( NULL )
176 , m_iSize ( 0 )
177 , m_uCMD ( 0 )
178 , m_uVer ( 0 )
179 , m_bMySQL ( false )
180 {
181 }
182 };
183
184 class SphCrashLogger_c
185 {
186 public:
SphCrashLogger_c()187 SphCrashLogger_c () {}
188
189 static void Init ();
190 static void Done ();
191
192 #if !USE_WINDOWS
193 static void HandleCrash ( int );
194 #else
195 static LONG WINAPI HandleCrash ( EXCEPTION_POINTERS * pExc );
196 #endif
197 static void SetLastQuery ( const CrashQuery_t & tQuery );
198 static void SetupTimePID ();
199 static CrashQuery_t GetQuery ();
200 void SetupTLS ();
201
202 private:
203 CrashQuery_t m_tQuery; // per thread copy of last query for thread mode
204 static CrashQuery_t m_tForkQuery; // copy of last query for fork / prefork modes
205 static SphThreadKey_t m_tLastQueryTLS; // last query ( non threaded workers could use dist_threads too )
206 };
207
208 enum LogFormat_e
209 {
210 LOG_FORMAT_PLAIN,
211 LOG_FORMAT_SPHINXQL
212 };
213
214 static ESphLogLevel g_eLogLevel = SPH_LOG_INFO;
215 static int g_iLogFile = STDOUT_FILENO; // log file descriptor
216 static bool g_bLogSyslog = false;
217 static bool g_bQuerySyslog = false;
218 static CSphString g_sLogFile; // log file name
219 static bool g_bLogTty = false; // cached isatty(g_iLogFile)
220 static LogFormat_e g_eLogFormat = LOG_FORMAT_PLAIN;
221
222 static int g_iReadTimeout = 5; // sec
223 static int g_iWriteTimeout = 5;
224 static int g_iClientTimeout = 300;
225 static int g_iMaxChildren = 0;
226 #if !USE_WINDOWS
227 static bool g_bPreopenIndexes = true;
228 #else
229 static bool g_bPreopenIndexes = false;
230 #endif
231 static bool g_bOnDiskDicts = false;
232 static bool g_bWatchdog = true;
233 static int g_iExpansionLimit = 0;
234 static bool g_bCompatResults = true;
235
236 struct Listener_t
237 {
238 int m_iSock;
239 ProtocolType_e m_eProto;
240 };
241 static CSphVector<Listener_t> g_dListeners;
242
243 static int g_iQueryLogFile = -1;
244 static CSphString g_sQueryLogFile;
245 static const char * g_sPidFile = NULL;
246 static int g_iPidFD = -1;
247 static int g_iMaxMatches = 1000;
248
249 static int g_iMaxCachedDocs = 0; // in bytes
250 static int g_iMaxCachedHits = 0; // in bytes
251
252 static int g_iAttrFlushPeriod = 0; // in seconds; 0 means "do not flush"
253 static int g_iMaxPacketSize = 8*1024*1024; // in bytes; for both query packets from clients and response packets from agents
254 static int g_iMaxFilters = 256;
255 static int g_iMaxFilterValues = 4096;
256 static int g_iMaxBatchQueries = 32;
257 static ESphCollation g_eCollation = SPH_COLLATION_DEFAULT;
258 #if !USE_WINDOWS
259 static CSphProcessSharedVariable<bool> g_tHaveTTY ( true );
260 #endif
261 enum Mpm_e
262 {
263 MPM_NONE, ///< process queries in a loop one by one (eg. in --console)
264 MPM_FORK, ///< fork a worker process for each query
265 MPM_PREFORK, ///< keep a number of pre-forked processes
266 MPM_THREADS ///< create a worker thread for each query
267 };
268
269 static Mpm_e g_eWorkers = USE_WINDOWS ? MPM_THREADS : MPM_FORK;
270
271 static int g_iPreforkChildren = 10; // how much workers to keep
272 static CSphVector<int> g_dChildren;
273 static int g_iClientFD = -1;
274 static int g_iDistThreads = 0;
275
276 enum ThdState_e
277 {
278 THD_HANDSHAKE,
279 THD_NET_READ,
280 THD_NET_WRITE,
281 THD_QUERY,
282
283 THD_STATE_TOTAL
284 };
285
286 static const char * g_dThdStates[THD_STATE_TOTAL] = {
287 "handshake", "net_read", "net_write", "query"
288 };
289
290 struct ThdDesc_t
291 {
292 SphThread_t m_tThd;
293 ProtocolType_e m_eProto;
294 int m_iClientSock;
295 CSphString m_sClientName;
296
297 ThdState_e m_eThdState;
298 const char * m_sCommand;
299
300 int m_iConnID; ///< current conn-id for this thread
301
ThdDesc_tThdDesc_t302 ThdDesc_t ()
303 : m_iClientSock ( 0 )
304 , m_sCommand ( NULL )
305 , m_iConnID ( -1 )
306 {}
307 };
308
309 struct StaticThreadsOnlyMutex_t
310 {
311 StaticThreadsOnlyMutex_t ();
312 ~StaticThreadsOnlyMutex_t ();
313 void Lock ();
314 void Unlock ();
315
316 private:
317 CSphMutex m_tLock;
318 };
319
320
321 static StaticThreadsOnlyMutex_t g_tThdMutex;
322 static CSphVector<ThdDesc_t*> g_dThd; ///< existing threads table
323
324 static int g_iConnID = 0; ///< global conn-id in none/fork/threads; current conn-id in prefork
325 static SphThreadKey_t g_tConnKey; ///< current conn-id TLS in threads
326 static int * g_pConnID = NULL; ///< global conn-id ptr in prefork
327 static CSphSharedBuffer<BYTE> g_dConnID; ///< global conn-id storage in prefork (protected by accept mutex)
328
329 // handshake
330 static char g_sMysqlHandshake[128];
331 static int g_iMysqlHandshake = 0;
332
333 //////////////////////////////////////////////////////////////////////////
334
335 static CSphString g_sConfigFile;
336 static DWORD g_uCfgCRC32 = 0;
337 static struct stat g_tCfgStat;
338
339 static CSphConfigParser g_pCfg;
340
341 #if USE_WINDOWS
342 static bool g_bSeamlessRotate = false;
343 #else
344 static bool g_bSeamlessRotate = true;
345 #endif
346
347 static bool g_bIOStats = false;
348 static bool g_bCpuStats = false;
349 static bool g_bOptNoDetach = false;
350 static bool g_bOptNoLock = false;
351 static bool g_bSafeTrace = false;
352 static bool g_bStripPath = false;
353
354 static volatile bool g_bDoDelete = false; // do we need to delete any indexes?
355 static volatile int g_iRotateCount = 0; // flag that we are rotating now; set from SIGHUP; cleared on rotation success
356 static volatile sig_atomic_t g_bGotSighup = 0; // we just received SIGHUP; need to log
357 static volatile sig_atomic_t g_bGotSigterm = 0; // we just received SIGTERM; need to shutdown
358 static volatile sig_atomic_t g_bGotSigchld = 0; // we just received SIGCHLD; need to count dead children
359 static volatile sig_atomic_t g_bGotSigusr1 = 0; // we just received SIGUSR1; need to reopen logs
360
361 // pipe to watchdog to inform that daemon is going to close, so no need to restart it in case of crash
362 static CSphSharedBuffer<DWORD> g_bDaemonAtShutdown;
363
364 static CSphVector<int> g_dHupChildren; // children to send hup signal on rotation is done
365 static int64_t g_tmRotateChildren = 0; // pause to next children term signal after rotation is done
366 static int g_iRotationThrottle = 0; // pause between children term signals after rotation is done
367
368 /// global index hash
369 /// used in both non-threaded and multi-threaded modes
370 ///
371 /// hash entry is a CSphIndex pointer, rwlock, and a few flags (see ServedIndex_t)
372 /// rlock on entry guarantees it won't change, eg. that index pointer will stay alive
373 /// wlock on entry allows to change (delete/replace) the index pointer
374 ///
375 /// note that entry locks are held outside the hash
376 /// and Delete() honours that by acquiring wlock on an entry first
377 class IndexHash_c : protected SmallStringHash_T<ServedIndex_t>
378 {
379 friend class IndexHashIterator_c;
380 typedef SmallStringHash_T<ServedIndex_t> BASE;
381
382 public:
383 explicit IndexHash_c ();
384 virtual ~IndexHash_c ();
385
GetLength() const386 int GetLength () const { return BASE::GetLength(); }
Reset()387 void Reset () { BASE::Reset(); }
388
389 bool Add ( const ServedDesc_t & tDesc, const CSphString & tKey );
390 bool Delete ( const CSphString & tKey );
391
392 const ServedIndex_t * GetRlockedEntry ( const CSphString & tKey ) const;
393 ServedIndex_t * GetWlockedEntry ( const CSphString & tKey ) const;
394 ServedIndex_t & GetUnlockedEntry ( const CSphString & tKey ) const;
395 bool Exists ( const CSphString & tKey ) const;
396
397 protected:
398 void Rlock () const;
399 void Wlock () const;
400 void Unlock () const;
401
402 private:
403 mutable CSphRwlock m_tLock;
404 };
405
406
407 /// multi-threaded hash iterator
408 class IndexHashIterator_c : public ISphNoncopyable
409 {
410 public:
411 explicit IndexHashIterator_c ( const IndexHash_c * pHash, bool bWrite=false );
412 ~IndexHashIterator_c ();
413
414 bool Next ();
415 ServedIndex_t & Get ();
416 const CSphString & GetKey ();
417
418 private:
419 const IndexHash_c * m_pHash;
420 IndexHash_c::HashEntry_t * m_pIterator;
421 };
422
423
424 static IndexHash_c * g_pIndexes = NULL; // served indexes hash
425 static CSphVector<const char *> g_dRotating; // names of indexes to be rotated this time
426 static const char * g_sPrereading = NULL; // name of index currently being preread
427 static CSphIndex * g_pPrereading = NULL; // rotation "buffer"
428
429 static StaticThreadsOnlyMutex_t g_tRotateQueueMutex;
430 static CSphVector<CSphString> g_dRotateQueue; // FIXME? maybe replace it with lockless ring buffer
431 static StaticThreadsOnlyMutex_t g_tRotateConfigMutex;
432 static SphThread_t g_tRotateThread;
433 static volatile bool g_bRotateShutdown = false;
434
435 /// flush parameters of rt indexes
436 static SphThread_t g_tRtFlushThread;
437 static volatile bool g_bRtFlushShutdown = false;
438
439 static StaticThreadsOnlyMutex_t g_tDistLock;
440
441 enum
442 {
443 SPH_PIPE_PREREAD
444 };
445
446 struct PipeInfo_t
447 {
448 int m_iFD; ///< read-pipe to child
449 int m_iHandler; ///< who's my handler (SPH_PIPE_xxx)
450
PipeInfo_tPipeInfo_t451 PipeInfo_t () : m_iFD ( -1 ), m_iHandler ( -1 ) {}
452 };
453
454 static CSphVector<PipeInfo_t> g_dPipes; ///< currently open read-pipes to children processes
455
456 struct PoolPtrs_t
457 {
458 const DWORD * m_pMva;
459 const BYTE * m_pStrings;
460
PoolPtrs_tPoolPtrs_t461 PoolPtrs_t ()
462 : m_pMva ( NULL )
463 , m_pStrings ( NULL )
464 {}
465 };
466
467 /////////////////////////////////////////////////////////////////////////////
468
469 /// known commands
470 enum SearchdCommand_e
471 {
472 SEARCHD_COMMAND_SEARCH = 0,
473 SEARCHD_COMMAND_EXCERPT = 1,
474 SEARCHD_COMMAND_UPDATE = 2,
475 SEARCHD_COMMAND_KEYWORDS = 3,
476 SEARCHD_COMMAND_PERSIST = 4,
477 SEARCHD_COMMAND_STATUS = 5,
478 SEARCHD_COMMAND_FLUSHATTRS = 7,
479 SEARCHD_COMMAND_SPHINXQL = 8,
480
481 SEARCHD_COMMAND_TOTAL
482 };
483
484
485 /// known command versions
486 enum
487 {
488 VER_COMMAND_SEARCH = 0x119,
489 VER_COMMAND_EXCERPT = 0x104,
490 VER_COMMAND_UPDATE = 0x102,
491 VER_COMMAND_KEYWORDS = 0x100,
492 VER_COMMAND_STATUS = 0x100,
493 VER_COMMAND_FLUSHATTRS = 0x100,
494 VER_COMMAND_SPHINXQL = 0x100
495 };
496
497
498 /// known status return codes
499 enum SearchdStatus_e
500 {
501 SEARCHD_OK = 0, ///< general success, command-specific reply follows
502 SEARCHD_ERROR = 1, ///< general failure, error message follows
503 SEARCHD_RETRY = 2, ///< temporary failure, error message follows, client should retry later
504 SEARCHD_WARNING = 3 ///< general success, warning message and command-specific reply follow
505 };
506
507
508 enum
509 {
510 VER_MASTER = 1
511 };
512
513
514 /// command names
515 static const char * g_dApiCommands[SEARCHD_COMMAND_TOTAL] =
516 {
517 "search", "excerpt", "update", "keywords", "persist", "status", "query", "flushattrs"
518 };
519
520 const int MAX_RETRY_COUNT = 8;
521 const int MAX_RETRY_DELAY = 1000;
522
523 //////////////////////////////////////////////////////////////////////////
524
525 const int STATS_MAX_AGENTS = 1024; ///< we'll track stats for this much remote agents
526
527 /// per-agent query stats
528 struct AgentStats_t
529 {
530 int64_t m_iTimeoutsQuery; ///< number of time-outed queries
531 int64_t m_iTimeoutsConnect; ///< number of time-outed connections
532 int64_t m_iConnectFailures; ///< failed to connect
533 int64_t m_iNetworkErrors; ///< network error
534 int64_t m_iWrongReplies; ///< incomplete reply
535 int64_t m_iUnexpectedClose; ///< agent closed the connection
536 };
537
538 struct SearchdStats_t
539 {
540 DWORD m_uStarted;
541 int64_t m_iConnections;
542 int64_t m_iMaxedOut;
543 int64_t m_iCommandCount[SEARCHD_COMMAND_TOTAL];
544 int64_t m_iAgentConnect;
545 int64_t m_iAgentRetry;
546
547 int64_t m_iQueries; ///< search queries count (differs from search commands count because of multi-queries)
548 int64_t m_iQueryTime; ///< wall time spent (including network wait time)
549 int64_t m_iQueryCpuTime; ///< CPU time spent
550
551 int64_t m_iDistQueries; ///< distributed queries count
552 int64_t m_iDistWallTime; ///< wall time spent on distributed queries
553 int64_t m_iDistLocalTime; ///< wall time spent searching local indexes in distributed queries
554 int64_t m_iDistWaitTime; ///< time spent waiting for remote agents in distributed queries
555
556 int64_t m_iDiskReads; ///< total read IO calls (fired by search queries)
557 int64_t m_iDiskReadBytes; ///< total read IO traffic
558 int64_t m_iDiskReadTime; ///< total read IO time
559
560 DWORD m_bmAgentStats[STATS_MAX_AGENTS/32]; ///< per-agent storage usage bitmap
561 AgentStats_t m_dAgentStats[STATS_MAX_AGENTS]; ///< per-agent storage
562 };
563
564 static SearchdStats_t * g_pStats = NULL;
565 static CSphSharedBuffer<SearchdStats_t> g_tStatsBuffer;
566 static CSphProcessSharedMutex g_tStatsMutex;
567
568 //////////////////////////////////////////////////////////////////////////
569
570 struct FlushState_t
571 {
572 int m_bFlushing; ///< update flushing in progress
573 int m_iFlushTag; ///< last flushed tag
574 bool m_bForceCheck; ///< forced check/flush flag
575 };
576
577 static volatile FlushState_t * g_pFlush = NULL;
578 static CSphSharedBuffer<FlushState_t> g_tFlushBuffer;
579
580 //////////////////////////////////////////////////////////////////////////
581
582 /// available uservar types
583 enum Uservar_e
584 {
585 USERVAR_INT_SET
586 };
587
588 /// uservar name to value binding
589 struct Uservar_t
590 {
591 Uservar_e m_eType;
592 UservarIntSet_c * m_pVal;
593
Uservar_tUservar_t594 Uservar_t ()
595 : m_eType ( USERVAR_INT_SET )
596 , m_pVal ( NULL )
597 {}
598 };
599
600 static StaticThreadsOnlyMutex_t g_tUservarsMutex;
601 static SmallStringHash_T<Uservar_t> g_hUservars;
602
603 /////////////////////////////////////////////////////////////////////////////
604 // MACHINE-DEPENDENT STUFF
605 /////////////////////////////////////////////////////////////////////////////
606
607 #if USE_WINDOWS
608
609 // Windows hacks
610 #undef EINTR
611 #define LOCK_EX 0
612 #define LOCK_UN 1
613 #define STDIN_FILENO fileno(stdin)
614 #define STDOUT_FILENO fileno(stdout)
615 #define STDERR_FILENO fileno(stderr)
616 #define ETIMEDOUT WSAETIMEDOUT
617 #define EWOULDBLOCK WSAEWOULDBLOCK
618 #define EINPROGRESS WSAEINPROGRESS
619 #define EINTR WSAEINTR
620 #define ECONNRESET WSAECONNRESET
621 #define ECONNABORTED WSAECONNABORTED
622 #define socklen_t int
623
624 #define ftruncate _chsize
625 #define getpid GetCurrentProcessId
626
627 #endif // USE_WINDOWS
628
629 const int EXT_COUNT = 8;
630 const int EXT_MVP = 8;
631 static const char * g_dNewExts[EXT_COUNT] = { ".new.sph", ".new.spa", ".new.spi", ".new.spd", ".new.spp", ".new.spm", ".new.spk", ".new.sps" };
632 static const char * g_dOldExts[] = { ".old.sph", ".old.spa", ".old.spi", ".old.spd", ".old.spp", ".old.spm", ".old.spk", ".old.sps", ".old.mvp" };
633 static const char * g_dCurExts[] = { ".sph", ".spa", ".spi", ".spd", ".spp", ".spm", ".spk", ".sps", ".mvp" };
634
635 /////////////////////////////////////////////////////////////////////////////
636 // MISC
637 /////////////////////////////////////////////////////////////////////////////
638
ReleaseTTYFlag()639 void ReleaseTTYFlag()
640 {
641 #if !USE_WINDOWS
642 g_tHaveTTY.WriteValue(false);
643 #endif
644 }
645
ServedDesc_t()646 ServedDesc_t::ServedDesc_t ()
647 {
648 m_pIndex = NULL;
649 m_bEnabled = true;
650 m_bMlock = false;
651 m_bPreopen = false;
652 m_bOnDiskDict = false;
653 m_bStar = false;
654 m_bExpand = false;
655 m_bToDelete = false;
656 m_bOnlyNew = false;
657 m_bRT = false;
658 }
659
~ServedDesc_t()660 ServedDesc_t::~ServedDesc_t ()
661 {
662 SafeDelete ( m_pIndex );
663 }
664
~ServedIndex_t()665 ServedIndex_t::~ServedIndex_t ()
666 {
667 if ( g_eWorkers==MPM_THREADS )
668 Verify ( m_tLock.Done() );
669 }
670
ReadLock() const671 void ServedIndex_t::ReadLock () const
672 {
673 if ( g_eWorkers==MPM_THREADS )
674 {
675 if ( m_tLock.ReadLock() )
676 sphLogDebugvv ( "ReadLock %p", this );
677 else
678 {
679 sphLogDebug ( "ReadLock %p failed", this );
680 assert ( false );
681 }
682 }
683 }
684
WriteLock() const685 void ServedIndex_t::WriteLock () const
686 {
687 if ( g_eWorkers==MPM_THREADS )
688 {
689 if ( m_tLock.WriteLock() )
690 sphLogDebugvv ( "WriteLock %p", this );
691 else
692 {
693 sphLogDebug ( "WriteLock %p failed", this );
694 assert ( false );
695 }
696 }
697 }
698
InitLock() const699 bool ServedIndex_t::InitLock () const
700 {
701 return ( g_eWorkers==MPM_THREADS ) ? m_tLock.Init () : true;
702 }
703
Unlock() const704 void ServedIndex_t::Unlock () const
705 {
706 if ( g_eWorkers==MPM_THREADS )
707 {
708 if ( m_tLock.Unlock() )
709 sphLogDebugvv ( "Unlock %p", this );
710 else
711 {
712 sphLogDebug ( "Unlock %p failed", this );
713 assert ( false );
714 }
715 }
716 }
717
718 //////////////////////////////////////////////////////////////////////////
719
IndexHashIterator_c(const IndexHash_c * pHash,bool bWrite)720 IndexHashIterator_c::IndexHashIterator_c ( const IndexHash_c * pHash, bool bWrite )
721 : m_pHash ( pHash )
722 , m_pIterator ( NULL )
723 {
724 if ( !bWrite )
725 m_pHash->Rlock();
726 else
727 m_pHash->Wlock();
728 }
729
~IndexHashIterator_c()730 IndexHashIterator_c::~IndexHashIterator_c ()
731 {
732 m_pHash->Unlock();
733 }
734
Next()735 bool IndexHashIterator_c::Next ()
736 {
737 m_pIterator = m_pIterator ? m_pIterator->m_pNextByOrder : m_pHash->m_pFirstByOrder;
738 return m_pIterator!=NULL;
739 }
740
Get()741 ServedIndex_t & IndexHashIterator_c::Get ()
742 {
743 assert ( m_pIterator );
744 return m_pIterator->m_tValue;
745 }
746
GetKey()747 const CSphString & IndexHashIterator_c::GetKey ()
748 {
749 assert ( m_pIterator );
750 return m_pIterator->m_tKey;
751 }
752
753 //////////////////////////////////////////////////////////////////////////
754
IndexHash_c()755 IndexHash_c::IndexHash_c ()
756 {
757 if ( g_eWorkers==MPM_THREADS )
758 if ( !m_tLock.Init() )
759 sphDie ( "failed to init hash indexes rwlock" );
760 }
761
762
~IndexHash_c()763 IndexHash_c::~IndexHash_c()
764 {
765 if ( g_eWorkers==MPM_THREADS )
766 Verify ( m_tLock.Done() );
767 }
768
769
Rlock() const770 void IndexHash_c::Rlock () const
771 {
772 if ( g_eWorkers==MPM_THREADS )
773 Verify ( m_tLock.ReadLock() );
774 }
775
776
Wlock() const777 void IndexHash_c::Wlock () const
778 {
779 if ( g_eWorkers==MPM_THREADS )
780 Verify ( m_tLock.WriteLock() );
781 }
782
783
Unlock() const784 void IndexHash_c::Unlock () const
785 {
786 if ( g_eWorkers==MPM_THREADS )
787 Verify ( m_tLock.Unlock() );
788 }
789
790
Add(const ServedDesc_t & tDesc,const CSphString & tKey)791 bool IndexHash_c::Add ( const ServedDesc_t & tDesc, const CSphString & tKey )
792 {
793 Wlock();
794 int iPrevSize = GetLength ();
795 ServedIndex_t & tVal = BASE::AddUnique ( tKey );
796 bool bAdded = ( iPrevSize<GetLength() );
797 if ( bAdded )
798 {
799 *( (ServedDesc_t *)&tVal ) = tDesc;
800 Verify ( tVal.InitLock() );
801 }
802 Unlock();
803 return bAdded;
804 }
805
806
Delete(const CSphString & tKey)807 bool IndexHash_c::Delete ( const CSphString & tKey )
808 {
809 // tricky part
810 // hash itself might be unlocked, but entry (!) might still be locked
811 // hence, we also need to acquire a lock on entry, and an exclusive one
812 Wlock();
813 bool bRes = false;
814 ServedIndex_t * pEntry = BASE::operator() ( tKey );
815 if ( pEntry )
816 {
817 pEntry->WriteLock();
818 pEntry->Unlock();
819 bRes = BASE::Delete ( tKey );
820 }
821 Unlock();
822 return bRes;
823 }
824
825
GetRlockedEntry(const CSphString & tKey) const826 const ServedIndex_t * IndexHash_c::GetRlockedEntry ( const CSphString & tKey ) const
827 {
828 Rlock();
829 ServedIndex_t * pEntry = BASE::operator() ( tKey );
830 if ( pEntry )
831 pEntry->ReadLock();
832 Unlock();
833 return pEntry;
834 }
835
836
GetWlockedEntry(const CSphString & tKey) const837 ServedIndex_t * IndexHash_c::GetWlockedEntry ( const CSphString & tKey ) const
838 {
839 Rlock();
840 ServedIndex_t * pEntry = BASE::operator() ( tKey );
841 if ( pEntry )
842 pEntry->WriteLock();
843 Unlock();
844 return pEntry;
845 }
846
847
GetUnlockedEntry(const CSphString & tKey) const848 ServedIndex_t & IndexHash_c::GetUnlockedEntry ( const CSphString & tKey ) const
849 {
850 Rlock();
851 ServedIndex_t & tRes = BASE::operator[] ( tKey );
852 Unlock();
853 return tRes;
854 }
855
856
Exists(const CSphString & tKey) const857 bool IndexHash_c::Exists ( const CSphString & tKey ) const
858 {
859 Rlock();
860 bool bRes = BASE::Exists ( tKey );
861 Unlock();
862 return bRes;
863 }
864
865 //////////////////////////////////////////////////////////////////////////
866
StaticThreadsOnlyMutex_t()867 StaticThreadsOnlyMutex_t::StaticThreadsOnlyMutex_t ()
868 {
869 if ( !m_tLock.Init() )
870 sphDie ( "failed to create static mutex" );
871 }
872
~StaticThreadsOnlyMutex_t()873 StaticThreadsOnlyMutex_t::~StaticThreadsOnlyMutex_t ()
874 {
875 m_tLock.Done();
876 }
877
Lock()878 void StaticThreadsOnlyMutex_t::Lock ()
879 {
880 if ( g_eWorkers==MPM_THREADS )
881 m_tLock.Lock();
882 }
883
Unlock()884 void StaticThreadsOnlyMutex_t::Unlock()
885 {
886 if ( g_eWorkers==MPM_THREADS )
887 m_tLock.Unlock();
888 }
889
890 /////////////////////////////////////////////////////////////////////////////
891 // LOGGING
892 /////////////////////////////////////////////////////////////////////////////
893
894 void Shutdown (); // forward ref for sphFatal()
895
896
897 /// format current timestamp for logging
sphFormatCurrentTime(char * sTimeBuf,int iBufLen)898 int sphFormatCurrentTime ( char * sTimeBuf, int iBufLen )
899 {
900 int64_t iNow = sphMicroTimer ();
901 time_t ts = (time_t) ( iNow/1000000 ); // on some systems (eg. FreeBSD 6.2), tv.tv_sec has another type and we can't just pass it
902
903 #if !USE_WINDOWS
904 struct tm tmp;
905 localtime_r ( &ts, &tmp );
906 #else
907 struct tm tmp;
908 tmp = *localtime ( &ts );
909 #endif
910
911 static const char * sWeekday[7] = { "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" };
912 static const char * sMonth[12] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" };
913
914 return snprintf ( sTimeBuf, iBufLen, "%.3s %.3s%3d %.2d:%.2d:%.2d.%.3d %d",
915 sWeekday [ tmp.tm_wday ],
916 sMonth [ tmp.tm_mon ],
917 tmp.tm_mday, tmp.tm_hour,
918 tmp.tm_min, tmp.tm_sec, (int)((iNow%1000000)/1000),
919 1900+tmp.tm_year );
920 }
921
922
923 /// physically emit log entry
924 /// buffer must have 1 extra byte for linefeed
925 #if USE_WINDOWS
sphLogEntry(ESphLogLevel eLevel,char * sBuf,char * sTtyBuf)926 void sphLogEntry ( ESphLogLevel eLevel, char * sBuf, char * sTtyBuf )
927 #else
928 void sphLogEntry ( ESphLogLevel , char * sBuf, char * sTtyBuf )
929 #endif
930 {
931 #if USE_WINDOWS
932 if ( g_bService && g_iLogFile==STDOUT_FILENO )
933 {
934 HANDLE hEventSource;
935 LPCTSTR lpszStrings[2];
936
937 hEventSource = RegisterEventSource ( NULL, g_sServiceName );
938 if ( hEventSource )
939 {
940 lpszStrings[0] = g_sServiceName;
941 lpszStrings[1] = sBuf;
942
943 WORD eType = EVENTLOG_INFORMATION_TYPE;
944 switch ( eLevel )
945 {
946 case SPH_LOG_FATAL: eType = EVENTLOG_ERROR_TYPE; break;
947 case SPH_LOG_WARNING: eType = EVENTLOG_WARNING_TYPE; break;
948 case SPH_LOG_INFO: eType = EVENTLOG_INFORMATION_TYPE; break;
949 }
950
951 ReportEvent ( hEventSource, // event log handle
952 eType, // event type
953 0, // event category
954 0, // event identifier
955 NULL, // no security identifier
956 2, // size of lpszStrings array
957 0, // no binary data
958 lpszStrings, // array of strings
959 NULL ); // no binary data
960
961 DeregisterEventSource ( hEventSource );
962 }
963
964 } else
965 #endif
966 {
967 strcat ( sBuf, "\n" ); // NOLINT
968
969 lseek ( g_iLogFile, 0, SEEK_END );
970 if ( g_bLogTty )
971 sphWrite ( g_iLogFile, sTtyBuf, strlen(sTtyBuf) );
972 else
973 sphWrite ( g_iLogFile, sBuf, strlen(sBuf) );
974
975 if ( g_bLogStdout && g_iLogFile!=STDOUT_FILENO )
976 sphWrite ( STDOUT_FILENO, sTtyBuf, strlen(sTtyBuf) );
977 }
978 }
979
980
981 /// log entry (with log levels, dupe catching, etc)
982 /// call with NULL format for dupe flushing
sphLog(ESphLogLevel eLevel,const char * sFmt,va_list ap)983 void sphLog ( ESphLogLevel eLevel, const char * sFmt, va_list ap )
984 {
985 // dupe catcher state
986 static const int FLUSH_THRESH_TIME = 1000000; // in microseconds
987 static const int FLUSH_THRESH_COUNT = 100;
988
989 static ESphLogLevel eLastLevel = SPH_LOG_INFO;
990 static DWORD uLastEntry = 0;
991 static int64_t tmLastStamp = -1000000-FLUSH_THRESH_TIME;
992 static int iLastRepeats = 0;
993
994 // only if we can
995 if ( sFmt && eLevel>g_eLogLevel )
996 return;
997
998 #if USE_SYSLOG
999 if ( g_bLogSyslog && sFmt )
1000 {
1001 const int levels[] = { LOG_EMERG, LOG_WARNING, LOG_INFO, LOG_DEBUG, LOG_DEBUG, LOG_DEBUG };
1002 vsyslog ( levels[eLevel], sFmt, ap );
1003 return;
1004 }
1005 #endif
1006
1007 if ( g_iLogFile<0 && !g_bService )
1008 return;
1009
1010 // format the banner
1011 char sTimeBuf[128];
1012 sphFormatCurrentTime ( sTimeBuf, sizeof(sTimeBuf) );
1013
1014 const char * sBanner = "";
1015 if ( sFmt==NULL ) eLevel = eLastLevel;
1016 if ( eLevel==SPH_LOG_FATAL ) sBanner = "FATAL: ";
1017 if ( eLevel==SPH_LOG_WARNING ) sBanner = "WARNING: ";
1018 if ( eLevel>=SPH_LOG_DEBUG ) sBanner = "DEBUG: ";
1019
1020 char sBuf [ 1024 ];
1021 snprintf ( sBuf, sizeof(sBuf)-1, "[%s] [%5d] ", sTimeBuf, (int)getpid() );
1022
1023 char * sTtyBuf = sBuf + strlen(sBuf);
1024 strncpy ( sTtyBuf, sBanner, 32 ); // 32 is arbitrary; just something that is enough and keeps lint happy
1025
1026 int iLen = strlen(sBuf);
1027
1028 // format the message
1029 if ( sFmt )
1030 vsnprintf ( sBuf+iLen, sizeof(sBuf)-iLen-1, sFmt, ap );
1031
1032 // catch dupes
1033 DWORD uEntry = sFmt ? sphCRC32 ( (const BYTE*)( sBuf+iLen ) ) : 0;
1034 int64_t tmNow = sphMicroTimer();
1035
1036 // accumulate while possible
1037 if ( sFmt && eLevel==eLastLevel && uEntry==uLastEntry && iLastRepeats<FLUSH_THRESH_COUNT && tmNow<tmLastStamp+FLUSH_THRESH_TIME )
1038 {
1039 tmLastStamp = tmNow;
1040 iLastRepeats++;
1041 return;
1042 }
1043
1044 // flush if needed
1045 if ( iLastRepeats!=0 && ( sFmt || tmNow>=tmLastStamp+FLUSH_THRESH_TIME ) )
1046 {
1047 // flush if we actually have something to flush, and
1048 // case 1: got a message we can't accumulate
1049 // case 2: got a periodic flush and been otherwise idle for a thresh period
1050 char sLast[256];
1051 strncpy ( sLast, sBuf, iLen );
1052 snprintf ( sLast+iLen, sizeof(sLast)-iLen, "last message repeated %d times", iLastRepeats );
1053 sphLogEntry ( eLastLevel, sLast, sLast + ( sTtyBuf-sBuf ) );
1054
1055 tmLastStamp = tmNow;
1056 iLastRepeats = 0;
1057 eLastLevel = SPH_LOG_INFO;
1058 uLastEntry = 0;
1059 }
1060
1061 // was that a flush-only call?
1062 if ( !sFmt )
1063 return;
1064
1065 tmLastStamp = tmNow;
1066 iLastRepeats = 0;
1067 eLastLevel = eLevel;
1068 uLastEntry = uEntry;
1069
1070 // do the logging
1071 sphLogEntry ( eLevel, sBuf, sTtyBuf );
1072 }
1073
1074 void sphFatal ( const char * sFmt, ... ) __attribute__ ( ( format ( printf, 1, 2 ) ) );
sphFatal(const char * sFmt,...)1075 void sphFatal ( const char * sFmt, ... )
1076 {
1077 va_list ap;
1078 va_start ( ap, sFmt );
1079 sphLog ( SPH_LOG_FATAL, sFmt, ap );
1080 va_end ( ap );
1081 Shutdown ();
1082 exit ( 1 );
1083 }
1084
1085 #if !USE_WINDOWS
GetNamedPipeName(int iPid)1086 static CSphString GetNamedPipeName ( int iPid )
1087 {
1088 CSphString sRes;
1089 sRes.SetSprintf ( "/tmp/searchd_%d", iPid );
1090 return sRes;
1091 }
1092 #endif
1093
LogWarning(const char * sWarning)1094 void LogWarning ( const char * sWarning )
1095 {
1096 sphWarning ( "%s", sWarning );
1097 }
1098
1099 /////////////////////////////////////////////////////////////////////////////
1100
CmpString(const CSphString & a,const CSphString & b)1101 static int CmpString ( const CSphString & a, const CSphString & b )
1102 {
1103 if ( !a.cstr() && !b.cstr() )
1104 return 0;
1105
1106 if ( !a.cstr() || !b.cstr() )
1107 return a.cstr() ? -1 : 1;
1108
1109 return strcmp ( a.cstr(), b.cstr() );
1110 }
1111
1112 struct SearchFailure_t
1113 {
1114 public:
1115 CSphString m_sIndex; ///< searched index name
1116 CSphString m_sError; ///< search error message
1117
1118 public:
SearchFailure_tSearchFailure_t1119 SearchFailure_t () {}
1120
1121 public:
operator ==SearchFailure_t1122 bool operator == ( const SearchFailure_t & r ) const
1123 {
1124 return m_sIndex==r.m_sIndex && m_sError==r.m_sError;
1125 }
1126
operator <SearchFailure_t1127 bool operator < ( const SearchFailure_t & r ) const
1128 {
1129 int iRes = CmpString ( m_sError.cstr(), r.m_sError.cstr() );
1130 if ( !iRes )
1131 iRes = CmpString ( m_sIndex.cstr(), r.m_sIndex.cstr() );
1132 return iRes<0;
1133 }
1134
operator =SearchFailure_t1135 const SearchFailure_t & operator = ( const SearchFailure_t & r )
1136 {
1137 if ( this!=&r )
1138 {
1139 m_sIndex = r.m_sIndex;
1140 m_sError = r.m_sError;
1141 }
1142 return *this;
1143 }
1144 };
1145
1146
1147 class SearchFailuresLog_c
1148 {
1149 protected:
1150 CSphVector<SearchFailure_t> m_dLog;
1151
1152 public:
Submit(const char * sIndex,const char * sError)1153 void Submit ( const char * sIndex, const char * sError )
1154 {
1155 SearchFailure_t & tEntry = m_dLog.Add ();
1156 tEntry.m_sIndex = sIndex;
1157 tEntry.m_sError = sError;
1158 }
1159
SubmitEx(const char * sIndex,const char * sTemplate,...)1160 void SubmitEx ( const char * sIndex, const char * sTemplate, ... ) __attribute__ ( ( format ( printf, 3, 4 ) ) )
1161 {
1162 SearchFailure_t & tEntry = m_dLog.Add ();
1163 va_list ap;
1164 va_start ( ap, sTemplate );
1165 tEntry.m_sIndex = sIndex;
1166 tEntry.m_sError.SetSprintfVa ( sTemplate, ap );
1167 va_end ( ap );
1168 }
1169
1170 public:
IsEmpty()1171 bool IsEmpty ()
1172 {
1173 return m_dLog.GetLength()==0;
1174 }
1175
BuildReport(CSphStringBuilder & sReport)1176 void BuildReport ( CSphStringBuilder & sReport )
1177 {
1178 if ( IsEmpty() )
1179 return;
1180
1181 // collapse same messages
1182 m_dLog.Uniq ();
1183 int iSpanStart = 0;
1184
1185 for ( int i=1; i<=m_dLog.GetLength(); i++ )
1186 {
1187 // keep scanning while error text is the same
1188 if ( i!=m_dLog.GetLength() )
1189 if ( m_dLog[i].m_sError==m_dLog[i-1].m_sError )
1190 continue;
1191
1192 // build current span
1193 CSphStringBuilder sSpan;
1194 if ( iSpanStart )
1195 sSpan += "; ";
1196 sSpan += "index ";
1197 for ( int j=iSpanStart; j<i; j++ )
1198 {
1199 if ( j!=iSpanStart )
1200 sSpan += ",";
1201 sSpan += m_dLog[j].m_sIndex.cstr();
1202 }
1203 sSpan += ": ";
1204 sSpan += m_dLog[iSpanStart].m_sError.cstr();
1205
1206 // flush current span
1207 sReport += sSpan.cstr();
1208
1209 // done
1210 iSpanStart = i;
1211 }
1212 }
1213 };
1214
1215 /////////////////////////////////////////////////////////////////////////////
1216 // SIGNAL HANDLERS
1217 /////////////////////////////////////////////////////////////////////////////
1218
1219
1220 #if !USE_WINDOWS
UpdateAliveChildrenList(CSphVector<int> & dChildren)1221 static void UpdateAliveChildrenList ( CSphVector<int> & dChildren )
1222 {
1223 ARRAY_FOREACH ( i, dChildren )
1224 {
1225 int iPID = dChildren[i];
1226 int iStatus = 0;
1227 if ( iPID>0 && waitpid ( iPID, &iStatus, WNOHANG )==iPID && ( WIFEXITED ( iStatus ) || WIFSIGNALED ( iStatus ) ) )
1228 iPID = 0;
1229
1230 if ( iPID<=0 )
1231 dChildren.RemoveFast ( i-- );
1232 }
1233 }
1234 #endif
1235
1236
SaveIndexes()1237 static bool SaveIndexes ()
1238 {
1239 bool bAllSaved = true;
1240 for ( IndexHashIterator_c it ( g_pIndexes ); it.Next(); )
1241 {
1242 const ServedIndex_t & tServed = it.Get();
1243 if ( !tServed.m_bEnabled )
1244 continue;
1245
1246 tServed.ReadLock();
1247 if ( !tServed.m_pIndex->SaveAttributes () )
1248 {
1249 sphWarning ( "index %s: attrs save failed: %s", it.GetKey().cstr(), tServed.m_pIndex->GetLastError().cstr() );
1250 bAllSaved = false;
1251 }
1252 tServed.Unlock();
1253 }
1254 return bAllSaved;
1255 }
1256
1257
Shutdown()1258 void Shutdown ()
1259 {
1260 bool bAttrsSaveOk = true;
1261 #if !USE_WINDOWS
1262 int fdStopwait = -1;
1263 #endif
1264 // some head-only shutdown procedures
1265 if ( g_bHeadDaemon )
1266 {
1267 if ( !g_bDaemonAtShutdown.IsEmpty() )
1268 {
1269 *g_bDaemonAtShutdown.GetWritePtr() = 1;
1270 }
1271
1272 #if !USE_WINDOWS
1273 // stopwait handshake
1274 CSphString sPipeName = GetNamedPipeName ( getpid() );
1275 fdStopwait = ::open ( sPipeName.cstr(), O_WRONLY | O_NONBLOCK );
1276 if ( fdStopwait>=0 )
1277 {
1278 DWORD uHandshakeOk = 0;
1279 int iDummy; // to avoid gcc unused result warning
1280 iDummy = ::write ( fdStopwait, &uHandshakeOk, sizeof(DWORD) );
1281 iDummy++; // to avoid gcc set but not used variable warning
1282 }
1283 #endif
1284
1285 const int iShutWaitPeriod = 3000000;
1286
1287 if ( g_eWorkers==MPM_THREADS )
1288 {
1289 // tell flush-rt thread to shutdown, and wait until it does
1290 g_bRtFlushShutdown = true;
1291 sphThreadJoin ( &g_tRtFlushThread );
1292
1293 // tell rotation thread to shutdown, and wait until it does
1294 g_bRotateShutdown = true;
1295 if ( g_bSeamlessRotate )
1296 {
1297 sphThreadJoin ( &g_tRotateThread );
1298 }
1299
1300 int64_t tmShutStarted = sphMicroTimer();
1301 // stop search threads; up to 3 seconds long
1302 while ( g_dThd.GetLength() > 0 && ( sphMicroTimer()-tmShutStarted )<iShutWaitPeriod )
1303 sphSleepMsec ( 50 );
1304
1305 g_tThdMutex.Lock();
1306 g_dThd.Reset();
1307 g_tThdMutex.Unlock();
1308 }
1309
1310
1311 #if !USE_WINDOWS
1312 if ( g_eWorkers==MPM_FORK || g_eWorkers==MPM_PREFORK )
1313 {
1314 // in *forked mode, explicitly kill all children
1315 ARRAY_FOREACH ( i, g_dChildren )
1316 {
1317 sphLogDebug ( "killing child %d", g_dChildren[i] );
1318 kill ( g_dChildren[i], SIGTERM );
1319 }
1320
1321 int64_t tmShutStarted = sphMicroTimer();
1322 // stop search children; up to 3 seconds long
1323 while ( g_dChildren.GetLength()>0 && ( sphMicroTimer()-tmShutStarted )<iShutWaitPeriod )
1324 {
1325 UpdateAliveChildrenList ( g_dChildren );
1326 sphSleepMsec ( 50 );
1327 }
1328
1329 if ( g_dChildren.GetLength() )
1330 {
1331 ARRAY_FOREACH ( i, g_dChildren )
1332 kill ( g_dChildren[i], SIGKILL );
1333
1334 sphSleepMsec ( 100 );
1335 UpdateAliveChildrenList ( g_dChildren );
1336 if ( g_dChildren.GetLength() )
1337 sphWarning ( "there are still %d alive children", g_dChildren.GetLength() );
1338 }
1339 }
1340 #endif
1341
1342 // save attribute updates for all local indexes
1343 bAttrsSaveOk = SaveIndexes();
1344
1345 // unlock indexes and release locks if needed
1346 for ( IndexHashIterator_c it ( g_pIndexes ); it.Next(); )
1347 if ( it.Get().m_pIndex )
1348 it.Get().m_pIndex->Unlock();
1349 g_pIndexes->Reset();
1350
1351 // clear shut down of rt indexes + binlog
1352 SafeDelete ( g_pIndexes );
1353 sphDoneIOStats();
1354 sphRTDone();
1355
1356 sphShutdownWordforms ();
1357 }
1358
1359 ARRAY_FOREACH ( i, g_dListeners )
1360 if ( g_dListeners[i].m_iSock>=0 )
1361 sphSockClose ( g_dListeners[i].m_iSock );
1362
1363 #if USE_WINDOWS
1364 CloseHandle ( g_hPipe );
1365 #else
1366 if ( g_bHeadDaemon && fdStopwait>=0 )
1367 {
1368 DWORD uStatus = bAttrsSaveOk;
1369 int iDummy; // to avoid gcc unused result warning
1370 iDummy = ::write ( fdStopwait, &uStatus, sizeof(DWORD) );
1371 iDummy++; // to avoid gcc set but not used variable warning
1372 ::close ( fdStopwait );
1373 }
1374 #endif
1375
1376 // remove pid
1377 if ( g_bHeadDaemon && g_sPidFile )
1378 {
1379 ::close ( g_iPidFD );
1380 ::unlink ( g_sPidFile );
1381 }
1382
1383 if ( g_bHeadDaemon )
1384 sphInfo ( "shutdown complete" );
1385
1386 if ( g_bHeadDaemon )
1387 {
1388 SphCrashLogger_c::Done();
1389 sphThreadDone ( g_iLogFile );
1390 }
1391 }
1392
1393 #if !USE_WINDOWS
sighup(int)1394 void sighup ( int )
1395 {
1396 g_bGotSighup = 1;
1397 }
1398
1399
sigterm(int)1400 void sigterm ( int )
1401 {
1402 // tricky bit
1403 // we can't call exit() here because malloc()/free() are not re-entrant
1404 // we could call _exit() but let's try to die gracefully on TERM
1405 // and let signal sender wait and send KILL as needed
1406 g_bGotSigterm = 1;
1407 sphInterruptNow();
1408 }
1409
1410
sigchld(int)1411 void sigchld ( int )
1412 {
1413 g_bGotSigchld = 1;
1414 }
1415
1416
sigusr1(int)1417 void sigusr1 ( int )
1418 {
1419 g_bGotSigusr1 = 1;
1420 }
1421 #endif // !USE_WINDOWS
1422
1423
1424 struct QueryCopyState_t
1425 {
1426 BYTE * m_pDst;
1427 BYTE * m_pDstEnd;
1428 const BYTE * m_pSrc;
1429 const BYTE * m_pSrcEnd;
1430 };
1431
1432 // crash query handler
1433 static const int g_iQueryLineLen = 80;
1434 static const char g_dEncodeBase64[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
sphCopyEncodedBase64(QueryCopyState_t & tEnc)1435 bool sphCopyEncodedBase64 ( QueryCopyState_t & tEnc )
1436 {
1437 BYTE * pDst = tEnc.m_pDst;
1438 const BYTE * pDstBase = tEnc.m_pDst;
1439 const BYTE * pSrc = tEnc.m_pSrc;
1440 const BYTE * pDstEnd = tEnc.m_pDstEnd-5;
1441 const BYTE * pSrcEnd = tEnc.m_pSrcEnd-3;
1442
1443 while ( pDst<=pDstEnd && pSrc<=pSrcEnd )
1444 {
1445 // put line delimiter at max line length
1446 if ( ( ( pDst-pDstBase ) % g_iQueryLineLen )>( ( pDst-pDstBase+4 ) % g_iQueryLineLen ) )
1447 *pDst++ = '\n';
1448
1449 // Convert to big endian
1450 DWORD uSrc = ( pSrc[0] << 16 ) | ( pSrc[1] << 8 ) | ( pSrc[2] );
1451 pSrc += 3;
1452
1453 *pDst++ = g_dEncodeBase64 [ ( uSrc & 0x00FC0000 ) >> 18 ];
1454 *pDst++ = g_dEncodeBase64 [ ( uSrc & 0x0003F000 ) >> 12 ];
1455 *pDst++ = g_dEncodeBase64 [ ( uSrc & 0x00000FC0 ) >> 6 ];
1456 *pDst++ = g_dEncodeBase64 [ ( uSrc & 0x0000003F ) ];
1457 }
1458
1459 // there is a tail in source data and a room for it at destination buffer
1460 if ( pSrc<tEnc.m_pSrcEnd && ( tEnc.m_pSrcEnd-pSrc<3 ) && ( pDst<=pDstEnd-4 ) )
1461 {
1462 int iLeft = ( tEnc.m_pSrcEnd - pSrc ) % 3;
1463 if ( iLeft==1 )
1464 {
1465 DWORD uSrc = pSrc[0]<<16;
1466 pSrc += 1;
1467 *pDst++ = g_dEncodeBase64 [ ( uSrc & 0x00FC0000 ) >> 18 ];
1468 *pDst++ = g_dEncodeBase64 [ ( uSrc & 0x0003F000 ) >> 12 ];
1469 *pDst++ = '=';
1470 *pDst++ = '=';
1471 } else if ( iLeft==2 )
1472 {
1473 DWORD uSrc = ( pSrc[0]<<16 ) | ( pSrc[1] << 8 );
1474 pSrc += 2;
1475 *pDst++ = g_dEncodeBase64 [ ( uSrc & 0x00FC0000 ) >> 18 ];
1476 *pDst++ = g_dEncodeBase64 [ ( uSrc & 0x0003F000 ) >> 12 ];
1477 *pDst++ = g_dEncodeBase64 [ ( uSrc & 0x00000FC0 ) >> 6 ];
1478 *pDst++ = '=';
1479 }
1480 }
1481
1482 tEnc.m_pDst = pDst;
1483 tEnc.m_pSrc = pSrc;
1484
1485 return ( tEnc.m_pSrc<tEnc.m_pSrcEnd );
1486 }
1487
sphCopySphinxQL(QueryCopyState_t & tState)1488 static bool sphCopySphinxQL ( QueryCopyState_t & tState )
1489 {
1490 BYTE * pDst = tState.m_pDst;
1491 const BYTE * pSrc = tState.m_pSrc;
1492 BYTE * pNextLine = pDst+g_iQueryLineLen;
1493
1494 while ( pDst<tState.m_pDstEnd && pSrc<tState.m_pSrcEnd )
1495 {
1496 if ( pDst>pNextLine && pDst+1<tState.m_pDstEnd && ( sphIsSpace ( *pSrc ) || *pSrc==',' ) )
1497 {
1498 *pDst++ = *pSrc++;
1499 *pDst++ = '\n';
1500 pNextLine = pDst + g_iQueryLineLen;
1501 } else
1502 {
1503 *pDst++ = *pSrc++;
1504 }
1505 }
1506
1507 tState.m_pDst = pDst;
1508 tState.m_pSrc = pSrc;
1509
1510 return ( tState.m_pSrc<tState.m_pSrcEnd );
1511 }
1512
1513 typedef bool CopyQuery_fn ( QueryCopyState_t & tState );
1514
1515 #define SPH_TIME_PID_MAX_SIZE 256
1516 const char g_sCrashedBannerAPI[] = "\n--- crashed SphinxAPI request dump ---\n";
1517 const char g_sCrashedBannerMySQL[] = "\n--- crashed SphinxQL request dump ---\n";
1518 const char g_sCrashedBannerTail[] = "\n--- request dump end ---\n";
1519 const char g_sMinidumpBanner[] = "minidump located at: ";
1520 const char g_sMemoryStatBanner[] = "\n--- memory statistics ---\n";
1521 static BYTE g_dCrashQueryBuff [4096];
1522 static char g_sCrashInfo [SPH_TIME_PID_MAX_SIZE] = "[][]\n";
1523 static int g_iCrashInfoLen = 0;
1524
1525 #if USE_WINDOWS
1526 static char g_sMinidump[SPH_TIME_PID_MAX_SIZE] = "";
1527 #endif
1528
1529 CrashQuery_t SphCrashLogger_c::m_tForkQuery = CrashQuery_t();
1530 SphThreadKey_t SphCrashLogger_c::m_tLastQueryTLS = SphThreadKey_t ();
1531
Init()1532 void SphCrashLogger_c::Init ()
1533 {
1534 Verify ( sphThreadKeyCreate ( &m_tLastQueryTLS ) );
1535 }
1536
Done()1537 void SphCrashLogger_c::Done ()
1538 {
1539 sphThreadKeyDelete ( m_tLastQueryTLS );
1540 }
1541
1542
1543 #if !USE_WINDOWS
HandleCrash(int sig)1544 void SphCrashLogger_c::HandleCrash ( int sig )
1545 #else
1546 LONG WINAPI SphCrashLogger_c::HandleCrash ( EXCEPTION_POINTERS * pExc )
1547 #endif // !USE_WINDOWS
1548 {
1549 if ( g_iLogFile<0 )
1550 CRASH_EXIT;
1551
1552 // log [time][pid]
1553 lseek ( g_iLogFile, 0, SEEK_END );
1554 sphWrite ( g_iLogFile, g_sCrashInfo, g_iCrashInfoLen );
1555
1556 // log query
1557 CrashQuery_t tQuery = SphCrashLogger_c::GetQuery();
1558
1559 // request dump banner
1560 int iBannerLen = ( tQuery.m_bMySQL ? sizeof(g_sCrashedBannerMySQL) : sizeof(g_sCrashedBannerAPI) ) - 1;
1561 const char * pBanner = tQuery.m_bMySQL ? g_sCrashedBannerMySQL : g_sCrashedBannerAPI;
1562 sphWrite ( g_iLogFile, pBanner, iBannerLen );
1563
1564 // query
1565 if ( tQuery.m_iSize )
1566 {
1567 QueryCopyState_t tCopyState;
1568 tCopyState.m_pDst = g_dCrashQueryBuff;
1569 tCopyState.m_pDstEnd = g_dCrashQueryBuff + sizeof(g_dCrashQueryBuff);
1570 tCopyState.m_pSrc = tQuery.m_pQuery;
1571 tCopyState.m_pSrcEnd = tQuery.m_pQuery + tQuery.m_iSize;
1572
1573 CopyQuery_fn * pfnCopy = NULL;
1574 if ( !tQuery.m_bMySQL )
1575 {
1576 pfnCopy = &sphCopyEncodedBase64;
1577
1578 // should be power of 3 to seamlessly convert to BASE64
1579 BYTE dHeader[] = {
1580 (BYTE)( ( tQuery.m_uCMD>>8 ) & 0xff ),
1581 (BYTE)( tQuery.m_uCMD & 0xff ),
1582 (BYTE)( ( tQuery.m_uVer>>8 ) & 0xff ),
1583 (BYTE)( tQuery.m_uVer & 0xff ),
1584 (BYTE)( ( tQuery.m_iSize>>24 ) & 0xff ),
1585 (BYTE)( ( tQuery.m_iSize>>16 ) & 0xff ),
1586 (BYTE)( ( tQuery.m_iSize>>8 ) & 0xff ),
1587 (BYTE)( tQuery.m_iSize & 0xff ),
1588 *tQuery.m_pQuery
1589 };
1590
1591 QueryCopyState_t tHeaderState;
1592 tHeaderState.m_pDst = g_dCrashQueryBuff;
1593 tHeaderState.m_pDstEnd = g_dCrashQueryBuff + sizeof(g_dCrashQueryBuff);
1594 tHeaderState.m_pSrc = dHeader;
1595 tHeaderState.m_pSrcEnd = dHeader + sizeof(dHeader);
1596 pfnCopy ( tHeaderState );
1597 assert ( tHeaderState.m_pSrc==tHeaderState.m_pSrcEnd );
1598 tCopyState.m_pDst = tHeaderState.m_pDst;
1599 tCopyState.m_pSrc++;
1600 } else
1601 {
1602 pfnCopy = &sphCopySphinxQL;
1603 }
1604
1605 while ( pfnCopy ( tCopyState ) )
1606 {
1607 sphWrite ( g_iLogFile, g_dCrashQueryBuff, tCopyState.m_pDst-g_dCrashQueryBuff );
1608 tCopyState.m_pDst = g_dCrashQueryBuff; // reset the destination buffer
1609 }
1610 assert ( tCopyState.m_pSrc==tCopyState.m_pSrcEnd );
1611
1612 int iLeft = tCopyState.m_pDst-g_dCrashQueryBuff;
1613 if ( iLeft>0 )
1614 {
1615 sphWrite ( g_iLogFile, g_dCrashQueryBuff, iLeft );
1616 }
1617 }
1618
1619 // tail
1620 sphWrite ( g_iLogFile, g_sCrashedBannerTail, sizeof(g_sCrashedBannerTail)-1 );
1621
1622 sphSafeInfo ( g_iLogFile, "Sphinx " SPHINX_VERSION );
1623
1624 #if USE_WINDOWS
1625 // mini-dump reference
1626 int iMiniDumpLen = snprintf ( (char *)g_dCrashQueryBuff, sizeof(g_dCrashQueryBuff), "%s %s.%p.mdmp\n", g_sMinidumpBanner, g_sMinidump, tQuery.m_pQuery );
1627 sphWrite ( g_iLogFile, g_dCrashQueryBuff, iMiniDumpLen );
1628 snprintf ( (char *)g_dCrashQueryBuff, sizeof(g_dCrashQueryBuff), "%s.%p.mdmp", g_sMinidump, tQuery.m_pQuery );
1629 #endif
1630
1631 // log trace
1632 #if !USE_WINDOWS
1633 sphSafeInfo ( g_iLogFile, "Handling signal %d", sig );
1634 // print message to stdout during daemon start
1635 if ( g_bLogStdout && g_iLogFile!=STDOUT_FILENO )
1636 sphSafeInfo ( STDOUT_FILENO, "Crash!!! Handling signal %d", sig );
1637 sphBacktrace ( g_iLogFile, g_bSafeTrace );
1638 #else
1639 sphBacktrace ( pExc, (char *)g_dCrashQueryBuff );
1640 #endif
1641
1642 // threads table
1643 if ( g_eWorkers==MPM_THREADS )
1644 {
1645 // FIXME? should we try to lock threads table somehow?
1646 sphSafeInfo ( g_iLogFile, "--- %d active threads ---", g_dThd.GetLength() );
1647 ARRAY_FOREACH ( iThd, g_dThd )
1648 {
1649 ThdDesc_t * pThd = g_dThd[iThd];
1650 sphSafeInfo ( g_iLogFile, "thd %d, proto %s, state %s, command %s",
1651 iThd,
1652 g_dProtoNames[pThd->m_eProto],
1653 g_dThdStates[pThd->m_eThdState],
1654 pThd->m_sCommand ? pThd->m_sCommand : "-" );
1655 }
1656 }
1657
1658 // memory info
1659 #if SPH_ALLOCS_PROFILER
1660 sphWrite ( g_iLogFile, g_sMemoryStatBanner, sizeof ( g_sMemoryStatBanner )-1 );
1661 sphMemStatDump ( g_iLogFile );
1662 #endif
1663
1664 sphSafeInfo ( g_iLogFile, "------- CRASH DUMP END -------" );
1665
1666 CRASH_EXIT;
1667 }
1668
SetLastQuery(const CrashQuery_t & tQuery)1669 void SphCrashLogger_c::SetLastQuery ( const CrashQuery_t & tQuery )
1670 {
1671 m_tForkQuery = tQuery;
1672 SphCrashLogger_c * pCrashLogger = (SphCrashLogger_c *)sphThreadGet ( m_tLastQueryTLS );
1673 if ( pCrashLogger )
1674 {
1675 pCrashLogger->m_tQuery = tQuery;
1676 }
1677 }
1678
SetupTimePID()1679 void SphCrashLogger_c::SetupTimePID ()
1680 {
1681 char sTimeBuf[SPH_TIME_PID_MAX_SIZE];
1682 sphFormatCurrentTime ( sTimeBuf, sizeof(sTimeBuf) );
1683
1684 g_iCrashInfoLen = snprintf ( g_sCrashInfo, SPH_TIME_PID_MAX_SIZE-1, "------- FATAL: CRASH DUMP -------\n[%s] [%5d]\n", sTimeBuf, (int)getpid() );
1685 }
1686
SetupTLS()1687 void SphCrashLogger_c::SetupTLS ()
1688 {
1689 Verify ( sphThreadSet ( m_tLastQueryTLS, this ) );
1690 }
1691
GetQuery()1692 CrashQuery_t SphCrashLogger_c::GetQuery()
1693 {
1694 SphCrashLogger_c * pCrashLogger = (SphCrashLogger_c *)sphThreadGet ( m_tLastQueryTLS );
1695 return pCrashLogger ? pCrashLogger->m_tQuery : m_tForkQuery;
1696 }
1697
1698
SetSignalHandlers()1699 void SetSignalHandlers ()
1700 {
1701 SphCrashLogger_c::Init();
1702
1703 #if !USE_WINDOWS
1704 struct sigaction sa;
1705 sigfillset ( &sa.sa_mask );
1706 sa.sa_flags = SA_NOCLDSTOP;
1707
1708 bool bSignalsSet = false;
1709 for ( ;; )
1710 {
1711 sa.sa_handler = sigterm; if ( sigaction ( SIGTERM, &sa, NULL )!=0 ) break;
1712 sa.sa_handler = sigterm; if ( sigaction ( SIGINT, &sa, NULL )!=0 ) break;
1713 sa.sa_handler = sighup; if ( sigaction ( SIGHUP, &sa, NULL )!=0 ) break;
1714 sa.sa_handler = sigusr1; if ( sigaction ( SIGUSR1, &sa, NULL )!=0 ) break;
1715 sa.sa_handler = sigchld; if ( sigaction ( SIGCHLD, &sa, NULL )!=0 ) break;
1716 sa.sa_handler = SIG_IGN; if ( sigaction ( SIGPIPE, &sa, NULL )!=0 ) break;
1717
1718 sa.sa_flags |= SA_RESETHAND;
1719 sa.sa_handler = SphCrashLogger_c::HandleCrash; if ( sigaction ( SIGSEGV, &sa, NULL )!=0 ) break;
1720 sa.sa_handler = SphCrashLogger_c::HandleCrash; if ( sigaction ( SIGBUS, &sa, NULL )!=0 ) break;
1721 sa.sa_handler = SphCrashLogger_c::HandleCrash; if ( sigaction ( SIGABRT, &sa, NULL )!=0 ) break;
1722 sa.sa_handler = SphCrashLogger_c::HandleCrash; if ( sigaction ( SIGILL, &sa, NULL )!=0 ) break;
1723 sa.sa_handler = SphCrashLogger_c::HandleCrash; if ( sigaction ( SIGFPE, &sa, NULL )!=0 ) break;
1724
1725 bSignalsSet = true;
1726 break;
1727 }
1728 if ( !bSignalsSet )
1729 sphFatal ( "sigaction(): %s", strerror(errno) );
1730 #else
1731 snprintf ( g_sMinidump, SPH_TIME_PID_MAX_SIZE-1, "%s.%d", g_sPidFile ? g_sPidFile : "", (int)getpid() );
1732 SetUnhandledExceptionFilter ( SphCrashLogger_c::HandleCrash );
1733 #endif
1734 }
1735
1736
1737 /////////////////////////////////////////////////////////////////////////////
1738 // NETWORK STUFF
1739 /////////////////////////////////////////////////////////////////////////////
1740
1741 const int WIN32_PIPE_BUFSIZE = 32;
1742
1743
1744 #if USE_WINDOWS
1745
1746 /// on Windows, the wrapper just prevents the warnings
sphFDSet(int fd,fd_set * fdset)1747 void sphFDSet ( int fd, fd_set * fdset )
1748 {
1749 #pragma warning(disable:4127) // conditional expr is const
1750 #pragma warning(disable:4389) // signed/unsigned mismatch
1751
1752 FD_SET ( fd, fdset );
1753
1754 #pragma warning(default:4127) // conditional expr is const
1755 #pragma warning(default:4389) // signed/unsigned mismatch
1756 }
1757
1758 #else // !USE_WINDOWS
1759
1760 #define SPH_FDSET_OVERFLOW(_fd) ( (_fd)<0 || (_fd)>=(int)FD_SETSIZE )
1761
1762 /// on UNIX, we also check that the descript won't corrupt the stack
sphFDSet(int fd,fd_set * set)1763 void sphFDSet ( int fd, fd_set * set )
1764 {
1765 if ( SPH_FDSET_OVERFLOW(fd) )
1766 sphFatal ( "sphFDSet() failed fd=%d, FD_SETSIZE=%d", fd, FD_SETSIZE );
1767 else
1768 FD_SET ( fd, set );
1769 }
1770
1771 #endif // USE_WINDOWS
1772
1773
1774 #if USE_WINDOWS
sphSockError(int iErr=0)1775 const char * sphSockError ( int iErr=0 )
1776 {
1777 if ( iErr==0 )
1778 iErr = WSAGetLastError ();
1779
1780 static char sBuf [ 256 ];
1781 _snprintf ( sBuf, sizeof(sBuf), "WSA error %d", iErr );
1782 return sBuf;
1783 }
1784 #else
sphSockError(int=0)1785 const char * sphSockError ( int =0 )
1786 {
1787 return strerror ( errno );
1788 }
1789 #endif
1790
1791
sphSockGetErrno()1792 int sphSockGetErrno ()
1793 {
1794 #if USE_WINDOWS
1795 return WSAGetLastError();
1796 #else
1797 return errno;
1798 #endif
1799 }
1800
1801
sphSockSetErrno(int iErr)1802 void sphSockSetErrno ( int iErr )
1803 {
1804 #if USE_WINDOWS
1805 WSASetLastError ( iErr );
1806 #else
1807 errno = iErr;
1808 #endif
1809 }
1810
1811
sphSockPeekErrno()1812 int sphSockPeekErrno ()
1813 {
1814 int iRes = sphSockGetErrno();
1815 sphSockSetErrno ( iRes );
1816 return iRes;
1817 }
1818
1819
1820 /// formats IP address given in network byte order into sBuffer
1821 /// returns the buffer
sphFormatIP(char * sBuffer,int iBufferSize,DWORD uAddress)1822 char * sphFormatIP ( char * sBuffer, int iBufferSize, DWORD uAddress )
1823 {
1824 const BYTE *a = (const BYTE *)&uAddress;
1825 snprintf ( sBuffer, iBufferSize, "%u.%u.%u.%u", a[0], a[1], a[2], a[3] );
1826 return sBuffer;
1827 }
1828
1829
1830 static const bool GETADDR_STRICT = true; ///< strict check, will die with sphFatal() on failure
1831
sphGetAddress(const char * sHost,bool bFatal=false)1832 DWORD sphGetAddress ( const char * sHost, bool bFatal=false )
1833 {
1834 struct hostent * pHost = gethostbyname ( sHost );
1835
1836 if ( pHost==NULL || pHost->h_addrtype!=AF_INET )
1837 {
1838 if ( bFatal )
1839 sphFatal ( "no AF_INET address found for: %s", sHost );
1840 return 0;
1841 }
1842
1843 struct in_addr ** ppAddrs = (struct in_addr **)pHost->h_addr_list;
1844 assert ( ppAddrs[0] );
1845
1846 assert ( sizeof(DWORD)==pHost->h_length );
1847 DWORD uAddr;
1848 memcpy ( &uAddr, ppAddrs[0], sizeof(DWORD) );
1849
1850 if ( ppAddrs[1] )
1851 {
1852 char sBuf [ SPH_ADDRESS_SIZE ];
1853 sphWarning ( "multiple addresses found for '%s', using the first one (ip=%s)",
1854 sHost, sphFormatIP ( sBuf, sizeof(sBuf), uAddr ) );
1855 }
1856
1857 return uAddr;
1858 }
1859
1860
1861 #if !USE_WINDOWS
sphCreateUnixSocket(const char * sPath)1862 int sphCreateUnixSocket ( const char * sPath )
1863 {
1864 static struct sockaddr_un uaddr;
1865 size_t len = strlen ( sPath );
1866
1867 if ( len + 1 > sizeof( uaddr.sun_path ) )
1868 sphFatal ( "UNIX socket path is too long (len=%d)", (int)len );
1869
1870 sphInfo ( "listening on UNIX socket %s", sPath );
1871
1872 memset ( &uaddr, 0, sizeof(uaddr) );
1873 uaddr.sun_family = AF_UNIX;
1874 memcpy ( uaddr.sun_path, sPath, len + 1 );
1875
1876 int iSock = socket ( AF_UNIX, SOCK_STREAM, 0 );
1877 if ( iSock==-1 )
1878 sphFatal ( "failed to create UNIX socket: %s", sphSockError() );
1879
1880 if ( unlink ( sPath )==-1 )
1881 {
1882 if ( errno!=ENOENT )
1883 sphFatal ( "unlink() on UNIX socket file failed: %s", sphSockError() );
1884 }
1885
1886 int iMask = umask ( 0 );
1887 if ( bind ( iSock, (struct sockaddr *)&uaddr, sizeof(uaddr) )!=0 )
1888 sphFatal ( "bind() on UNIX socket failed: %s", sphSockError() );
1889 umask ( iMask );
1890
1891 return iSock;
1892 }
1893 #endif // !USE_WINDOWS
1894
1895
sphCreateInetSocket(DWORD uAddr,int iPort)1896 int sphCreateInetSocket ( DWORD uAddr, int iPort )
1897 {
1898 char sAddress[SPH_ADDRESS_SIZE];
1899 sphFormatIP ( sAddress, SPH_ADDRESS_SIZE, uAddr );
1900
1901 if ( uAddr==htonl ( INADDR_ANY ) )
1902 sphInfo ( "listening on all interfaces, port=%d", iPort );
1903 else
1904 sphInfo ( "listening on %s:%d", sAddress, iPort );
1905
1906 static struct sockaddr_in iaddr;
1907 memset ( &iaddr, 0, sizeof(iaddr) );
1908 iaddr.sin_family = AF_INET;
1909 iaddr.sin_addr.s_addr = uAddr;
1910 iaddr.sin_port = htons ( (short)iPort );
1911
1912 int iSock = socket ( AF_INET, SOCK_STREAM, 0 );
1913 if ( iSock==-1 )
1914 sphFatal ( "failed to create TCP socket: %s", sphSockError() );
1915
1916 int iOn = 1;
1917 if ( setsockopt ( iSock, SOL_SOCKET, SO_REUSEADDR, (char*)&iOn, sizeof(iOn) ) )
1918 sphFatal ( "setsockopt() failed: %s", sphSockError() );
1919
1920 int iTries = 12;
1921 int iRes;
1922 do
1923 {
1924 iRes = bind ( iSock, (struct sockaddr *)&iaddr, sizeof(iaddr) );
1925 if ( iRes==0 )
1926 break;
1927
1928 sphInfo ( "bind() failed on %s, retrying...", sAddress );
1929 sphSleepMsec ( 3000 );
1930 } while ( --iTries>0 );
1931 if ( iRes )
1932 sphFatal ( "bind() failed on %s: %s", sAddress, sphSockError() );
1933
1934 return iSock;
1935 }
1936
1937
IsPortInRange(int iPort)1938 inline bool IsPortInRange ( int iPort )
1939 {
1940 return ( iPort>0 ) && ( iPort<=0xFFFF );
1941 }
1942
1943
CheckPort(int iPort)1944 void CheckPort ( int iPort )
1945 {
1946 if ( !IsPortInRange(iPort) )
1947 sphFatal ( "port %d is out of range", iPort );
1948 }
1949
1950
ProtoByName(const CSphString & sProto)1951 ProtocolType_e ProtoByName ( const CSphString & sProto )
1952 {
1953 if ( sProto=="sphinx" ) return PROTO_SPHINX;
1954 else if ( sProto=="mysql41" ) return PROTO_MYSQL41;
1955
1956 sphFatal ( "unknown listen protocol type '%s'", sProto.cstr() ? sProto.cstr() : "(NULL)" );
1957
1958 // funny magic
1959 // MSVC -O2 whines about unreachable code
1960 // everyone else whines about missing return value
1961 #if !(USE_WINDOWS && defined(NDEBUG))
1962 return PROTO_SPHINX;
1963 #endif
1964 }
1965
1966
1967 struct ListenerDesc_t
1968 {
1969 ProtocolType_e m_eProto;
1970 CSphString m_sUnix;
1971 DWORD m_uIP;
1972 int m_iPort;
1973 };
1974
1975
ParseListener(const char * sSpec)1976 ListenerDesc_t ParseListener ( const char * sSpec )
1977 {
1978 ListenerDesc_t tRes;
1979 tRes.m_eProto = PROTO_SPHINX;
1980 tRes.m_sUnix = "";
1981 tRes.m_uIP = htonl ( INADDR_ANY );
1982 tRes.m_iPort = SPHINXAPI_PORT;
1983
1984 // split by colon
1985 int iParts = 0;
1986 CSphString sParts[3];
1987
1988 const char * sPart = sSpec;
1989 for ( const char * p = sSpec; ; p++ )
1990 if ( *p=='\0' || *p==':' )
1991 {
1992 if ( iParts==3 )
1993 sphFatal ( "invalid listen format (too many fields)" );
1994
1995 sParts[iParts++].SetBinary ( sPart, p-sPart );
1996 if ( !*p )
1997 break; // bail out on zero
1998
1999 sPart = p+1;
2000 }
2001 assert ( iParts>=1 && iParts<=3 );
2002
2003 // handle UNIX socket case
2004 // might be either name on itself (1 part), or name+protocol (2 parts)
2005 sPart = sParts[0].cstr();
2006 if ( sPart[0]=='/' )
2007 {
2008 if ( iParts>2 )
2009 sphFatal ( "invalid listen format (too many fields)" );
2010
2011 if ( iParts==2 )
2012 tRes.m_eProto = ProtoByName ( sParts[1] );
2013
2014 #if USE_WINDOWS
2015 sphFatal ( "UNIX sockets are not supported on Windows" );
2016 #else
2017 tRes.m_sUnix = sPart;
2018 return tRes;
2019 #endif
2020 }
2021
2022 // check if it all starts with a valid port number
2023 sPart = sParts[0].cstr();
2024 int iLen = strlen(sPart);
2025
2026 bool bAllDigits = true;
2027 for ( int i=0; i<iLen && bAllDigits; i++ )
2028 if ( !isdigit ( sPart[i] ) )
2029 bAllDigits = false;
2030
2031 int iPort = 0;
2032 if ( bAllDigits && iLen<=5 )
2033 {
2034 iPort = atol(sPart);
2035 CheckPort ( iPort ); // lets forbid ambiguous magic like 0:sphinx or 99999:mysql41
2036 }
2037
2038 // handle TCP port case
2039 // one part. might be either port name, or host name, or UNIX socket name
2040 if ( iParts==1 )
2041 {
2042 if ( iPort )
2043 {
2044 // port name on itself
2045 tRes.m_uIP = htonl ( INADDR_ANY );
2046 tRes.m_iPort = iPort;
2047 } else
2048 {
2049 // host name on itself
2050 tRes.m_uIP = sphGetAddress ( sSpec, GETADDR_STRICT );
2051 tRes.m_iPort = SPHINXAPI_PORT;
2052 }
2053 return tRes;
2054 }
2055
2056 // two or three parts
2057 if ( iPort )
2058 {
2059 // 1st part is a valid port number; must be port:proto
2060 if ( iParts!=2 )
2061 sphFatal ( "invalid listen format (expected port:proto, got extra trailing part in listen=%s)", sSpec );
2062
2063 tRes.m_uIP = htonl ( INADDR_ANY );
2064 tRes.m_iPort = iPort;
2065 tRes.m_eProto = ProtoByName ( sParts[1] );
2066
2067 } else
2068 {
2069 // 1st part must be a host name; must be host:port[:proto]
2070 if ( iParts==3 )
2071 tRes.m_eProto = ProtoByName ( sParts[2] );
2072
2073 tRes.m_iPort = atol ( sParts[1].cstr() );
2074 CheckPort ( tRes.m_iPort );
2075
2076 tRes.m_uIP = sParts[0].IsEmpty()
2077 ? htonl ( INADDR_ANY )
2078 : sphGetAddress ( sParts[0].cstr(), GETADDR_STRICT );
2079 }
2080 return tRes;
2081 }
2082
2083
AddListener(const CSphString & sListen)2084 void AddListener ( const CSphString & sListen )
2085 {
2086 ListenerDesc_t tDesc = ParseListener ( sListen.cstr() );
2087
2088 Listener_t tListener;
2089 tListener.m_eProto = tDesc.m_eProto;
2090
2091 #if !USE_WINDOWS
2092 if ( !tDesc.m_sUnix.IsEmpty() )
2093 tListener.m_iSock = sphCreateUnixSocket ( tDesc.m_sUnix.cstr() );
2094 else
2095 #endif
2096 tListener.m_iSock = sphCreateInetSocket ( tDesc.m_uIP, tDesc.m_iPort );
2097
2098 g_dListeners.Add ( tListener );
2099 }
2100
2101
sphSetSockNB(int iSock)2102 int sphSetSockNB ( int iSock )
2103 {
2104 #if USE_WINDOWS
2105 u_long uMode = 1;
2106 return ioctlsocket ( iSock, FIONBIO, &uMode );
2107 #else
2108 return fcntl ( iSock, F_SETFL, O_NONBLOCK );
2109 #endif
2110 }
2111
2112
sphSockRead(int iSock,void * buf,int iLen,int iReadTimeout,bool bIntr)2113 int sphSockRead ( int iSock, void * buf, int iLen, int iReadTimeout, bool bIntr )
2114 {
2115 assert ( iLen>0 );
2116
2117 int64_t tmMaxTimer = sphMicroTimer() + I64C(1000000)*Max ( 1, iReadTimeout ); // in microseconds
2118 int iLeftBytes = iLen; // bytes to read left
2119
2120 char * pBuf = (char*) buf;
2121 int iRes = -1, iErr = 0;
2122
2123 while ( iLeftBytes>0 )
2124 {
2125 int64_t tmMicroLeft = tmMaxTimer - sphMicroTimer();
2126 if ( tmMicroLeft<=0 )
2127 break; // timed out
2128
2129 fd_set fdRead;
2130 FD_ZERO ( &fdRead );
2131 sphFDSet ( iSock, &fdRead );
2132
2133 fd_set fdExcept;
2134 FD_ZERO ( &fdExcept );
2135 sphFDSet ( iSock, &fdExcept );
2136
2137 #if USE_WINDOWS
2138 // Windows EINTR emulation
2139 // Ctrl-C will not interrupt select on Windows, so let's handle that manually
2140 // forcibly limit select() to 100 ms, and check flag afterwards
2141 if ( bIntr )
2142 tmMicroLeft = Min ( tmMicroLeft, 100000 );
2143 #endif
2144
2145 struct timeval tv;
2146 tv.tv_sec = (int)( tmMicroLeft / 1000000 );
2147 tv.tv_usec = (int)( tmMicroLeft % 1000000 );
2148
2149 iRes = ::select ( iSock+1, &fdRead, NULL, &fdExcept, &tv );
2150
2151 // if there was EINTR, retry
2152 // if any other error, bail
2153 if ( iRes==-1 )
2154 {
2155 // only let SIGTERM (of all them) to interrupt, and only if explicitly allowed
2156 iErr = sphSockGetErrno();
2157 if ( iErr==EINTR && !( g_bGotSigterm && bIntr ))
2158 continue;
2159
2160 if ( iErr==EINTR )
2161 sphLogDebug ( "sphSockRead: select got SIGTERM, exit -1" );
2162
2163 sphSockSetErrno ( iErr );
2164 return -1;
2165 }
2166
2167 // if there was a timeout, report it as an error
2168 if ( iRes==0 )
2169 {
2170 #if USE_WINDOWS
2171 // Windows EINTR emulation
2172 if ( bIntr )
2173 {
2174 // got that SIGTERM
2175 if ( g_bGotSigterm )
2176 {
2177 sphLogDebug ( "sphSockRead: got SIGTERM emulation on Windows, exit -1" );
2178 sphSockSetErrno ( EINTR );
2179 return -1;
2180 }
2181
2182 // timeout might not be fully over just yet, so re-loop
2183 continue;
2184 }
2185 #endif
2186
2187 sphSockSetErrno ( ETIMEDOUT );
2188 return -1;
2189 }
2190
2191 // try to receive next chunk
2192 iRes = sphSockRecv ( iSock, pBuf, iLeftBytes );
2193
2194 // if there was eof, we're done
2195 if ( iRes==0 )
2196 {
2197 sphSockSetErrno ( ECONNRESET );
2198 return -1;
2199 }
2200
2201 // if there was EINTR, retry
2202 // if any other error, bail
2203 if ( iRes==-1 )
2204 {
2205 // only let SIGTERM (of all them) to interrupt, and only if explicitly allowed
2206 iErr = sphSockGetErrno();
2207 if ( iErr==EINTR && !( g_bGotSigterm && bIntr ))
2208 continue;
2209
2210 if ( iErr==EINTR )
2211 sphLogDebug ( "sphSockRead: select got SIGTERM, exit -1" );
2212
2213 sphSockSetErrno ( iErr );
2214 return -1;
2215 }
2216
2217 // update
2218 pBuf += iRes;
2219 iLeftBytes -= iRes;
2220
2221 // avoid partial buffer loss in case of signal during the 2nd (!) read
2222 bIntr = false;
2223 }
2224
2225 // if there was a timeout, report it as an error
2226 if ( iLeftBytes!=0 )
2227 {
2228 sphSockSetErrno ( ETIMEDOUT );
2229 return -1;
2230 }
2231
2232 return iLen;
2233 }
2234
2235 /////////////////////////////////////////////////////////////////////////////
2236 // NETWORK BUFFERS
2237 /////////////////////////////////////////////////////////////////////////////
2238
2239 /// fixed-memory response buffer
2240 /// tracks usage, and flushes to network when necessary
2241 class NetOutputBuffer_c
2242 {
2243 public:
2244 explicit NetOutputBuffer_c ( int iSock );
2245
SendInt(int iValue)2246 bool SendInt ( int iValue ) { return SendT<int> ( htonl ( iValue ) ); }
SendAsDword(int64_t iValue)2247 bool SendAsDword ( int64_t iValue ) ///< sends the 32bit MAX_UINT if the value is greater than it.
2248 {
2249 if ( iValue < 0 )
2250 return SendDword ( 0 );
2251 if ( iValue > UINT_MAX )
2252 return SendDword ( UINT_MAX );
2253 return SendDword ( DWORD(iValue) );
2254 }
SendDword(DWORD iValue)2255 bool SendDword ( DWORD iValue ) { return SendT<DWORD> ( htonl ( iValue ) ); }
SendLSBDword(DWORD v)2256 bool SendLSBDword ( DWORD v ) { SendByte ( (BYTE)( v&0xff ) ); SendByte ( (BYTE)( (v>>8)&0xff ) ); SendByte ( (BYTE)( (v>>16)&0xff ) ); return SendByte ( (BYTE)( (v>>24)&0xff) ); }
SendWord(WORD iValue)2257 bool SendWord ( WORD iValue ) { return SendT<WORD> ( htons ( iValue ) ); }
SendUint64(uint64_t iValue)2258 bool SendUint64 ( uint64_t iValue ) { SendT<DWORD> ( htonl ( (DWORD)(iValue>>32) ) ); return SendT<DWORD> ( htonl ( (DWORD)(iValue&0xffffffffUL) ) ); }
SendFloat(float fValue)2259 bool SendFloat ( float fValue ) { return SendT<DWORD> ( htonl ( sphF2DW ( fValue ) ) ); }
SendByte(BYTE uValue)2260 bool SendByte ( BYTE uValue ) { return SendT<BYTE> ( uValue ); }
2261
2262 #if USE_64BIT
SendDocid(SphDocID_t iValue)2263 bool SendDocid ( SphDocID_t iValue ) { return SendUint64 ( iValue ); }
2264 #else
SendDocid(SphDocID_t iValue)2265 bool SendDocid ( SphDocID_t iValue ) { return SendDword ( iValue ); }
2266 #endif
2267
2268 bool SendString ( const char * sStr );
2269 bool SendMysqlString ( const char * sStr );
2270
2271 bool Flush ( bool bUnfreeze=false );
GetError()2272 bool GetError () { return m_bError; }
GetSentCount()2273 int GetSentCount () { return m_iSent; }
2274 void FreezeBlock ( const char * sError, int iLen );
2275
2276 protected:
2277 BYTE m_dBuffer[NETOUTBUF]; ///< my buffer
2278 BYTE * m_pBuffer; ///< my current buffer position
2279 int m_iSock; ///< my socket
2280 bool m_bError; ///< if there were any write errors
2281 int m_iSent;
2282 const char *m_sError; ///< fallback message if the frozen buf overloaded
2283 int m_iErrorLength;
2284 bool m_bFlushEnabled; ///< in frozen state we never flush until special command
2285 BYTE * m_pSize; ///< the pointer to the size of frozen block
2286
2287 protected:
2288 bool SetError ( bool bValue ); ///< set error flag
2289 bool FlushIf ( int iToAdd ); ///< flush if there's not enough free space to add iToAdd bytes
2290
2291 public:
2292 bool SendBytes ( const void * pBuf, int iLen ); ///< (was) protected to avoid network-vs-host order bugs
2293 template < typename T > bool SendT ( T tValue ); ///< (was) protected to avoid network-vs-host order bugs
2294 };
2295
2296
2297 /// generic request buffer
2298 class InputBuffer_c
2299 {
2300 public:
2301 InputBuffer_c ( const BYTE * pBuf, int iLen );
~InputBuffer_c()2302 virtual ~InputBuffer_c () {}
2303
GetInt()2304 int GetInt () { return ntohl ( GetT<int> () ); }
GetWord()2305 WORD GetWord () { return ntohs ( GetT<WORD> () ); }
GetDword()2306 DWORD GetDword () { return ntohl ( GetT<DWORD> () ); }
GetLSBDword()2307 DWORD GetLSBDword () { return GetByte() + ( GetByte()<<8 ) + ( GetByte()<<16 ) + ( GetByte()<<24 ); }
GetUint64()2308 uint64_t GetUint64() { uint64_t uRes = GetDword(); return (uRes<<32)+GetDword(); }
GetByte()2309 BYTE GetByte () { return GetT<BYTE> (); }
GetFloat()2310 float GetFloat () { return sphDW2F ( ntohl ( GetT<DWORD> () ) ); }
2311 CSphString GetString ();
2312 CSphString GetRawString ( int iLen );
2313 int GetDwords ( DWORD ** pBuffer, int iMax, const char * sErrorTemplate );
GetError()2314 bool GetError () { return m_bError; }
2315
2316 template < typename T > bool GetDwords ( CSphVector<T> & dBuffer, int iMax, const char * sErrorTemplate );
2317 template < typename T > bool GetQwords ( CSphVector<T> & dBuffer, int iMax, const char * sErrorTemplate );
2318
2319 virtual void SendErrorReply ( const char *, ... ) __attribute__ ( ( format ( printf, 2, 3 ) ) ) = 0;
2320
2321 protected:
2322 const BYTE * m_pBuf;
2323 const BYTE * m_pCur;
2324 bool m_bError;
2325 int m_iLen;
2326
2327 protected:
SetError(bool bError)2328 void SetError ( bool bError ) { m_bError = bError; }
2329 bool GetBytes ( void * pBuf, int iLen );
2330 template < typename T > T GetT ();
2331 };
2332
2333
2334 /// simple memory request buffer
2335 class MemInputBuffer_c : public InputBuffer_c
2336 {
2337 public:
MemInputBuffer_c(const BYTE * pBuf,int iLen)2338 MemInputBuffer_c ( const BYTE * pBuf, int iLen ) : InputBuffer_c ( pBuf, iLen ) {}
SendErrorReply(const char *,...)2339 virtual void SendErrorReply ( const char *, ... ) __attribute__ ( ( format ( printf, 2, 3 ) ) ) {}
2340 };
2341
2342
2343 /// simple network request buffer
2344 class NetInputBuffer_c : public InputBuffer_c
2345 {
2346 public:
2347 explicit NetInputBuffer_c ( int iSock );
2348 virtual ~NetInputBuffer_c ();
2349
2350 bool ReadFrom ( int iLen, int iTimeout, bool bIntr=false, bool bAppend=false );
ReadFrom(int iLen)2351 bool ReadFrom ( int iLen ) { return ReadFrom ( iLen, g_iReadTimeout ); }
2352
2353 virtual void SendErrorReply ( const char *, ... ) __attribute__ ( ( format ( printf, 2, 3 ) ) );
2354
GetBufferPtr() const2355 const BYTE * GetBufferPtr () const { return m_pBuf; }
IsIntr() const2356 bool IsIntr () const { return m_bIntr; }
2357
2358 protected:
2359 static const int NET_MINIBUFFER_SIZE = 4096;
2360
2361 int m_iSock;
2362 bool m_bIntr;
2363
2364 BYTE m_dMinibufer[NET_MINIBUFFER_SIZE];
2365 int m_iMaxibuffer;
2366 BYTE * m_pMaxibuffer;
2367 };
2368
2369 /////////////////////////////////////////////////////////////////////////////
2370
NetOutputBuffer_c(int iSock)2371 NetOutputBuffer_c::NetOutputBuffer_c ( int iSock )
2372 : m_pBuffer ( m_dBuffer )
2373 , m_iSock ( iSock )
2374 , m_bError ( false )
2375 , m_iSent ( 0 )
2376 , m_bFlushEnabled ( true )
2377 {
2378 assert ( m_iSock>0 );
2379 }
2380
2381
SendT(T tValue)2382 template < typename T > bool NetOutputBuffer_c::SendT ( T tValue )
2383 {
2384 if ( m_bError )
2385 return false;
2386
2387 FlushIf ( sizeof(T) );
2388
2389 sphUnalignedWrite ( m_pBuffer, tValue );
2390 m_pBuffer += sizeof(T);
2391 assert ( m_pBuffer<m_dBuffer+sizeof(m_dBuffer) );
2392 return true;
2393 }
2394
2395
SendString(const char * sStr)2396 bool NetOutputBuffer_c::SendString ( const char * sStr )
2397 {
2398 if ( m_bError )
2399 return false;
2400
2401 FlushIf ( sizeof(DWORD) );
2402
2403 int iLen = sStr ? strlen(sStr) : 0;
2404 SendInt ( iLen );
2405 return SendBytes ( sStr, iLen );
2406 }
2407
2408
MysqlPackedLen(const char * sStr)2409 int MysqlPackedLen ( const char * sStr )
2410 {
2411 int iLen = strlen(sStr);
2412 if ( iLen<251 )
2413 return 1 + iLen;
2414 if ( iLen<=0xffff )
2415 return 3 + iLen;
2416 if ( iLen<=0xffffff )
2417 return 4 + iLen;
2418 return 9 + iLen;
2419 }
2420
2421
2422
2423 // encodes Mysql Length-coded binary
MysqlPack(void * pBuffer,int iValue)2424 void * MysqlPack ( void * pBuffer, int iValue )
2425 {
2426 char * pOutput = (char*)pBuffer;
2427 if ( iValue<0 )
2428 return (void*)pOutput;
2429
2430 if ( iValue<251 )
2431 {
2432 *pOutput++ = (char)iValue;
2433 return (void*)pOutput;
2434 }
2435
2436 if ( iValue<=0xFFFF )
2437 {
2438 *pOutput++ = '\xFC';
2439 *pOutput++ = (char)iValue;
2440 *pOutput++ = (char)( iValue>>8 );
2441 return (void*)pOutput;
2442 }
2443
2444 if ( iValue<=0xFFFFFF )
2445 {
2446 *pOutput++ = '\xFD';
2447 *pOutput++ = (char)iValue;
2448 *pOutput++ = (char)( iValue>>8 );
2449 *pOutput++ = (char)( iValue>>16 );
2450 return (void *) pOutput;
2451 }
2452
2453 *pOutput++ = '\xFE';
2454 *pOutput++ = (char)iValue;
2455 *pOutput++ = (char)( iValue>>8 );
2456 *pOutput++ = (char)( iValue>>16 );
2457 *pOutput++ = (char)( iValue>>24 );
2458 *pOutput++ = 0;
2459 *pOutput++ = 0;
2460 *pOutput++ = 0;
2461 *pOutput++ = 0;
2462 return (void*)pOutput;
2463 }
2464
MysqlUnpack(InputBuffer_c & tReq,DWORD * pSize)2465 int MysqlUnpack ( InputBuffer_c & tReq, DWORD * pSize )
2466 {
2467 assert ( pSize );
2468
2469 int iRes = tReq.GetByte();
2470 --*pSize;
2471 if ( iRes < 251 )
2472 return iRes;
2473
2474 if ( iRes==0xFC )
2475 {
2476 *pSize -=2;
2477 return tReq.GetByte() + ((int)tReq.GetByte()<<8);
2478 }
2479
2480 if ( iRes==0xFD )
2481 {
2482 *pSize -= 3;
2483 return tReq.GetByte() + ((int)tReq.GetByte()<<8) + ((int)tReq.GetByte()<<16);
2484 }
2485
2486 if ( iRes==0xFE )
2487 iRes = tReq.GetByte() + ((int)tReq.GetByte()<<8) + ((int)tReq.GetByte()<<16) + ((int)tReq.GetByte()<<24);
2488
2489 tReq.GetByte();
2490 tReq.GetByte();
2491 tReq.GetByte();
2492 tReq.GetByte();
2493 *pSize -= 8;
2494 return iRes;
2495 }
2496
2497
SendMysqlString(const char * sStr)2498 bool NetOutputBuffer_c::SendMysqlString ( const char * sStr )
2499 {
2500 if ( m_bError )
2501 return false;
2502
2503 int iLen = strlen(sStr);
2504
2505 BYTE dBuf[12];
2506 BYTE * pBuf = (BYTE*) MysqlPack ( dBuf, iLen );
2507 SendBytes ( dBuf, (int)( pBuf-dBuf ) );
2508 return SendBytes ( sStr, iLen );
2509 }
2510
2511
SendBytes(const void * pBuf,int iLen)2512 bool NetOutputBuffer_c::SendBytes ( const void * pBuf, int iLen )
2513 {
2514 BYTE * pMy = (BYTE*)pBuf;
2515 while ( iLen>0 && !m_bError )
2516 {
2517 int iLeft = sizeof(m_dBuffer) - ( m_pBuffer - m_dBuffer );
2518 if ( iLen<=iLeft )
2519 {
2520 memcpy ( m_pBuffer, pMy, iLen );
2521 m_pBuffer += iLen;
2522 break;
2523 }
2524
2525 memcpy ( m_pBuffer, pMy, iLeft );
2526 m_pBuffer += iLeft;
2527 Flush ();
2528
2529 pMy += iLeft;
2530 iLen -= iLeft;
2531 }
2532 return !m_bError;
2533 }
2534
2535
Flush(bool bUnfreeze)2536 bool NetOutputBuffer_c::Flush ( bool bUnfreeze )
2537 {
2538 if ( m_bError )
2539 return false;
2540
2541 int iLen = m_pBuffer-m_dBuffer;
2542 if ( iLen==0 )
2543 return true;
2544
2545 if ( g_bGotSigterm )
2546 sphLogDebug ( "SIGTERM in NetOutputBuffer::Flush" );
2547
2548 if ( bUnfreeze )
2549 {
2550 BYTE * pBuf = m_pBuffer;
2551 m_pBuffer = m_pSize;
2552 SendDword ( pBuf-m_pSize-4 );
2553 m_pBuffer = pBuf;
2554 m_bFlushEnabled = true;
2555 }
2556
2557 // buffer overloaded. It is fail. Send the error message.
2558 if ( !m_bFlushEnabled )
2559 {
2560 sphLogDebug ( "NetOutputBuffer with disabled flush is overloaded" );
2561 m_pBuffer = m_dBuffer;
2562 SendBytes ( m_sError, m_iErrorLength );
2563 iLen = m_pBuffer-m_dBuffer;
2564 if ( iLen==0 )
2565 return true;
2566 }
2567
2568 assert ( iLen>0 );
2569 assert ( iLen<=(int)sizeof(m_dBuffer) );
2570 char * pBuffer = (char *)&m_dBuffer[0];
2571
2572 const int64_t tmMaxTimer = sphMicroTimer() + g_iWriteTimeout*1000000; // in microseconds
2573 while ( !m_bError )
2574 {
2575 int iRes = sphSockSend ( m_iSock, pBuffer, iLen );
2576 if ( iRes < 0 )
2577 {
2578 int iErrno = sphSockGetErrno();
2579 if ( iErrno==EINTR ) // interrupted before any data was sent; just loop
2580 continue;
2581 if ( iErrno!=EAGAIN && iErrno!=EWOULDBLOCK )
2582 {
2583 sphWarning ( "send() failed: %d: %s", iErrno, sphSockError(iErrno) );
2584 m_bError = true;
2585 break;
2586 }
2587 } else
2588 {
2589 m_iSent += iRes;
2590 pBuffer += iRes;
2591 iLen -= iRes;
2592 if ( iLen==0 )
2593 break;
2594 }
2595
2596 int64_t tmMicroLeft = tmMaxTimer - sphMicroTimer();
2597 if ( tmMicroLeft>0 )
2598 {
2599 fd_set fdWrite;
2600 FD_ZERO ( &fdWrite );
2601 sphFDSet ( m_iSock, &fdWrite );
2602
2603 struct timeval tvTimeout;
2604 tvTimeout.tv_sec = (int)( tmMicroLeft / 1000000 );
2605 tvTimeout.tv_usec = (int)( tmMicroLeft % 1000000 );
2606
2607 iRes = select ( m_iSock+1, NULL, &fdWrite, NULL, &tvTimeout );
2608 } else
2609 iRes = 0;
2610
2611 switch ( iRes )
2612 {
2613 case 1: // ready for writing
2614 break;
2615
2616 case 0: // timed out
2617 {
2618 sphWarning ( "timed out while trying to flush network buffers" );
2619 m_bError = true;
2620 break;
2621 }
2622
2623 case -1: // error
2624 {
2625 int iErrno = sphSockGetErrno();
2626 if ( iErrno==EINTR )
2627 break;
2628 sphWarning ( "select() failed: %d: %s", iErrno, sphSockError(iErrno) );
2629 m_bError = true;
2630 break;
2631 }
2632 }
2633 }
2634
2635 m_pBuffer = m_dBuffer;
2636 return !m_bError;
2637 }
2638
FreezeBlock(const char * sError,int iLen)2639 void NetOutputBuffer_c::FreezeBlock ( const char * sError, int iLen )
2640 {
2641 m_sError = sError;
2642 m_iErrorLength = iLen;
2643 m_bFlushEnabled = false;
2644 // reserve the DWORD for the size
2645 m_pSize = m_pBuffer;
2646 SendDword ( 0 );
2647 }
2648
2649
FlushIf(int iToAdd)2650 bool NetOutputBuffer_c::FlushIf ( int iToAdd )
2651 {
2652 if ( ( m_pBuffer+iToAdd )>=( m_dBuffer+sizeof(m_dBuffer) ) )
2653 return Flush ();
2654
2655 return !m_bError;
2656 }
2657
2658 /////////////////////////////////////////////////////////////////////////////
2659
InputBuffer_c(const BYTE * pBuf,int iLen)2660 InputBuffer_c::InputBuffer_c ( const BYTE * pBuf, int iLen )
2661 : m_pBuf ( pBuf )
2662 , m_pCur ( pBuf )
2663 , m_bError ( !pBuf || iLen<0 )
2664 , m_iLen ( iLen )
2665 {}
2666
2667
GetT()2668 template < typename T > T InputBuffer_c::GetT ()
2669 {
2670 if ( m_bError || ( m_pCur+sizeof(T) > m_pBuf+m_iLen ) )
2671 {
2672 SetError ( true );
2673 return 0;
2674 }
2675
2676 T iRes = sphUnalignedRead ( *(T*)m_pCur );
2677 m_pCur += sizeof(T);
2678 return iRes;
2679 }
2680
2681
GetString()2682 CSphString InputBuffer_c::GetString ()
2683 {
2684 CSphString sRes;
2685
2686 int iLen = GetInt ();
2687 if ( m_bError || iLen<0 || iLen>g_iMaxPacketSize || ( m_pCur+iLen > m_pBuf+m_iLen ) )
2688 {
2689 SetError ( true );
2690 return sRes;
2691 }
2692
2693 sRes.SetBinary ( (char*)m_pCur, iLen );
2694 m_pCur += iLen;
2695 return sRes;
2696 }
2697
2698
GetRawString(int iLen)2699 CSphString InputBuffer_c::GetRawString ( int iLen )
2700 {
2701 CSphString sRes;
2702
2703 if ( m_bError || iLen<0 || iLen>g_iMaxPacketSize || ( m_pCur+iLen > m_pBuf+m_iLen ) )
2704 {
2705 SetError ( true );
2706 return sRes;
2707 }
2708
2709 sRes.SetBinary ( (char*)m_pCur, iLen );
2710 m_pCur += iLen;
2711 return sRes;
2712 }
2713
2714
GetBytes(void * pBuf,int iLen)2715 bool InputBuffer_c::GetBytes ( void * pBuf, int iLen )
2716 {
2717 assert ( pBuf );
2718 assert ( iLen>0 && iLen<=g_iMaxPacketSize );
2719
2720 if ( m_bError || ( m_pCur+iLen > m_pBuf+m_iLen ) )
2721 {
2722 SetError ( true );
2723 return false;
2724 }
2725
2726 memcpy ( pBuf, m_pCur, iLen );
2727 m_pCur += iLen;
2728 return true;
2729 }
2730
2731
GetDwords(DWORD ** ppBuffer,int iMax,const char * sErrorTemplate)2732 int InputBuffer_c::GetDwords ( DWORD ** ppBuffer, int iMax, const char * sErrorTemplate )
2733 {
2734 assert ( ppBuffer );
2735 assert ( !(*ppBuffer) );
2736
2737 int iCount = GetInt ();
2738 if ( iCount<0 || iCount>iMax )
2739 {
2740 SendErrorReply ( sErrorTemplate, iCount, iMax );
2741 SetError ( true );
2742 return -1;
2743 }
2744 if ( iCount )
2745 {
2746 assert ( !(*ppBuffer) ); // potential leak
2747 (*ppBuffer) = new DWORD [ iCount ];
2748 if ( !GetBytes ( (*ppBuffer), sizeof(DWORD)*iCount ) )
2749 {
2750 SafeDeleteArray ( (*ppBuffer) );
2751 return -1;
2752 }
2753 for ( int i=0; i<iCount; i++ )
2754 (*ppBuffer)[i] = htonl ( (*ppBuffer)[i] );
2755 }
2756 return iCount;
2757 }
2758
2759
GetDwords(CSphVector<T> & dBuffer,int iMax,const char * sErrorTemplate)2760 template < typename T > bool InputBuffer_c::GetDwords ( CSphVector<T> & dBuffer, int iMax, const char * sErrorTemplate )
2761 {
2762 int iCount = GetInt ();
2763 if ( iCount<0 || iCount>iMax )
2764 {
2765 SendErrorReply ( sErrorTemplate, iCount, iMax );
2766 SetError ( true );
2767 return false;
2768 }
2769
2770 dBuffer.Resize ( iCount );
2771 ARRAY_FOREACH ( i, dBuffer )
2772 dBuffer[i] = GetDword ();
2773
2774 if ( m_bError )
2775 dBuffer.Reset ();
2776
2777 return !m_bError;
2778 }
2779
2780
GetQwords(CSphVector<T> & dBuffer,int iMax,const char * sErrorTemplate)2781 template < typename T > bool InputBuffer_c::GetQwords ( CSphVector<T> & dBuffer, int iMax, const char * sErrorTemplate )
2782 {
2783 int iCount = GetInt ();
2784 if ( iCount<0 || iCount>iMax )
2785 {
2786 SendErrorReply ( sErrorTemplate, iCount, iMax );
2787 SetError ( true );
2788 return false;
2789 }
2790
2791 dBuffer.Resize ( iCount );
2792 ARRAY_FOREACH ( i, dBuffer )
2793 dBuffer[i] = GetUint64 ();
2794
2795 if ( m_bError )
2796 dBuffer.Reset ();
2797
2798 return !m_bError;
2799 }
2800 /////////////////////////////////////////////////////////////////////////////
2801
NetInputBuffer_c(int iSock)2802 NetInputBuffer_c::NetInputBuffer_c ( int iSock )
2803 : InputBuffer_c ( m_dMinibufer, sizeof(m_dMinibufer) )
2804 , m_iSock ( iSock )
2805 , m_bIntr ( false )
2806 , m_iMaxibuffer ( 0 )
2807 , m_pMaxibuffer ( NULL )
2808 {}
2809
2810
~NetInputBuffer_c()2811 NetInputBuffer_c::~NetInputBuffer_c ()
2812 {
2813 SafeDeleteArray ( m_pMaxibuffer );
2814 }
2815
2816
ReadFrom(int iLen,int iTimeout,bool bIntr,bool bAppend)2817 bool NetInputBuffer_c::ReadFrom ( int iLen, int iTimeout, bool bIntr, bool bAppend )
2818 {
2819 assert (!( bAppend && m_pCur!=m_pBuf && m_pBuf!=m_pMaxibuffer )); // only allow appends to untouched maxi-buffers
2820 int iCur = bAppend ? m_iLen : 0;
2821
2822 m_bIntr = false;
2823 if ( iLen<=0 || iLen>g_iMaxPacketSize || m_iSock<0 )
2824 return false;
2825
2826 BYTE * pBuf = m_dMinibufer + iCur;
2827 if ( ( iCur+iLen )>NET_MINIBUFFER_SIZE )
2828 {
2829 if ( ( iCur+iLen )>m_iMaxibuffer )
2830 {
2831 if ( iCur )
2832 {
2833 BYTE * pNew = new BYTE [ iCur+iLen ];
2834 memcpy ( pNew, m_pCur, iCur );
2835 SafeDeleteArray ( m_pMaxibuffer );
2836 m_pMaxibuffer = pNew;
2837 m_iMaxibuffer = iCur+iLen;
2838 } else
2839 {
2840 SafeDeleteArray ( m_pMaxibuffer );
2841 m_pMaxibuffer = new BYTE [ iLen ];
2842 m_iMaxibuffer = iLen;
2843 }
2844 }
2845 pBuf = m_pMaxibuffer;
2846 }
2847
2848 m_pCur = m_pBuf = pBuf;
2849 int iGot = sphSockRead ( m_iSock, pBuf + iCur, iLen, iTimeout, bIntr );
2850 if ( g_bGotSigterm )
2851 {
2852 sphLogDebug ( "NetInputBuffer_c::ReadFrom: got SIGTERM, return false" );
2853 m_bError = true;
2854 m_bIntr = true;
2855 return false;
2856 }
2857
2858 m_bError = ( iGot!=iLen );
2859 m_bIntr = m_bError && ( sphSockPeekErrno()==EINTR );
2860 m_iLen = m_bError ? 0 : iCur+iLen;
2861 return !m_bError;
2862 }
2863
2864
SendErrorReply(const char * sTemplate,...)2865 void NetInputBuffer_c::SendErrorReply ( const char * sTemplate, ... )
2866 {
2867 char dBuf [ 2048 ];
2868
2869 const int iHeaderLen = 12;
2870 const int iMaxStrLen = sizeof(dBuf) - iHeaderLen - 1;
2871
2872 // fill header
2873 WORD * p0 = (WORD*)&dBuf[0];
2874 p0[0] = htons ( SEARCHD_ERROR ); // error code
2875 p0[1] = 0; // version doesn't matter
2876
2877 // fill error string
2878 char * sBuf = dBuf + iHeaderLen;
2879
2880 va_list ap;
2881 va_start ( ap, sTemplate );
2882 vsnprintf ( sBuf, iMaxStrLen, sTemplate, ap );
2883 va_end ( ap );
2884
2885 sBuf[iMaxStrLen] = '\0';
2886 int iStrLen = strlen(sBuf);
2887
2888 // fixup lengths
2889 DWORD * p4 = (DWORD*)&dBuf[4];
2890 p4[0] = htonl ( 4+iStrLen );
2891 p4[1] = htonl ( iStrLen );
2892
2893 // send!
2894 sphSockSend ( m_iSock, dBuf, iHeaderLen+iStrLen );
2895
2896 // --console logging
2897 if ( g_bOptNoDetach && g_eLogFormat!=LOG_FORMAT_SPHINXQL )
2898 sphInfo ( "query error: %s", sBuf );
2899 }
2900
2901 // fix MSVC 2005 fuckup
2902 #if USE_WINDOWS
2903 #pragma conform(forScope,on)
2904 #endif
2905
2906 /////////////////////////////////////////////////////////////////////////////
2907 // DISTRIBUTED QUERIES
2908 /////////////////////////////////////////////////////////////////////////////
2909
2910 /// remote agent descriptor (stored in a global hash)
2911 struct AgentDesc_t
2912 {
2913 CSphString m_sHost; ///< remote searchd host
2914 int m_iPort; ///< remote searchd port, 0 if local
2915 CSphString m_sPath; ///< local searchd UNIX socket path
2916 CSphString m_sIndexes; ///< remote index names to query
2917 bool m_bBlackhole; ///< blackhole agent flag
2918 int m_iFamily; ///< TCP or UNIX socket
2919 DWORD m_uAddr; ///< IP address
2920 int m_iStatsIndex; ///< index into global searchd stats array
2921
2922 public:
AgentDesc_tAgentDesc_t2923 AgentDesc_t ()
2924 : m_iPort ( -1 )
2925 , m_bBlackhole ( false )
2926 , m_iFamily ( AF_INET )
2927 , m_uAddr ( 0 )
2928 , m_iStatsIndex ( -1 )
2929 {}
2930 };
2931
2932 /// remote agent state
2933 enum AgentState_e
2934 {
2935 AGENT_UNUSED, ///< agent is unused for this request
2936 AGENT_CONNECT, ///< connecting to agent
2937 AGENT_HELLO, ///< waiting for "VER x" hello
2938 AGENT_QUERY, ///< query sent, waiting for reply
2939 AGENT_PREREPLY, ///< query sent, activity detected, need to read reply
2940 AGENT_REPLY, ///< reading reply
2941 AGENT_RETRY ///< should retry
2942 };
2943
2944 /// remote agent connection (local per-query state)
2945 struct AgentConn_t : public AgentDesc_t
2946 {
2947 int m_iSock; ///< socket number, -1 if not connected
2948 AgentState_e m_eState; ///< current state
2949
2950 bool m_bSuccess; ///< whether last request was successful (ie. there are available results)
2951 CSphString m_sFailure; ///< failure message
2952
2953 int m_iReplyStatus; ///< reply status code
2954 int m_iReplySize; ///< how many reply bytes are there
2955 int m_iReplyRead; ///< how many reply bytes are alredy received
2956 BYTE * m_pReplyBuf; ///< reply buffer
2957
2958 CSphVector<CSphQueryResult> m_dResults; ///< multi-query results
2959
2960 int64_t m_iWall; ///< wall time spent vs this agent
2961
2962 public:
AgentConn_tAgentConn_t2963 AgentConn_t ()
2964 : m_iSock ( -1 )
2965 , m_eState ( AGENT_UNUSED )
2966 , m_bSuccess ( false )
2967 , m_iReplyStatus ( -1 )
2968 , m_iReplySize ( 0 )
2969 , m_iReplyRead ( 0 )
2970 , m_pReplyBuf ( NULL )
2971 , m_iWall ( 0 )
2972 {}
2973
~AgentConn_tAgentConn_t2974 ~AgentConn_t ()
2975 {
2976 Close ();
2977 }
2978
CloseAgentConn_t2979 void Close ()
2980 {
2981 SafeDeleteArray ( m_pReplyBuf );
2982 if ( m_iSock>0 )
2983 {
2984 sphSockClose ( m_iSock );
2985 m_iSock = -1;
2986 if ( m_eState!=AGENT_RETRY )
2987 m_eState = AGENT_UNUSED;
2988 }
2989 m_iWall += sphMicroTimer ();
2990 }
2991
GetNameAgentConn_t2992 CSphString GetName() const
2993 {
2994 CSphString sName;
2995 switch ( m_iFamily )
2996 {
2997 case AF_INET: sName.SetSprintf ( "%s:%u", m_sHost.cstr(), m_iPort ); break;
2998 case AF_UNIX: sName = m_sPath; break;
2999 }
3000 return sName;
3001 }
3002
operator =AgentConn_t3003 AgentConn_t & operator = ( const AgentDesc_t & rhs )
3004 {
3005 m_sHost = rhs.m_sHost;
3006 m_iPort = rhs.m_iPort;
3007 m_sPath = rhs.m_sPath;
3008 m_sIndexes = rhs.m_sIndexes;
3009 m_bBlackhole = rhs.m_bBlackhole;
3010 m_iFamily = rhs.m_iFamily;
3011 m_uAddr = rhs.m_uAddr;
3012 m_iStatsIndex = rhs.m_iStatsIndex;
3013 return *this;
3014 }
3015 };
3016
3017 /// distributed index
3018 struct DistributedIndex_t
3019 {
3020 CSphVector<AgentDesc_t> m_dAgents; ///< remote agents
3021 CSphVector<CSphString> m_dLocal; ///< local indexes
3022 int m_iAgentConnectTimeout; ///< in msec
3023 int m_iAgentQueryTimeout; ///< in msec
3024 bool m_bToDelete; ///< should be deleted
3025
3026 public:
DistributedIndex_tDistributedIndex_t3027 DistributedIndex_t ()
3028 : m_iAgentConnectTimeout ( 1000 )
3029 , m_iAgentQueryTimeout ( 3000 )
3030 , m_bToDelete ( false )
3031 {}
3032 };
3033
3034 /// global distributed index definitions hash
3035 static SmallStringHash_T < DistributedIndex_t > g_hDistIndexes;
3036
3037 /////////////////////////////////////////////////////////////////////////////
3038
3039 struct IRequestBuilder_t : public ISphNoncopyable
3040 {
~IRequestBuilder_tIRequestBuilder_t3041 virtual ~IRequestBuilder_t () {} // to avoid gcc4 warns
3042 virtual void BuildRequest ( const char * sIndexes, NetOutputBuffer_c & tOut, int iAgent ) const = 0;
3043 };
3044
3045
3046 struct IReplyParser_t
3047 {
~IReplyParser_tIReplyParser_t3048 virtual ~IReplyParser_t () {} // to avoid gcc4 warns
3049 virtual bool ParseReply ( MemInputBuffer_c & tReq, AgentConn_t & tAgent, int iAgent ) const = 0;
3050 };
3051
3052
3053 #define AGENT_STATS_INC(_agent,_counter) \
3054 if ( g_pStats && _agent.m_iStatsIndex>=0 && _agent.m_iStatsIndex<STATS_MAX_AGENTS ) \
3055 { \
3056 g_tStatsMutex.Lock (); \
3057 g_pStats->m_dAgentStats [ _agent.m_iStatsIndex ]._counter++; \
3058 g_tStatsMutex.Unlock (); \
3059 }
3060
3061
ConnectToRemoteAgents(CSphVector<AgentConn_t> & dAgents,bool bRetryOnly)3062 void ConnectToRemoteAgents ( CSphVector<AgentConn_t> & dAgents, bool bRetryOnly )
3063 {
3064 ARRAY_FOREACH ( iAgent, dAgents )
3065 {
3066 AgentConn_t & tAgent = dAgents[iAgent];
3067 if ( bRetryOnly && ( tAgent.m_eState!=AGENT_RETRY ) )
3068 continue;
3069
3070 tAgent.m_eState = AGENT_UNUSED;
3071 tAgent.m_bSuccess = false;
3072
3073 socklen_t len = 0;
3074 struct sockaddr_storage ss;
3075 memset ( &ss, 0, sizeof(ss) );
3076 ss.ss_family = (short)tAgent.m_iFamily;
3077
3078 if ( ss.ss_family==AF_INET )
3079 {
3080 struct sockaddr_in *in = (struct sockaddr_in *)&ss;
3081 in->sin_port = htons ( (unsigned short)tAgent.m_iPort );
3082 in->sin_addr.s_addr = tAgent.m_uAddr;
3083 len = sizeof(*in);
3084 }
3085 #if !USE_WINDOWS
3086 else if ( ss.ss_family==AF_UNIX )
3087 {
3088 struct sockaddr_un *un = (struct sockaddr_un *)&ss;
3089 snprintf ( un->sun_path, sizeof(un->sun_path), "%s", tAgent.m_sPath.cstr() );
3090 len = sizeof(*un);
3091 }
3092 #endif
3093
3094 tAgent.m_iSock = socket ( tAgent.m_iFamily, SOCK_STREAM, 0 );
3095 if ( tAgent.m_iSock<0 )
3096 {
3097 tAgent.m_sFailure.SetSprintf ( "socket() failed: %s", sphSockError() );
3098 return;
3099 }
3100
3101 if ( sphSetSockNB ( tAgent.m_iSock )<0 )
3102 {
3103 tAgent.m_sFailure.SetSprintf ( "sphSetSockNB() failed: %s", sphSockError() );
3104 return;
3105 }
3106
3107 // count connects
3108 if ( g_pStats )
3109 {
3110 g_tStatsMutex.Lock();
3111 g_pStats->m_iAgentConnect++;
3112 if ( bRetryOnly )
3113 g_pStats->m_iAgentRetry++;
3114 g_tStatsMutex.Unlock();
3115 }
3116
3117 if ( !bRetryOnly )
3118 tAgent.m_iWall = 0;
3119 tAgent.m_iWall -= sphMicroTimer();
3120
3121 if ( connect ( tAgent.m_iSock, (struct sockaddr*)&ss, len )<0 )
3122 {
3123 int iErr = sphSockGetErrno();
3124 if ( iErr!=EINPROGRESS && iErr!=EINTR && iErr!=EWOULDBLOCK ) // check for EWOULDBLOCK is for winsock only
3125 {
3126 tAgent.Close ();
3127 tAgent.m_sFailure.SetSprintf ( "connect() failed: %s", sphSockError(iErr) );
3128 tAgent.m_eState = AGENT_RETRY; // do retry on connect() failures
3129 AGENT_STATS_INC ( tAgent, m_iConnectFailures );
3130 return;
3131
3132 } else
3133 {
3134 // connection in progress
3135 tAgent.m_eState = AGENT_CONNECT;
3136 }
3137 } else
3138 {
3139 // socket connected, ready to read hello message
3140 tAgent.m_eState = AGENT_HELLO;
3141 }
3142 }
3143 }
3144
3145
QueryRemoteAgents(CSphVector<AgentConn_t> & dAgents,int iTimeout,const IRequestBuilder_t & tBuilder,int64_t * pWaited)3146 int QueryRemoteAgents ( CSphVector<AgentConn_t> & dAgents, int iTimeout, const IRequestBuilder_t & tBuilder, int64_t * pWaited )
3147 {
3148 int iAgents = 0;
3149 assert ( iTimeout>=0 );
3150
3151 int64_t tmMaxTimer = sphMicroTimer() + iTimeout*1000; // in microseconds
3152 for ( ;; )
3153 {
3154 fd_set fdsRead, fdsWrite;
3155 FD_ZERO ( &fdsRead );
3156 FD_ZERO ( &fdsWrite );
3157
3158 int iMax = 0;
3159 bool bDone = true;
3160 ARRAY_FOREACH ( i, dAgents )
3161 {
3162 const AgentConn_t & tAgent = dAgents[i];
3163 if ( tAgent.m_eState==AGENT_CONNECT || tAgent.m_eState==AGENT_HELLO || tAgent.m_eState==AGENT_QUERY )
3164 {
3165 assert ( !tAgent.m_sPath.IsEmpty() || tAgent.m_iPort>0 );
3166 assert ( tAgent.m_iSock>0 );
3167
3168 sphFDSet ( tAgent.m_iSock, ( tAgent.m_eState==AGENT_CONNECT ) ? &fdsWrite : &fdsRead );
3169 iMax = Max ( iMax, tAgent.m_iSock );
3170 if ( tAgent.m_eState!=AGENT_QUERY )
3171 bDone = false;
3172 }
3173 }
3174 if ( bDone )
3175 break;
3176
3177 int64_t tmSelect = sphMicroTimer();
3178 int64_t tmMicroLeft = tmMaxTimer - tmSelect;
3179 if ( tmMicroLeft<=0 )
3180 break; // FIXME? what about iTimeout==0 case?
3181
3182 struct timeval tvTimeout;
3183 tvTimeout.tv_sec = (int)( tmMicroLeft/ 1000000 ); // full seconds
3184 tvTimeout.tv_usec = (int)( tmMicroLeft % 1000000 ); // microseconds
3185
3186 // we don't care about exceptfds; they're for OOB only
3187 int iSelected = select ( 1+iMax, &fdsRead, &fdsWrite, NULL, &tvTimeout );
3188
3189 if ( pWaited )
3190 *pWaited += sphMicroTimer() - tmSelect;
3191
3192 if ( iSelected<=0 )
3193 continue;
3194
3195 ARRAY_FOREACH ( i, dAgents )
3196 {
3197 AgentConn_t & tAgent = dAgents[i];
3198
3199 // check if connection completed
3200 // tricky part, we MUST use write-set ONLY here at this check
3201 // even though we can't tell connect() success from just OS send buffer availability
3202 // but any check involving read-set just never ever completes, so...
3203 if ( tAgent.m_eState==AGENT_CONNECT && FD_ISSET ( tAgent.m_iSock, &fdsWrite ) )
3204 {
3205 int iErr = 0;
3206 socklen_t iErrLen = sizeof(iErr);
3207 getsockopt ( tAgent.m_iSock, SOL_SOCKET, SO_ERROR, (char*)&iErr, &iErrLen );
3208 if ( iErr )
3209 {
3210 // connect() failure
3211 tAgent.m_sFailure.SetSprintf ( "connect() failed: %s", sphSockError(iErr) );
3212 tAgent.Close ();
3213 AGENT_STATS_INC ( tAgent, m_iConnectFailures );
3214 } else
3215 {
3216 // connect() success
3217 tAgent.m_eState = AGENT_HELLO;
3218 }
3219 continue;
3220 }
3221
3222 // check if hello was received
3223 if ( tAgent.m_eState==AGENT_HELLO && FD_ISSET ( tAgent.m_iSock, &fdsRead ) )
3224 {
3225 // read reply
3226 int iRemoteVer;
3227 int iRes = sphSockRecv ( tAgent.m_iSock, (char*)&iRemoteVer, sizeof(iRemoteVer) );
3228 if ( iRes!=sizeof(iRemoteVer) )
3229 {
3230 tAgent.Close ();
3231 if ( iRes<0 )
3232 {
3233 // network error
3234 int iErr = sphSockGetErrno();
3235 tAgent.m_sFailure.SetSprintf ( "handshake failure (errno=%d, msg=%s)", iErr, sphSockError(iErr) );
3236 AGENT_STATS_INC ( tAgent, m_iNetworkErrors );
3237
3238 } else if ( iRes>0 )
3239 {
3240 // incomplete reply
3241 tAgent.m_sFailure.SetSprintf ( "handshake failure (exp=%d, recv=%d)", (int)sizeof(iRemoteVer), iRes );
3242 AGENT_STATS_INC ( tAgent, m_iWrongReplies );
3243
3244 } else
3245 {
3246 // agent closed the connection
3247 // this might happen in out-of-sync connect-accept case; so let's retry
3248 tAgent.m_sFailure = "handshake failure (connection was closed)";
3249 tAgent.m_eState = AGENT_RETRY;
3250 AGENT_STATS_INC ( tAgent, m_iUnexpectedClose );
3251 }
3252 continue;
3253 }
3254
3255 iRemoteVer = ntohl ( iRemoteVer );
3256 if (!( iRemoteVer==SPHINX_SEARCHD_PROTO || iRemoteVer==0x01000000UL ) ) // workaround for all the revisions that sent it in host order...
3257 {
3258 tAgent.m_sFailure.SetSprintf ( "handshake failure (unexpected protocol version=%d)", iRemoteVer );
3259 AGENT_STATS_INC ( tAgent, m_iWrongReplies );
3260 tAgent.Close ();
3261 continue;
3262 }
3263
3264 // send request
3265 NetOutputBuffer_c tOut ( tAgent.m_iSock );
3266 tBuilder.BuildRequest ( tAgent.m_sIndexes.cstr(), tOut, i );
3267 bool bFlushed = tOut.Flush (); // FIXME! handle flush failure?
3268
3269 #ifdef TCP_NODELAY
3270 int iDisable = 1;
3271 if ( bFlushed && tAgent.m_iFamily==AF_INET )
3272 setsockopt ( tAgent.m_iSock, IPPROTO_TCP, TCP_NODELAY, (char*)&iDisable, sizeof(iDisable) );
3273 #endif
3274
3275 tAgent.m_eState = AGENT_QUERY;
3276 iAgents++;
3277 continue;
3278 }
3279
3280 // check if queried agent replied while we were querying others
3281 if ( tAgent.m_eState==AGENT_QUERY && FD_ISSET ( tAgent.m_iSock, &fdsRead ) )
3282 {
3283 // do not account agent wall time from here; agent is probably ready
3284 tAgent.m_iWall += sphMicroTimer();
3285 tAgent.m_eState = AGENT_PREREPLY;
3286 continue;
3287 }
3288 }
3289 }
3290
3291 ARRAY_FOREACH ( i, dAgents )
3292 {
3293 // check if connection timed out
3294 AgentConn_t & tAgent = dAgents[i];
3295 if ( tAgent.m_eState!=AGENT_QUERY && tAgent.m_eState!=AGENT_UNUSED && tAgent.m_eState!=AGENT_RETRY && tAgent.m_eState!=AGENT_PREREPLY )
3296 {
3297 // technically, we can end up here via two different routes
3298 // a) connect() never finishes in given time frame
3299 // b) agent actually accept()s the connection but keeps silence
3300 // however, there's no way to tell the two from each other
3301 // so we just account both cases as connect() failure
3302 tAgent.Close ();
3303 tAgent.m_sFailure.SetSprintf ( "connect() timed out" );
3304 tAgent.m_eState = AGENT_RETRY; // do retry on connect() failures
3305 AGENT_STATS_INC ( tAgent, m_iTimeoutsConnect );
3306 }
3307 }
3308
3309 return iAgents;
3310 }
3311
3312
WaitForRemoteAgents(CSphVector<AgentConn_t> & dAgents,int iTimeout,IReplyParser_t & tParser,int64_t * pWaited)3313 int WaitForRemoteAgents ( CSphVector<AgentConn_t> & dAgents, int iTimeout, IReplyParser_t & tParser, int64_t * pWaited )
3314 {
3315 assert ( iTimeout>=0 );
3316
3317 int iAgents = 0;
3318 int64_t tmMaxTimer = sphMicroTimer() + iTimeout*1000; // in microseconds
3319 for ( ;; )
3320 {
3321 fd_set fdsRead;
3322 FD_ZERO ( &fdsRead );
3323
3324 int iMax = 0;
3325 bool bDone = true;
3326 ARRAY_FOREACH ( iAgent, dAgents )
3327 {
3328 AgentConn_t & tAgent = dAgents[iAgent];
3329 if ( tAgent.m_bBlackhole )
3330 continue;
3331
3332 if ( tAgent.m_eState==AGENT_QUERY || tAgent.m_eState==AGENT_REPLY || tAgent.m_eState==AGENT_PREREPLY )
3333 {
3334 assert ( !tAgent.m_sPath.IsEmpty() || tAgent.m_iPort>0 );
3335 assert ( tAgent.m_iSock>0 );
3336
3337 sphFDSet ( tAgent.m_iSock, &fdsRead );
3338 iMax = Max ( iMax, tAgent.m_iSock );
3339 bDone = false;
3340 }
3341 }
3342 if ( bDone )
3343 break;
3344
3345 int64_t tmSelect = sphMicroTimer();
3346 int64_t tmMicroLeft = tmMaxTimer - tmSelect;
3347 if ( tmMicroLeft<=0 ) // FIXME? what about iTimeout==0 case?
3348 break;
3349
3350 struct timeval tvTimeout;
3351 tvTimeout.tv_sec = (int)( tmMicroLeft / 1000000 ); // full seconds
3352 tvTimeout.tv_usec = (int)( tmMicroLeft % 1000000 ); // microseconds
3353
3354 int iSelected = select ( 1+iMax, &fdsRead, NULL, NULL, &tvTimeout );
3355
3356 if ( pWaited )
3357 *pWaited += sphMicroTimer() - tmSelect;
3358
3359 if ( iSelected<=0 )
3360 continue;
3361
3362 ARRAY_FOREACH ( iAgent, dAgents )
3363 {
3364 AgentConn_t & tAgent = dAgents[iAgent];
3365 if ( tAgent.m_bBlackhole )
3366 continue;
3367 if (!( tAgent.m_eState==AGENT_QUERY || tAgent.m_eState==AGENT_REPLY || tAgent.m_eState==AGENT_PREREPLY ))
3368 continue;
3369 if ( !FD_ISSET ( tAgent.m_iSock, &fdsRead ) )
3370 continue;
3371
3372 // if there was no reply yet, read reply header
3373 bool bFailure = true;
3374 for ( ;; )
3375 {
3376 if ( tAgent.m_eState==AGENT_QUERY || tAgent.m_eState==AGENT_PREREPLY )
3377 {
3378 if ( tAgent.m_eState==AGENT_PREREPLY )
3379 {
3380 tAgent.m_iWall -= sphMicroTimer();
3381 tAgent.m_eState = AGENT_QUERY;
3382 }
3383
3384 // try to read
3385 struct
3386 {
3387 WORD m_iStatus;
3388 WORD m_iVer;
3389 int m_iLength;
3390 } tReplyHeader;
3391 STATIC_SIZE_ASSERT ( tReplyHeader, 8 );
3392
3393 if ( sphSockRecv ( tAgent.m_iSock, (char*)&tReplyHeader, sizeof(tReplyHeader) )!=sizeof(tReplyHeader) )
3394 {
3395 // bail out if failed
3396 tAgent.m_sFailure.SetSprintf ( "failed to receive reply header" );
3397 AGENT_STATS_INC ( tAgent, m_iNetworkErrors );
3398 break;
3399 }
3400
3401 tReplyHeader.m_iStatus = ntohs ( tReplyHeader.m_iStatus );
3402 tReplyHeader.m_iVer = ntohs ( tReplyHeader.m_iVer );
3403 tReplyHeader.m_iLength = ntohl ( tReplyHeader.m_iLength );
3404
3405 // check the packet
3406 if ( tReplyHeader.m_iLength<0 || tReplyHeader.m_iLength>g_iMaxPacketSize ) // FIXME! add reasonable max packet len too
3407 {
3408 tAgent.m_sFailure.SetSprintf ( "invalid packet size (status=%d, len=%d, max_packet_size=%d)", tReplyHeader.m_iStatus, tReplyHeader.m_iLength, g_iMaxPacketSize );
3409 AGENT_STATS_INC ( tAgent, m_iWrongReplies );
3410 break;
3411 }
3412
3413 // header received, switch the status
3414 assert ( tAgent.m_pReplyBuf==NULL );
3415 tAgent.m_eState = AGENT_REPLY;
3416 tAgent.m_pReplyBuf = new BYTE [ tReplyHeader.m_iLength ];
3417 tAgent.m_iReplySize = tReplyHeader.m_iLength;
3418 tAgent.m_iReplyRead = 0;
3419 tAgent.m_iReplyStatus = tReplyHeader.m_iStatus;
3420
3421 if ( !tAgent.m_pReplyBuf )
3422 {
3423 // bail out if failed
3424 tAgent.m_sFailure.SetSprintf ( "failed to alloc %d bytes for reply buffer", tAgent.m_iReplySize );
3425 break;
3426 }
3427 }
3428
3429 // if we are reading reply, read another chunk
3430 if ( tAgent.m_eState==AGENT_REPLY )
3431 {
3432 // do read
3433 assert ( tAgent.m_iReplyRead<tAgent.m_iReplySize );
3434 int iRes = sphSockRecv ( tAgent.m_iSock, (char*)tAgent.m_pReplyBuf+tAgent.m_iReplyRead,
3435 tAgent.m_iReplySize-tAgent.m_iReplyRead );
3436
3437 // bail out if read failed
3438 if ( iRes<0 )
3439 {
3440 tAgent.m_sFailure.SetSprintf ( "failed to receive reply body: %s", sphSockError() );
3441 AGENT_STATS_INC ( tAgent, m_iNetworkErrors );
3442 break;
3443 }
3444
3445 assert ( iRes>0 );
3446 assert ( tAgent.m_iReplyRead+iRes<=tAgent.m_iReplySize );
3447 tAgent.m_iReplyRead += iRes;
3448 }
3449
3450 // if reply was fully received, parse it
3451 if ( tAgent.m_eState==AGENT_REPLY && tAgent.m_iReplyRead==tAgent.m_iReplySize )
3452 {
3453 MemInputBuffer_c tReq ( tAgent.m_pReplyBuf, tAgent.m_iReplySize );
3454
3455 // absolve thy former sins
3456 tAgent.m_sFailure = "";
3457
3458 // check for general errors/warnings first
3459 if ( tAgent.m_iReplyStatus==SEARCHD_WARNING )
3460 {
3461 CSphString sAgentWarning = tReq.GetString ();
3462 tAgent.m_sFailure.SetSprintf ( "remote warning: %s", sAgentWarning.cstr() );
3463
3464 } else if ( tAgent.m_iReplyStatus==SEARCHD_RETRY )
3465 {
3466 tAgent.m_eState = AGENT_RETRY;
3467 CSphString sAgentError = tReq.GetString ();
3468 tAgent.m_sFailure.SetSprintf ( "remote warning: %s", sAgentError.cstr() );
3469 break;
3470
3471 } else if ( tAgent.m_iReplyStatus!=SEARCHD_OK )
3472 {
3473 CSphString sAgentError = tReq.GetString ();
3474 tAgent.m_sFailure.SetSprintf ( "remote error: %s", sAgentError.cstr() );
3475 break;
3476 }
3477
3478 // call parser
3479 if ( !tParser.ParseReply ( tReq, tAgent, iAgent ) )
3480 break;
3481
3482 // check if there was enough data
3483 if ( tReq.GetError() )
3484 {
3485 tAgent.m_sFailure.SetSprintf ( "incomplete reply" );
3486 AGENT_STATS_INC ( tAgent, m_iWrongReplies );
3487 break;
3488 }
3489
3490 // all is well
3491 iAgents++;
3492 tAgent.Close ();
3493
3494 tAgent.m_bSuccess = true;
3495 }
3496
3497 bFailure = false;
3498 break;
3499 }
3500
3501 if ( bFailure )
3502 {
3503 tAgent.Close ();
3504 tAgent.m_dResults.Reset ();
3505 }
3506 }
3507 }
3508
3509 // close timed-out agents
3510 ARRAY_FOREACH ( iAgent, dAgents )
3511 {
3512 AgentConn_t & tAgent = dAgents[iAgent];
3513 if ( tAgent.m_bBlackhole )
3514 tAgent.Close ();
3515 else if ( tAgent.m_eState==AGENT_QUERY || tAgent.m_eState==AGENT_PREREPLY )
3516 {
3517 assert ( !tAgent.m_dResults.GetLength() );
3518 assert ( !tAgent.m_bSuccess );
3519 tAgent.Close ();
3520 tAgent.m_sFailure.SetSprintf ( "query timed out" );
3521 AGENT_STATS_INC ( tAgent, m_iTimeoutsQuery );
3522 }
3523 }
3524
3525 return iAgents;
3526 }
3527
3528 /////////////////////////////////////////////////////////////////////////////
3529 // SEARCH HANDLER
3530 /////////////////////////////////////////////////////////////////////////////
3531
3532 struct SearchRequestBuilder_t : public IRequestBuilder_t
3533 {
SearchRequestBuilder_tSearchRequestBuilder_t3534 SearchRequestBuilder_t ( const CSphVector<CSphQuery> & dQueries, int iStart, int iEnd ) : m_dQueries ( dQueries ), m_iStart ( iStart ), m_iEnd ( iEnd ) {}
3535 virtual void BuildRequest ( const char * sIndexes, NetOutputBuffer_c & tOut, int ) const;
3536
3537 protected:
3538 int CalcQueryLen ( const char * sIndexes, const CSphQuery & q ) const;
3539 void SendQuery ( const char * sIndexes, NetOutputBuffer_c & tOut, const CSphQuery & q ) const;
3540
3541 protected:
3542 const CSphVector<CSphQuery> & m_dQueries;
3543 int m_iStart;
3544 int m_iEnd;
3545 };
3546
3547
3548 struct SearchReplyParser_t : public IReplyParser_t, public ISphNoncopyable
3549 {
SearchReplyParser_tSearchReplyParser_t3550 SearchReplyParser_t ( int iStart, int iEnd, CSphVector<DWORD> & dMvaStorage, CSphVector<BYTE> & dStringsStorage )
3551 : m_iStart ( iStart )
3552 , m_iEnd ( iEnd )
3553 , m_dMvaStorage ( dMvaStorage )
3554 , m_dStringsStorage ( dStringsStorage )
3555 {}
3556
3557 virtual bool ParseReply ( MemInputBuffer_c & tReq, AgentConn_t & tAgent, int iAgent ) const;
3558
3559 protected:
3560 int m_iStart;
3561 int m_iEnd;
3562 CSphVector<DWORD> & m_dMvaStorage;
3563 CSphVector<BYTE> & m_dStringsStorage;
3564 };
3565
3566 /////////////////////////////////////////////////////////////////////////////
3567
CalcQueryLen(const char * sIndexes,const CSphQuery & q) const3568 int SearchRequestBuilder_t::CalcQueryLen ( const char * sIndexes, const CSphQuery & q ) const
3569 {
3570 int iReqSize = 108 + 2*sizeof(SphDocID_t) + 4*q.m_iWeights
3571 + q.m_sSortBy.Length()
3572 + strlen ( sIndexes )
3573 + q.m_sGroupBy.Length()
3574 + q.m_sGroupSortBy.Length()
3575 + q.m_sGroupDistinct.Length()
3576 + q.m_sComment.Length()
3577 + q.m_sSelect.Length();
3578 iReqSize += q.m_sRawQuery.IsEmpty()
3579 ? q.m_sQuery.Length()
3580 : q.m_sRawQuery.Length();
3581 if ( q.m_eRanker==SPH_RANK_EXPR )
3582 iReqSize += q.m_sRankerExpr.Length() + 4;
3583 ARRAY_FOREACH ( j, q.m_dFilters )
3584 {
3585 const CSphFilterSettings & tFilter = q.m_dFilters[j];
3586 iReqSize += 12 + tFilter.m_sAttrName.Length(); // string attr-name; int type; int exclude-flag
3587 switch ( tFilter.m_eType )
3588 {
3589 case SPH_FILTER_VALUES: iReqSize += 4 + 8*tFilter.GetNumValues (); break; // int values-count; uint64[] values
3590 case SPH_FILTER_RANGE: iReqSize += 16; break; // uint64 min-val, max-val
3591 case SPH_FILTER_FLOATRANGE: iReqSize += 8; break; // int/float min-val,max-val
3592 }
3593 }
3594 if ( q.m_bGeoAnchor )
3595 iReqSize += 16 + q.m_sGeoLatAttr.Length() + q.m_sGeoLongAttr.Length(); // string lat-attr, long-attr; float lat, long
3596 ARRAY_FOREACH ( i, q.m_dIndexWeights )
3597 iReqSize += 8 + q.m_dIndexWeights[i].m_sName.Length(); // string index-name; int index-weight
3598 ARRAY_FOREACH ( i, q.m_dFieldWeights )
3599 iReqSize += 8 + q.m_dFieldWeights[i].m_sName.Length(); // string field-name; int field-weight
3600 ARRAY_FOREACH ( i, q.m_dOverrides )
3601 iReqSize += 12 + q.m_dOverrides[i].m_sAttr.Length() + // string attr-name; int type; int values-count
3602 ( q.m_dOverrides[i].m_eAttrType==SPH_ATTR_BIGINT ? 16 : 12 )*q.m_dOverrides[i].m_dValues.GetLength(); // ( bigint id; int/float/bigint value )[] values
3603 return iReqSize;
3604 }
3605
3606
SendQuery(const char * sIndexes,NetOutputBuffer_c & tOut,const CSphQuery & q) const3607 void SearchRequestBuilder_t::SendQuery ( const char * sIndexes, NetOutputBuffer_c & tOut, const CSphQuery & q ) const
3608 {
3609 tOut.SendInt ( 0 ); // offset is 0
3610 tOut.SendInt ( q.m_iMaxMatches ); // limit is MAX_MATCHES
3611 tOut.SendInt ( (DWORD)q.m_eMode ); // match mode
3612 tOut.SendInt ( (DWORD)q.m_eRanker ); // ranking mode
3613 if ( q.m_eRanker==SPH_RANK_EXPR )
3614 tOut.SendString ( q.m_sRankerExpr.cstr() );
3615 tOut.SendInt ( q.m_eSort ); // sort mode
3616 tOut.SendString ( q.m_sSortBy.cstr() ); // sort attr
3617 if ( q.m_sRawQuery.IsEmpty() )
3618 tOut.SendString ( q.m_sQuery.cstr() );
3619 else
3620 tOut.SendString ( q.m_sRawQuery.cstr() ); // query
3621 tOut.SendInt ( q.m_iWeights );
3622 for ( int j=0; j<q.m_iWeights; j++ )
3623 tOut.SendInt ( q.m_pWeights[j] ); // weights
3624 tOut.SendString ( sIndexes ); // indexes
3625 tOut.SendInt ( USE_64BIT ); // id range bits
3626 tOut.SendDocid ( 0 ); // default full id range (any client range must be in filters at this stage)
3627 tOut.SendDocid ( DOCID_MAX );
3628 tOut.SendInt ( q.m_dFilters.GetLength() );
3629 ARRAY_FOREACH ( j, q.m_dFilters )
3630 {
3631 const CSphFilterSettings & tFilter = q.m_dFilters[j];
3632 tOut.SendString ( tFilter.m_sAttrName.cstr() );
3633 tOut.SendInt ( tFilter.m_eType );
3634 switch ( tFilter.m_eType )
3635 {
3636 case SPH_FILTER_VALUES:
3637 tOut.SendInt ( tFilter.GetNumValues () );
3638 for ( int k = 0; k < tFilter.GetNumValues (); k++ )
3639 tOut.SendUint64 ( tFilter.GetValue ( k ) );
3640 break;
3641
3642 case SPH_FILTER_RANGE:
3643 tOut.SendUint64 ( tFilter.m_iMinValue );
3644 tOut.SendUint64 ( tFilter.m_iMaxValue );
3645 break;
3646
3647 case SPH_FILTER_FLOATRANGE:
3648 tOut.SendFloat ( tFilter.m_fMinValue );
3649 tOut.SendFloat ( tFilter.m_fMaxValue );
3650 break;
3651 }
3652 tOut.SendInt ( tFilter.m_bExclude );
3653 }
3654 tOut.SendInt ( q.m_eGroupFunc );
3655 tOut.SendString ( q.m_sGroupBy.cstr() );
3656 tOut.SendInt ( q.m_iMaxMatches );
3657 tOut.SendString ( q.m_sGroupSortBy.cstr() );
3658 tOut.SendInt ( q.m_iCutoff );
3659 tOut.SendInt ( q.m_iRetryCount );
3660 tOut.SendInt ( q.m_iRetryDelay );
3661 tOut.SendString ( q.m_sGroupDistinct.cstr() );
3662 tOut.SendInt ( q.m_bGeoAnchor );
3663 if ( q.m_bGeoAnchor )
3664 {
3665 tOut.SendString ( q.m_sGeoLatAttr.cstr() );
3666 tOut.SendString ( q.m_sGeoLongAttr.cstr() );
3667 tOut.SendFloat ( q.m_fGeoLatitude );
3668 tOut.SendFloat ( q.m_fGeoLongitude );
3669 }
3670 tOut.SendInt ( q.m_dIndexWeights.GetLength() );
3671 ARRAY_FOREACH ( i, q.m_dIndexWeights )
3672 {
3673 tOut.SendString ( q.m_dIndexWeights[i].m_sName.cstr() );
3674 tOut.SendInt ( q.m_dIndexWeights[i].m_iValue );
3675 }
3676 tOut.SendDword ( q.m_uMaxQueryMsec );
3677 tOut.SendInt ( q.m_dFieldWeights.GetLength() );
3678 ARRAY_FOREACH ( i, q.m_dFieldWeights )
3679 {
3680 tOut.SendString ( q.m_dFieldWeights[i].m_sName.cstr() );
3681 tOut.SendInt ( q.m_dFieldWeights[i].m_iValue );
3682 }
3683 tOut.SendString ( q.m_sComment.cstr() );
3684 tOut.SendInt ( q.m_dOverrides.GetLength() );
3685 ARRAY_FOREACH ( i, q.m_dOverrides )
3686 {
3687 const CSphAttrOverride & tEntry = q.m_dOverrides[i];
3688 tOut.SendString ( tEntry.m_sAttr.cstr() );
3689 tOut.SendDword ( tEntry.m_eAttrType );
3690 tOut.SendInt ( tEntry.m_dValues.GetLength() );
3691 ARRAY_FOREACH ( j, tEntry.m_dValues )
3692 {
3693 tOut.SendUint64 ( tEntry.m_dValues[j].m_uDocID );
3694 switch ( tEntry.m_eAttrType )
3695 {
3696 case SPH_ATTR_FLOAT: tOut.SendFloat ( tEntry.m_dValues[j].m_fValue ); break;
3697 case SPH_ATTR_BIGINT: tOut.SendUint64 ( tEntry.m_dValues[j].m_uValue ); break;
3698 default: tOut.SendDword ( (DWORD)tEntry.m_dValues[j].m_uValue ); break;
3699 }
3700 }
3701 }
3702 tOut.SendString ( q.m_sSelect.cstr() );
3703 // master v.1.0
3704 tOut.SendDword ( q.m_eCollation );
3705 }
3706
3707
BuildRequest(const char * sIndexes,NetOutputBuffer_c & tOut,int) const3708 void SearchRequestBuilder_t::BuildRequest ( const char * sIndexes, NetOutputBuffer_c & tOut, int ) const
3709 {
3710 int iReqLen = 8; // int num-queries
3711 for ( int i=m_iStart; i<=m_iEnd; i++ )
3712 iReqLen += CalcQueryLen ( sIndexes, m_dQueries[i] );
3713
3714 tOut.SendDword ( SPHINX_SEARCHD_PROTO );
3715 tOut.SendWord ( SEARCHD_COMMAND_SEARCH ); // command id
3716 tOut.SendWord ( VER_COMMAND_SEARCH ); // command version
3717 tOut.SendInt ( iReqLen ); // request body length
3718
3719 tOut.SendInt ( VER_MASTER );
3720 tOut.SendInt ( m_iEnd-m_iStart+1 );
3721 for ( int i=m_iStart; i<=m_iEnd; i++ )
3722 SendQuery ( sIndexes, tOut, m_dQueries[i] );
3723 }
3724
3725 /////////////////////////////////////////////////////////////////////////////
3726
ParseReply(MemInputBuffer_c & tReq,AgentConn_t & tAgent,int) const3727 bool SearchReplyParser_t::ParseReply ( MemInputBuffer_c & tReq, AgentConn_t & tAgent, int ) const
3728 {
3729 int iResults = m_iEnd-m_iStart+1;
3730 assert ( iResults>0 );
3731
3732 tAgent.m_dResults.Resize ( iResults );
3733 for ( int iRes=0; iRes<iResults; iRes++ )
3734 tAgent.m_dResults[iRes].m_iSuccesses = 0;
3735
3736 for ( int iRes=0; iRes<iResults; iRes++ )
3737 {
3738 CSphQueryResult & tRes = tAgent.m_dResults [ iRes ];
3739 tRes.m_sError = "";
3740 tRes.m_sWarning = "";
3741
3742 // get status and message
3743 DWORD eStatus = tReq.GetDword ();
3744 if ( eStatus!=SEARCHD_OK )
3745 {
3746 CSphString sMessage = tReq.GetString ();
3747 switch ( eStatus )
3748 {
3749 case SEARCHD_ERROR: tRes.m_sError = sMessage; continue;
3750 case SEARCHD_RETRY: tRes.m_sError = sMessage; break;
3751 case SEARCHD_WARNING: tRes.m_sWarning = sMessage; break;
3752 default: tAgent.m_sFailure.SetSprintf ( "internal error: unknown status %d", eStatus ); break;
3753 }
3754 }
3755
3756 // get schema
3757 CSphSchema & tSchema = tRes.m_tSchema;
3758 tSchema.Reset ();
3759
3760 tSchema.m_dFields.Resize ( tReq.GetInt() ); // FIXME! add a sanity check
3761 ARRAY_FOREACH ( j, tSchema.m_dFields )
3762 tSchema.m_dFields[j].m_sName = tReq.GetString ();
3763
3764 int iNumAttrs = tReq.GetInt(); // FIXME! add a sanity check
3765 for ( int j=0; j<iNumAttrs; j++ )
3766 {
3767 CSphColumnInfo tCol;
3768 tCol.m_sName = tReq.GetString ();
3769 tCol.m_eAttrType = (ESphAttr) tReq.GetDword (); // FIXME! add a sanity check
3770 tSchema.AddAttr ( tCol, true ); // all attributes received from agents are dynamic
3771 }
3772
3773 // get matches
3774 int iMatches = tReq.GetInt ();
3775 if ( iMatches<0 || iMatches>g_iMaxMatches )
3776 {
3777 tAgent.m_sFailure.SetSprintf ( "invalid match count received (count=%d)", iMatches );
3778 return false;
3779 }
3780
3781 int bAgent64 = tReq.GetInt ();
3782 #if !USE_64BIT
3783 if ( bAgent64 )
3784 tAgent.m_sFailure.SetSprintf ( "id64 agent, id32 master, docids might be wrapped" );
3785 #endif
3786
3787 assert ( !tRes.m_dMatches.GetLength() );
3788 if ( iMatches )
3789 {
3790 tRes.m_dMatches.Resize ( iMatches );
3791 ARRAY_FOREACH ( i, tRes.m_dMatches )
3792 {
3793 CSphMatch & tMatch = tRes.m_dMatches[i];
3794 tMatch.Reset ( tSchema.GetRowSize() );
3795 tMatch.m_iDocID = bAgent64 ? (SphDocID_t)tReq.GetUint64() : tReq.GetDword();
3796 tMatch.m_iWeight = tReq.GetInt ();
3797 for ( int j=0; j<tSchema.GetAttrsCount(); j++ )
3798 {
3799 const CSphColumnInfo & tAttr = tSchema.GetAttr(j);
3800 if ( tAttr.m_eAttrType==SPH_ATTR_UINT32SET || tAttr.m_eAttrType==SPH_ATTR_INT64SET )
3801 {
3802 tMatch.SetAttr ( tAttr.m_tLocator, m_dMvaStorage.GetLength() );
3803
3804 int iValues = tReq.GetDword ();
3805 m_dMvaStorage.Add ( iValues );
3806 if ( tAttr.m_eAttrType==SPH_ATTR_UINT32SET )
3807 {
3808 while ( iValues-- )
3809 m_dMvaStorage.Add ( tReq.GetDword() );
3810 } else
3811 {
3812 assert ( ( iValues%2 )==0 );
3813 for ( ; iValues; iValues -= 2 )
3814 {
3815 uint64_t uMva = tReq.GetUint64();
3816 m_dMvaStorage.Add ( (DWORD)uMva );
3817 m_dMvaStorage.Add ( (DWORD)( uMva>>32 ) );
3818 }
3819 }
3820
3821 } else if ( tAttr.m_eAttrType==SPH_ATTR_FLOAT )
3822 {
3823 float fRes = tReq.GetFloat();
3824 tMatch.SetAttr ( tAttr.m_tLocator, sphF2DW(fRes) );
3825
3826 } else if ( tAttr.m_eAttrType==SPH_ATTR_BIGINT )
3827 {
3828 tMatch.SetAttr ( tAttr.m_tLocator, tReq.GetUint64() );
3829
3830 } else if ( tAttr.m_eAttrType==SPH_ATTR_STRING )
3831 {
3832 CSphString sValue = tReq.GetString();
3833 int iLen = sValue.Length();
3834
3835 int iOff = m_dStringsStorage.GetLength();
3836 tMatch.SetAttr ( tAttr.m_tLocator, iOff );
3837
3838 m_dStringsStorage.Resize ( iOff+3+iLen );
3839 BYTE * pBuf = &m_dStringsStorage[iOff];
3840 pBuf += sphPackStrlen ( pBuf, iLen );
3841 memcpy ( pBuf, sValue.cstr(), iLen );
3842
3843 } else
3844 {
3845 tMatch.SetAttr ( tAttr.m_tLocator, tReq.GetDword() );
3846 }
3847 }
3848 }
3849 }
3850
3851 // read totals (retrieved count, total count, query time, word count)
3852 int iRetrieved = tReq.GetInt ();
3853 tRes.m_iTotalMatches = (unsigned int)tReq.GetInt ();
3854 tRes.m_iQueryTime = tReq.GetInt ();
3855 const int iWordsCount = tReq.GetInt (); // FIXME! sanity check?
3856 if ( iRetrieved!=iMatches )
3857 {
3858 tAgent.m_sFailure.SetSprintf ( "expected %d retrieved documents, got %d", iMatches, iRetrieved );
3859 return false;
3860 }
3861
3862 // read per-word stats
3863 for ( int i=0; i<iWordsCount; i++ )
3864 {
3865 const CSphString sWord = tReq.GetString ();
3866 const int64_t iDocs = (unsigned int)tReq.GetInt ();
3867 const int64_t iHits = (unsigned int)tReq.GetInt ();
3868 const bool bExpanded = ( tReq.GetByte()!=0 ); // agents always send expanded flag to master
3869
3870 tRes.AddStat ( sWord, iDocs, iHits, bExpanded );
3871 }
3872
3873 // mark this result as ok
3874 tRes.m_iSuccesses = 1;
3875 }
3876
3877 // all seems OK (and buffer length checks are performed by caller)
3878 return true;
3879 }
3880
3881 /////////////////////////////////////////////////////////////////////////////
3882
3883 // returns true if incoming schema (src) is equal to existing (dst); false otherwise
MinimizeSchema(CSphSchema & tDst,const CSphSchema & tSrc)3884 bool MinimizeSchema ( CSphSchema & tDst, const CSphSchema & tSrc )
3885 {
3886 // if dst is empty, result is also empty
3887 if ( tDst.GetAttrsCount()==0 )
3888 return tSrc.GetAttrsCount()==0;
3889
3890 // check for equality, and remove all dst attributes that are not present in src
3891 CSphVector<CSphColumnInfo> dDst;
3892 for ( int i=0; i<tDst.GetAttrsCount(); i++ )
3893 dDst.Add ( tDst.GetAttr(i) );
3894
3895 bool bEqual = ( tDst.GetAttrsCount()==tSrc.GetAttrsCount() );
3896 ARRAY_FOREACH ( i, dDst )
3897 {
3898 int iSrcIdx = tSrc.GetAttrIndex ( dDst[i].m_sName.cstr() );
3899
3900 // check for index mismatch
3901 if ( iSrcIdx!=i )
3902 bEqual = false;
3903
3904 // check for type/size mismatch (and fixup if needed)
3905 if ( iSrcIdx>=0 )
3906 {
3907 const CSphColumnInfo & tSrcAttr = tSrc.GetAttr ( iSrcIdx );
3908
3909 // should seamlessly convert ( bool > float ) | ( bool > int > bigint )
3910 ESphAttr eDst = dDst[i].m_eAttrType;
3911 ESphAttr eSrc = tSrcAttr.m_eAttrType;
3912 bool bSame = ( eDst==eSrc )
3913 || ( ( eDst==SPH_ATTR_FLOAT && eSrc==SPH_ATTR_BOOL ) || ( eDst==SPH_ATTR_BOOL && eSrc==SPH_ATTR_FLOAT ) )
3914 || ( ( eDst==SPH_ATTR_BOOL || eDst==SPH_ATTR_INTEGER || eDst==SPH_ATTR_BIGINT )
3915 && ( eSrc==SPH_ATTR_BOOL || eSrc==SPH_ATTR_INTEGER || eSrc==SPH_ATTR_BIGINT ) );
3916
3917 int iDstBitCount = dDst[i].m_tLocator.m_iBitCount;
3918 int iSrcBitCount = tSrcAttr.m_tLocator.m_iBitCount;
3919
3920 if ( !bSame )
3921 {
3922 // different types? remove the attr
3923 iSrcIdx = -1;
3924 bEqual = false;
3925
3926 } else if ( iDstBitCount!=iSrcBitCount )
3927 {
3928 // different bit sizes? choose the max one
3929 dDst[i].m_tLocator.m_iBitCount = Max ( iDstBitCount, iSrcBitCount );
3930 bEqual = false;
3931 if ( iDstBitCount<iSrcBitCount )
3932 dDst[i].m_eAttrType = tSrcAttr.m_eAttrType;
3933 }
3934
3935 if ( tSrcAttr.m_tLocator.m_iBitOffset!=dDst[i].m_tLocator.m_iBitOffset )
3936 {
3937 // different offsets? have to force target dynamic then, since we can't use one locator for all matches
3938 bEqual = false;
3939 }
3940
3941 if ( tSrcAttr.m_tLocator.m_bDynamic!=dDst[i].m_tLocator.m_bDynamic )
3942 {
3943 // different location? have to force target dynamic then
3944 bEqual = false;
3945 }
3946 }
3947
3948 // check for presence
3949 if ( iSrcIdx<0 )
3950 {
3951 dDst.Remove ( i );
3952 i--;
3953 }
3954 }
3955
3956 tDst.ResetAttrs ();
3957 ARRAY_FOREACH ( i, dDst )
3958 tDst.AddAttr ( dDst[i], dDst[i].m_tLocator.m_bDynamic | !bEqual ); // force dynamic attrs on inequality
3959
3960 return bEqual;
3961 }
3962
3963
FixupQuery(CSphQuery * pQuery,const CSphSchema * pSchema,const char * sIndexName,CSphString & sError)3964 bool FixupQuery ( CSphQuery * pQuery, const CSphSchema * pSchema, const char * sIndexName, CSphString & sError )
3965 {
3966 // already?
3967 if ( !pQuery->m_iOldVersion )
3968 return true;
3969
3970 if ( pQuery->m_iOldGroups>0 || pQuery->m_iOldMinGID!=0 || pQuery->m_iOldMaxGID!=UINT_MAX )
3971 {
3972 int iAttr = -1;
3973 for ( int i=0; i<pSchema->GetAttrsCount(); i++ )
3974 if ( pSchema->GetAttr(i).m_eAttrType==SPH_ATTR_INTEGER )
3975 {
3976 iAttr = i;
3977 break;
3978 }
3979
3980 if ( iAttr<0 )
3981 {
3982 sError.SetSprintf ( "index '%s': no group attribute found", sIndexName );
3983 return false;
3984 }
3985
3986 CSphFilterSettings tFilter;
3987 tFilter.m_sAttrName = pSchema->GetAttr(iAttr).m_sName;
3988 tFilter.m_dValues.Resize ( pQuery->m_iOldGroups );
3989 ARRAY_FOREACH ( i, tFilter.m_dValues )
3990 tFilter.m_dValues[i] = pQuery->m_pOldGroups[i];
3991 tFilter.m_iMinValue = pQuery->m_iOldMinGID;
3992 tFilter.m_iMaxValue = pQuery->m_iOldMaxGID;
3993 pQuery->m_dFilters.Add ( tFilter );
3994 }
3995
3996 if ( pQuery->m_iOldMinTS!=0 || pQuery->m_iOldMaxTS!=UINT_MAX )
3997 {
3998 int iAttr = -1;
3999 for ( int i=0; i<pSchema->GetAttrsCount(); i++ )
4000 if ( pSchema->GetAttr(i).m_eAttrType==SPH_ATTR_TIMESTAMP )
4001 {
4002 iAttr = i;
4003 break;
4004 }
4005
4006 if ( iAttr<0 )
4007 {
4008 sError.SetSprintf ( "index '%s': no timestamp attribute found", sIndexName );
4009 return false;
4010 }
4011
4012 CSphFilterSettings tFilter;
4013 tFilter.m_sAttrName = pSchema->GetAttr(iAttr).m_sName;
4014 tFilter.m_iMinValue = pQuery->m_iOldMinTS;
4015 tFilter.m_iMaxValue = pQuery->m_iOldMaxTS;
4016 pQuery->m_dFilters.Add ( tFilter );
4017 }
4018
4019 pQuery->m_iOldVersion = 0;
4020 return true;
4021 }
4022
4023
ParseIndexList(const CSphString & sIndexes,CSphVector<CSphString> & dOut)4024 void ParseIndexList ( const CSphString & sIndexes, CSphVector<CSphString> & dOut )
4025 {
4026 CSphString sSplit = sIndexes;
4027 char * p = (char*)sSplit.cstr();
4028 while ( *p )
4029 {
4030 // skip non-alphas
4031 while ( (*p) && !sphIsAlpha(*p) ) p++;
4032 if ( !(*p) ) break;
4033
4034 // this is my next index name
4035 const char * sNext = p;
4036 while ( sphIsAlpha(*p) ) p++;
4037
4038 assert ( sNext!=p );
4039 if ( *p ) *p++ = '\0'; // if it was not the end yet, we'll continue from next char
4040
4041 dOut.Add ( sNext );
4042 }
4043 }
4044
4045
CheckQuery(const CSphQuery & tQuery,CSphString & sError)4046 void CheckQuery ( const CSphQuery & tQuery, CSphString & sError )
4047 {
4048 sError = NULL;
4049 if ( tQuery.m_eMode<0 || tQuery.m_eMode>SPH_MATCH_TOTAL )
4050 {
4051 sError.SetSprintf ( "invalid match mode %d", tQuery.m_eMode );
4052 return;
4053 }
4054 if ( tQuery.m_eRanker<0 || tQuery.m_eRanker>SPH_RANK_TOTAL )
4055 {
4056 sError.SetSprintf ( "invalid ranking mode %d", tQuery.m_eRanker );
4057 return;
4058 }
4059 if ( tQuery.m_iMaxMatches<1 || tQuery.m_iMaxMatches>g_iMaxMatches )
4060 {
4061 sError.SetSprintf ( "per-query max_matches=%d out of bounds (per-server max_matches=%d)",
4062 tQuery.m_iMaxMatches, g_iMaxMatches );
4063 return;
4064 }
4065 if ( tQuery.m_iOffset<0 || tQuery.m_iOffset>=tQuery.m_iMaxMatches )
4066 {
4067 sError.SetSprintf ( "offset out of bounds (offset=%d, max_matches=%d)",
4068 tQuery.m_iOffset, tQuery.m_iMaxMatches );
4069 return;
4070 }
4071 if ( tQuery.m_iLimit<0 )
4072 {
4073 sError.SetSprintf ( "limit out of bounds (limit=%d)", tQuery.m_iLimit );
4074 return;
4075 }
4076 if ( tQuery.m_iCutoff<0 )
4077 {
4078 sError.SetSprintf ( "cutoff out of bounds (cutoff=%d)", tQuery.m_iCutoff );
4079 return;
4080 }
4081 if ( tQuery.m_iRetryCount<0 || tQuery.m_iRetryCount>MAX_RETRY_COUNT )
4082 {
4083 sError.SetSprintf ( "retry count out of bounds (count=%d)", tQuery.m_iRetryCount );
4084 return;
4085 }
4086 if ( tQuery.m_iRetryDelay<0 || tQuery.m_iRetryDelay>MAX_RETRY_DELAY )
4087 {
4088 sError.SetSprintf ( "retry delay out of bounds (delay=%d)", tQuery.m_iRetryDelay );
4089 return;
4090 }
4091 }
4092
4093
PrepareQueryEmulation(CSphQuery * pQuery)4094 void PrepareQueryEmulation ( CSphQuery * pQuery )
4095 {
4096 assert ( pQuery && pQuery->m_sRawQuery.cstr() );
4097
4098 // sort filters
4099 ARRAY_FOREACH ( i, pQuery->m_dFilters )
4100 pQuery->m_dFilters[i].m_dValues.Sort();
4101
4102 // sort overrides
4103 ARRAY_FOREACH ( i, pQuery->m_dOverrides )
4104 pQuery->m_dOverrides[i].m_dValues.Sort ();
4105
4106 // fixup query
4107 pQuery->m_sQuery = pQuery->m_sRawQuery;
4108
4109 if ( pQuery->m_eMode==SPH_MATCH_BOOLEAN )
4110 pQuery->m_eRanker = SPH_RANK_NONE;
4111
4112 if ( pQuery->m_eMode==SPH_MATCH_FULLSCAN )
4113 pQuery->m_sQuery = "";
4114
4115 if ( pQuery->m_eMode!=SPH_MATCH_ALL && pQuery->m_eMode!=SPH_MATCH_ANY && pQuery->m_eMode!=SPH_MATCH_PHRASE )
4116 return;
4117
4118 const char * szQuery = pQuery->m_sRawQuery.cstr ();
4119 int iQueryLen = strlen(szQuery);
4120
4121 pQuery->m_sQuery.Reserve ( iQueryLen*2+8 );
4122 char * szRes = (char*) pQuery->m_sQuery.cstr ();
4123 char c;
4124
4125 if ( pQuery->m_eMode==SPH_MATCH_ANY || pQuery->m_eMode==SPH_MATCH_PHRASE )
4126 *szRes++ = '\"';
4127
4128 while ( ( c = *szQuery++ )!=0 )
4129 {
4130 // must be in sync with EscapeString (php api)
4131 const char sMagics[] = "<\\()|-!@~\"&/^$=";
4132 for ( const char * s = sMagics; *s; s++ )
4133 if ( c==*s )
4134 {
4135 *szRes++ = '\\';
4136 break;
4137 }
4138
4139 *szRes++ = c;
4140 }
4141
4142 switch ( pQuery->m_eMode )
4143 {
4144 case SPH_MATCH_ALL: pQuery->m_eRanker = SPH_RANK_PROXIMITY; *szRes = '\0'; break;
4145 case SPH_MATCH_ANY: pQuery->m_eRanker = SPH_RANK_MATCHANY; strncpy ( szRes, "\"/1", 8 ); break;
4146 case SPH_MATCH_PHRASE: pQuery->m_eRanker = SPH_RANK_PROXIMITY; *szRes++ = '\"'; *szRes = '\0'; break;
4147 default: return;
4148 }
4149 }
4150
4151
ParseSearchQuery(InputBuffer_c & tReq,CSphQuery & tQuery,int iVer,int iMasterVer)4152 bool ParseSearchQuery ( InputBuffer_c & tReq, CSphQuery & tQuery, int iVer, int iMasterVer )
4153 {
4154 tQuery.m_iOldVersion = iVer;
4155
4156 // v.1.0. mode, limits, weights, ID/TS ranges
4157 tQuery.m_iOffset = tReq.GetInt ();
4158 tQuery.m_iLimit = tReq.GetInt ();
4159 tQuery.m_eMode = (ESphMatchMode) tReq.GetInt ();
4160 if ( iVer>=0x110 )
4161 {
4162 tQuery.m_eRanker = (ESphRankMode) tReq.GetInt ();
4163 if ( tQuery.m_eRanker==SPH_RANK_EXPR )
4164 tQuery.m_sRankerExpr = tReq.GetString();
4165 }
4166 tQuery.m_eSort = (ESphSortOrder) tReq.GetInt ();
4167 if ( iVer<=0x101 )
4168 tQuery.m_iOldGroups = tReq.GetDwords ( &tQuery.m_pOldGroups, g_iMaxFilterValues, "invalid group count %d (should be in 0..%d range)" );
4169 if ( iVer>=0x102 )
4170 {
4171 tQuery.m_sSortBy = tReq.GetString ();
4172 tQuery.m_sSortBy.ToLower ();
4173 }
4174 tQuery.m_sRawQuery = tReq.GetString ();
4175 tQuery.m_iWeights = tReq.GetDwords ( (DWORD**)&tQuery.m_pWeights, SPH_MAX_FIELDS, "invalid weight count %d (should be in 0..%d range)" );
4176 tQuery.m_sIndexes = tReq.GetString ();
4177
4178 bool bIdrange64 = false;
4179 if ( iVer>=0x108 )
4180 bIdrange64 = ( tReq.GetInt()!=0 );
4181
4182 SphDocID_t uMinID = 0;
4183 SphDocID_t uMaxID = DOCID_MAX;
4184 if ( bIdrange64 )
4185 {
4186 uMinID = (SphDocID_t)tReq.GetUint64 ();
4187 uMaxID = (SphDocID_t)tReq.GetUint64 ();
4188 // FIXME? could report clamp here if I'm id32 and client passed id64 range,
4189 // but frequently this won't affect anything at all
4190 } else
4191 {
4192 uMinID = tReq.GetDword ();
4193 uMaxID = tReq.GetDword ();
4194 }
4195
4196 if ( iVer<0x108 && uMaxID==0xffffffffUL )
4197 uMaxID = 0; // fixup older clients which send 32-bit UINT_MAX by default
4198
4199 if ( uMaxID==0 )
4200 uMaxID = DOCID_MAX;
4201
4202 // v.1.0, v.1.1
4203 if ( iVer<=0x101 )
4204 {
4205 tQuery.m_iOldMinTS = tReq.GetDword ();
4206 tQuery.m_iOldMaxTS = tReq.GetDword ();
4207 }
4208
4209 // v.1.1 specific
4210 if ( iVer==0x101 )
4211 {
4212 tQuery.m_iOldMinGID = tReq.GetDword ();
4213 tQuery.m_iOldMaxGID = tReq.GetDword ();
4214 }
4215
4216 // v.1.2
4217 if ( iVer>=0x102 )
4218 {
4219 int iAttrFilters = tReq.GetInt ();
4220 if ( iAttrFilters>g_iMaxFilters )
4221 {
4222 tReq.SendErrorReply ( "too much attribute filters (req=%d, max=%d)", iAttrFilters, g_iMaxFilters );
4223 return false;
4224 }
4225
4226 const int MAX_ERROR_SET_BUFFER = 128;
4227 char sSetError[MAX_ERROR_SET_BUFFER];
4228 tQuery.m_dFilters.Resize ( iAttrFilters );
4229 ARRAY_FOREACH ( iFilter, tQuery.m_dFilters )
4230 {
4231 CSphFilterSettings & tFilter = tQuery.m_dFilters[iFilter];
4232 tFilter.m_sAttrName = tReq.GetString ();
4233 tFilter.m_sAttrName.ToLower ();
4234
4235 snprintf ( sSetError, MAX_ERROR_SET_BUFFER
4236 , "invalid attribute '%s'(%d) set length %s (should be in 0..%s range)", tFilter.m_sAttrName.cstr(), iFilter, "%d", "%d" );
4237
4238 if ( iVer>=0x10E )
4239 {
4240 // v.1.14+
4241 tFilter.m_eType = (ESphFilter) tReq.GetDword ();
4242 switch ( tFilter.m_eType )
4243 {
4244 case SPH_FILTER_RANGE:
4245 tFilter.m_iMinValue = ( iVer>=0x114 ) ? tReq.GetUint64() : tReq.GetDword ();
4246 tFilter.m_iMaxValue = ( iVer>=0x114 ) ? tReq.GetUint64() : tReq.GetDword ();
4247 break;
4248
4249 case SPH_FILTER_FLOATRANGE:
4250 tFilter.m_fMinValue = tReq.GetFloat ();
4251 tFilter.m_fMaxValue = tReq.GetFloat ();
4252 break;
4253
4254 case SPH_FILTER_VALUES:
4255 {
4256 bool bRes = ( iVer>=0x114 )
4257 ? tReq.GetQwords ( tFilter.m_dValues, g_iMaxFilterValues, sSetError )
4258 : tReq.GetDwords ( tFilter.m_dValues, g_iMaxFilterValues, sSetError );
4259 if ( !bRes )
4260 return false;
4261 }
4262 break;
4263
4264 default:
4265 tReq.SendErrorReply ( "unknown filter type (type-id=%d)", tFilter.m_eType );
4266 return false;
4267 }
4268
4269 } else
4270 {
4271 // pre-1.14
4272 if ( !tReq.GetDwords ( tFilter.m_dValues, g_iMaxFilterValues, sSetError ) )
4273 return false;
4274
4275 if ( !tFilter.m_dValues.GetLength() )
4276 {
4277 // 0 length means this is range, not set
4278 tFilter.m_iMinValue = tReq.GetDword ();
4279 tFilter.m_iMaxValue = tReq.GetDword ();
4280 }
4281
4282 tFilter.m_eType = tFilter.m_dValues.GetLength() ? SPH_FILTER_VALUES : SPH_FILTER_RANGE;
4283 }
4284
4285 if ( iVer>=0x106 )
4286 tFilter.m_bExclude = !!tReq.GetDword ();
4287 }
4288 }
4289
4290 // now add id range filter
4291 if ( uMinID!=0 || uMaxID!=DOCID_MAX )
4292 {
4293 CSphFilterSettings & tFilter = tQuery.m_dFilters.Add();
4294 tFilter.m_sAttrName = "@id";
4295 tFilter.m_eType = SPH_FILTER_RANGE;
4296 tFilter.m_iMinValue = uMinID;
4297 tFilter.m_iMaxValue = uMaxID;
4298 }
4299
4300 // v.1.3
4301 if ( iVer>=0x103 )
4302 {
4303 tQuery.m_eGroupFunc = (ESphGroupBy) tReq.GetDword ();
4304 tQuery.m_sGroupBy = tReq.GetString ();
4305 tQuery.m_sGroupBy.ToLower ();
4306 }
4307
4308 // v.1.4
4309 tQuery.m_iMaxMatches = g_iMaxMatches;
4310 if ( iVer>=0x104 )
4311 tQuery.m_iMaxMatches = tReq.GetInt ();
4312
4313 // v.1.5, v.1.7
4314 if ( iVer>=0x107 )
4315 {
4316 tQuery.m_sGroupSortBy = tReq.GetString ();
4317 } else if ( iVer>=0x105 )
4318 {
4319 bool bSortByGroup = ( tReq.GetInt()!=0 );
4320 if ( !bSortByGroup )
4321 {
4322 char sBuf[256];
4323 switch ( tQuery.m_eSort )
4324 {
4325 case SPH_SORT_RELEVANCE:
4326 tQuery.m_sGroupSortBy = "@weight desc";
4327 break;
4328
4329 case SPH_SORT_ATTR_DESC:
4330 case SPH_SORT_ATTR_ASC:
4331 snprintf ( sBuf, sizeof(sBuf), "%s %s", tQuery.m_sSortBy.cstr(),
4332 tQuery.m_eSort==SPH_SORT_ATTR_ASC ? "asc" : "desc" );
4333 tQuery.m_sGroupSortBy = sBuf;
4334 break;
4335
4336 case SPH_SORT_EXTENDED:
4337 tQuery.m_sGroupSortBy = tQuery.m_sSortBy;
4338 break;
4339
4340 default:
4341 tReq.SendErrorReply ( "INTERNAL ERROR: unsupported sort mode %d in groupby sort fixup", tQuery.m_eSort );
4342 return false;
4343 }
4344 }
4345 }
4346
4347 // v.1.9
4348 if ( iVer>=0x109 )
4349 tQuery.m_iCutoff = tReq.GetInt();
4350
4351 // v.1.10
4352 if ( iVer>=0x10A )
4353 {
4354 tQuery.m_iRetryCount = tReq.GetInt ();
4355 tQuery.m_iRetryDelay = tReq.GetInt ();
4356 }
4357
4358 // v.1.11
4359 if ( iVer>=0x10B )
4360 {
4361 tQuery.m_sGroupDistinct = tReq.GetString ();
4362 tQuery.m_sGroupDistinct.ToLower();
4363 }
4364
4365 // v.1.14
4366 if ( iVer>=0x10E )
4367 {
4368 tQuery.m_bGeoAnchor = ( tReq.GetInt()!=0 );
4369 if ( tQuery.m_bGeoAnchor )
4370 {
4371 tQuery.m_sGeoLatAttr = tReq.GetString ();
4372 tQuery.m_sGeoLongAttr = tReq.GetString ();
4373 tQuery.m_fGeoLatitude = tReq.GetFloat ();
4374 tQuery.m_fGeoLongitude = tReq.GetFloat ();
4375 }
4376 }
4377
4378 // v.1.15
4379 if ( iVer>=0x10F )
4380 {
4381 tQuery.m_dIndexWeights.Resize ( tReq.GetInt() ); // FIXME! add sanity check
4382 ARRAY_FOREACH ( i, tQuery.m_dIndexWeights )
4383 {
4384 tQuery.m_dIndexWeights[i].m_sName = tReq.GetString ();
4385 tQuery.m_dIndexWeights[i].m_iValue = tReq.GetInt ();
4386 }
4387 }
4388
4389 // v.1.17
4390 if ( iVer>=0x111 )
4391 tQuery.m_uMaxQueryMsec = tReq.GetDword ();
4392
4393 // v.1.18
4394 if ( iVer>=0x112 )
4395 {
4396 tQuery.m_dFieldWeights.Resize ( tReq.GetInt() ); // FIXME! add sanity check
4397 ARRAY_FOREACH ( i, tQuery.m_dFieldWeights )
4398 {
4399 tQuery.m_dFieldWeights[i].m_sName = tReq.GetString ();
4400 tQuery.m_dFieldWeights[i].m_iValue = tReq.GetInt ();
4401 }
4402 }
4403
4404 // v.1.19
4405 if ( iVer>=0x113 )
4406 tQuery.m_sComment = tReq.GetString ();
4407
4408 // v.1.21
4409 if ( iVer>=0x115 )
4410 {
4411 tQuery.m_dOverrides.Resize ( tReq.GetInt() ); // FIXME! add sanity check
4412 ARRAY_FOREACH ( i, tQuery.m_dOverrides )
4413 {
4414 CSphAttrOverride & tOverride = tQuery.m_dOverrides[i];
4415 tOverride.m_sAttr = tReq.GetString ();
4416 tOverride.m_eAttrType = (ESphAttr) tReq.GetDword ();
4417
4418 tOverride.m_dValues.Resize ( tReq.GetInt() ); // FIXME! add sanity check
4419 ARRAY_FOREACH ( iVal, tOverride.m_dValues )
4420 {
4421 CSphAttrOverride::IdValuePair_t & tEntry = tOverride.m_dValues[iVal];
4422 tEntry.m_uDocID = (SphDocID_t) tReq.GetUint64 ();
4423
4424 if ( tOverride.m_eAttrType==SPH_ATTR_FLOAT ) tEntry.m_fValue = tReq.GetFloat ();
4425 else if ( tOverride.m_eAttrType==SPH_ATTR_BIGINT ) tEntry.m_uValue = tReq.GetUint64 ();
4426 else tEntry.m_uValue = tReq.GetDword ();
4427 }
4428 }
4429 }
4430
4431 // v.1.22
4432 if ( iVer>=0x116 )
4433 {
4434 tQuery.m_sSelect = tReq.GetString ();
4435 tQuery.m_bAgent = ( iMasterVer>0 );
4436 CSphString sError;
4437 if ( !tQuery.ParseSelectList ( sError ) )
4438 {
4439 tReq.SendErrorReply ( "select: %s", sError.cstr() );
4440 return false;
4441 }
4442 }
4443
4444 // master v.1.0
4445 tQuery.m_eCollation = g_eCollation;
4446 if ( iMasterVer>=0x001 )
4447 {
4448 tQuery.m_eCollation = (ESphCollation)tReq.GetDword();
4449 }
4450
4451 /////////////////////
4452 // additional checks
4453 /////////////////////
4454
4455 if ( tReq.GetError() )
4456 {
4457 tReq.SendErrorReply ( "invalid or truncated request" );
4458 return false;
4459 }
4460
4461 CSphString sError;
4462 CheckQuery ( tQuery, sError );
4463 if ( !sError.IsEmpty() )
4464 {
4465 tReq.SendErrorReply ( "%s", sError.cstr() );
4466 return false;
4467 }
4468
4469 // now prepare it for the engine
4470 PrepareQueryEmulation ( &tQuery );
4471
4472 // all ok
4473 return true;
4474 }
4475
4476 //////////////////////////////////////////////////////////////////////////
4477
LogQueryPlain(const CSphQuery & tQuery,const CSphQueryResult & tRes)4478 void LogQueryPlain ( const CSphQuery & tQuery, const CSphQueryResult & tRes )
4479 {
4480 assert ( g_eLogFormat==LOG_FORMAT_PLAIN );
4481 if ( ( !g_bQuerySyslog && g_iQueryLogFile<0 ) || !tRes.m_sError.IsEmpty() )
4482 return;
4483
4484 CSphStringBuilder tBuf;
4485
4486 // [time]
4487 #if USE_SYSLOG
4488 if ( !g_bQuerySyslog )
4489 {
4490 #endif
4491
4492 char sTimeBuf[SPH_TIME_PID_MAX_SIZE];
4493 sphFormatCurrentTime ( sTimeBuf, sizeof(sTimeBuf) );
4494 tBuf.Appendf ( "[%s]", sTimeBuf );
4495
4496 #if USE_SYSLOG
4497 } else
4498 tBuf += "[query]";
4499 #endif
4500
4501 // querytime sec
4502 int iQueryTime = Max ( tRes.m_iQueryTime, 0 );
4503 tBuf.Appendf ( " %d.%03d sec", iQueryTime/1000, iQueryTime%1000 );
4504
4505 // optional multi-query multiplier
4506 if ( tRes.m_iMultiplier>1 )
4507 tBuf.Appendf ( " x%d", tRes.m_iMultiplier );
4508
4509 // [matchmode/numfilters/sortmode matches (offset,limit)
4510 static const char * sModes [ SPH_MATCH_TOTAL ] = { "all", "any", "phr", "bool", "ext", "scan", "ext2" };
4511 static const char * sSort [ SPH_SORT_TOTAL ] = { "rel", "attr-", "attr+", "tsegs", "ext", "expr" };
4512 tBuf.Appendf ( " [%s/%d/%s "INT64_FMT" (%d,%d)",
4513 sModes [ tQuery.m_eMode ], tQuery.m_dFilters.GetLength(), sSort [ tQuery.m_eSort ],
4514 tRes.m_iTotalMatches, tQuery.m_iOffset, tQuery.m_iLimit );
4515
4516 // optional groupby info
4517 if ( !tQuery.m_sGroupBy.IsEmpty() )
4518 tBuf.Appendf ( " @%s", tQuery.m_sGroupBy.cstr() );
4519
4520 // ] [indexes]
4521 tBuf.Appendf ( "] [%s]", tQuery.m_sIndexes.cstr() );
4522
4523 // optional performance counters
4524 if ( g_bIOStats || g_bCpuStats )
4525 {
4526 const CSphIOStats & IOStats = tRes.m_tIOStats;
4527
4528 tBuf += " [";
4529
4530 if ( g_bIOStats )
4531 tBuf.Appendf ( "ios=%d kb=%d.%d ioms=%d.%d",
4532 IOStats.m_iReadOps, (int)( IOStats.m_iReadBytes/1024 ), (int)( IOStats.m_iReadBytes%1024 )*10/1024,
4533 (int)( IOStats.m_iReadTime/1000 ), (int)( IOStats.m_iReadTime%1000 )/100 );
4534
4535 if ( g_bIOStats && g_bCpuStats )
4536 tBuf += " ";
4537
4538 if ( g_bCpuStats )
4539 tBuf.Appendf ( "cpums=%d.%d", (int)( tRes.m_iCpuTime/1000 ), (int)( tRes.m_iCpuTime%1000 )/100 );
4540
4541 tBuf += "]";
4542 }
4543
4544 // optional query comment
4545 if ( !tQuery.m_sComment.IsEmpty() )
4546 tBuf.Appendf ( " [%s]", tQuery.m_sComment.cstr() );
4547
4548 // query
4549 // (m_sRawQuery is empty when using MySQL handler)
4550 const CSphString & sQuery = tQuery.m_sRawQuery.IsEmpty()
4551 ? tQuery.m_sQuery
4552 : tQuery.m_sRawQuery;
4553
4554 if ( !sQuery.IsEmpty() )
4555 {
4556 tBuf += " ";
4557 tBuf.AppendEscaped ( sQuery.cstr(), false, true );
4558 }
4559
4560 #if USE_SYSLOG
4561 if ( !g_bQuerySyslog )
4562 {
4563 #endif
4564
4565 // line feed
4566 tBuf += "\n";
4567
4568 lseek ( g_iQueryLogFile, 0, SEEK_END );
4569 sphWrite ( g_iQueryLogFile, tBuf.cstr(), tBuf.Length() );
4570
4571 #if USE_SYSLOG
4572 } else
4573 {
4574 syslog ( LOG_INFO, "%s", tBuf.cstr() );
4575 }
4576 #endif
4577 }
4578
4579
FormatOrderBy(CSphStringBuilder * pBuf,const char * sPrefix,ESphSortOrder eSort,const CSphString & sSort)4580 void FormatOrderBy ( CSphStringBuilder * pBuf, const char * sPrefix, ESphSortOrder eSort, const CSphString & sSort )
4581 {
4582 assert ( pBuf );
4583 if ( eSort==SPH_SORT_EXTENDED && sSort=="@weight desc" )
4584 return;
4585
4586 switch ( eSort )
4587 {
4588 case SPH_SORT_ATTR_DESC: pBuf->Appendf ( " %s %s DESC", sPrefix, sSort.cstr() ); break;
4589 case SPH_SORT_ATTR_ASC: pBuf->Appendf ( " %s %s ASC", sPrefix, sSort.cstr() ); break;
4590 case SPH_SORT_TIME_SEGMENTS: pBuf->Appendf ( " %s TIME_SEGMENT(%s)", sPrefix, sSort.cstr() ); break;
4591 case SPH_SORT_EXTENDED: pBuf->Appendf ( " %s %s", sPrefix, sSort.cstr() ); break;
4592 case SPH_SORT_EXPR: pBuf->Appendf ( " %s BUILTIN_EXPR()", sPrefix ); break;
4593 default: pBuf->Appendf ( " %s mode-%d", sPrefix, (int)eSort ); break;
4594 }
4595 }
4596
4597
LogQuerySphinxql(const CSphQuery & q,const CSphQueryResult & tRes,const CSphVector<int64_t> & dAgentTimes)4598 void LogQuerySphinxql ( const CSphQuery & q, const CSphQueryResult & tRes, const CSphVector<int64_t> & dAgentTimes )
4599 {
4600 assert ( g_eLogFormat==LOG_FORMAT_SPHINXQL );
4601 if ( g_iQueryLogFile<0 )
4602 return;
4603
4604 CSphStringBuilder tBuf;
4605
4606 // get connection id
4607 int iCid = ( g_eWorkers!=MPM_THREADS ) ? g_iConnID : *(int*) sphThreadGet ( g_tConnKey );
4608
4609 // time, conn id, wall, found
4610 int iQueryTime = Max ( tRes.m_iQueryTime, 0 );
4611
4612 char sTimeBuf[SPH_TIME_PID_MAX_SIZE];
4613 sphFormatCurrentTime ( sTimeBuf, sizeof(sTimeBuf) );
4614
4615 tBuf += "/""* ";
4616 tBuf += sTimeBuf;
4617
4618 if ( tRes.m_iMultiplier>1 )
4619 tBuf.Appendf ( " conn %d wall %d.%03d x%d found "INT64_FMT" *""/ ",
4620 iCid, iQueryTime/1000, iQueryTime%1000, tRes.m_iMultiplier, tRes.m_iTotalMatches );
4621 else
4622 tBuf.Appendf ( " conn %d wall %d.%03d found "INT64_FMT" *""/ ",
4623 iCid, iQueryTime/1000, iQueryTime%1000, tRes.m_iTotalMatches );
4624
4625 ///////////////////////////////////
4626 // format request as SELECT query
4627 ///////////////////////////////////
4628
4629
4630 CSphString sIndexes = "";
4631 if ( q.m_sIndexes=="*" )
4632 {
4633 // search through all local indexes
4634
4635 for ( IndexHashIterator_c it ( g_pIndexes ); it.Next(); )
4636 {
4637 if ( it.Get ().m_bEnabled )
4638 {
4639 if ( sIndexes.IsEmpty() )
4640 sIndexes = it.GetKey();
4641 else
4642 sIndexes.SetSprintf ( "%s,%s", sIndexes.cstr(), it.GetKey().cstr() );
4643 }
4644 }
4645 } else
4646 sIndexes = q.m_sIndexes;
4647
4648 CSphString sGeodist = "";
4649 ARRAY_FOREACH ( i, q.m_dFilters )
4650 {
4651 if ( q.m_dFilters[i].m_sAttrName=="@geodist" )
4652 sGeodist.SetSprintf ( ", GEODIST(%s,%s,%f,%f) AS _geodist", q.m_sGeoLatAttr.cstr(), q.m_sGeoLongAttr.cstr(), q.m_fGeoLatitude, q.m_fGeoLongitude );
4653 }
4654
4655
4656 tBuf.Appendf ( "SELECT %s%s FROM %s", q.m_sSelect.cstr(), sGeodist.cstr(), sIndexes.cstr() );
4657
4658 // WHERE clause
4659 // (m_sRawQuery is empty when using MySQL handler)
4660 const CSphString & sQuery = q.m_sQuery;
4661 if ( !sQuery.IsEmpty() || q.m_dFilters.GetLength() )
4662 {
4663 bool bDeflowered = false;
4664
4665 tBuf += " WHERE";
4666 if ( !sQuery.IsEmpty() )
4667 {
4668 tBuf += " MATCH('";
4669 tBuf.AppendEscaped ( sQuery.cstr() );
4670 tBuf += "')";
4671 bDeflowered = true;
4672 }
4673
4674 ARRAY_FOREACH ( i, q.m_dFilters )
4675 {
4676 if ( bDeflowered )
4677 tBuf += " AND";
4678
4679 const CSphFilterSettings & f = q.m_dFilters[i];
4680
4681 const char* sAttr = "_geodist";
4682 if ( f.m_sAttrName!="@geodist" )
4683 sAttr = f.m_sAttrName.cstr();
4684
4685
4686 switch ( f.m_eType )
4687 {
4688 case SPH_FILTER_VALUES:
4689 if ( f.m_dValues.GetLength()==1 )
4690 {
4691 if ( f.m_bExclude )
4692 tBuf.Appendf ( " %s!="INT64_FMT, sAttr, (int64_t)f.m_dValues[0] );
4693 else
4694 tBuf.Appendf ( " %s="INT64_FMT, sAttr, (int64_t)f.m_dValues[0] );
4695 } else
4696 {
4697 if ( f.m_bExclude )
4698 tBuf.Appendf ( " %s NOT IN (", sAttr );
4699 else
4700 tBuf.Appendf ( " %s IN (", sAttr );
4701
4702 ARRAY_FOREACH ( j, f.m_dValues )
4703 {
4704 if ( j )
4705 tBuf.Appendf ( ","INT64_FMT, (int64_t)f.m_dValues[j] );
4706 else
4707 tBuf.Appendf ( INT64_FMT, (int64_t)f.m_dValues[j] );
4708 }
4709 tBuf += ")";
4710 }
4711 break;
4712
4713 case SPH_FILTER_RANGE:
4714 if ( f.m_bExclude )
4715 tBuf.Appendf ( " %s NOT BETWEEN "INT64_FMT" AND "INT64_FMT,
4716 sAttr, f.m_iMinValue, f.m_iMaxValue );
4717 else
4718 tBuf.Appendf ( " %s BETWEEN "INT64_FMT" AND "INT64_FMT,
4719 sAttr, f.m_iMinValue, f.m_iMaxValue );
4720 break;
4721
4722 case SPH_FILTER_FLOATRANGE:
4723 if ( f.m_bExclude )
4724 tBuf.Appendf ( " %s NOT BETWEEN %f AND %f",
4725 sAttr, f.m_fMinValue, f.m_fMaxValue );
4726 else
4727 tBuf.Appendf ( " %s BETWEEN %f AND %f",
4728 sAttr, f.m_fMinValue, f.m_fMaxValue );
4729 break;
4730
4731 default:
4732 tBuf += " 1 /""* oops, unknown filter type *""/";
4733 break;
4734 }
4735 }
4736 }
4737
4738 // ORDER BY and/or GROUP BY clause
4739 if ( q.m_sGroupBy.IsEmpty() )
4740 {
4741 if ( !q.m_sSortBy.IsEmpty() ) // case API SPH_MATCH_EXTENDED2 - SPH_SORT_RELEVANCE
4742 FormatOrderBy ( &tBuf, " ORDER BY", q.m_eSort, q.m_sSortBy );
4743 } else
4744 {
4745 tBuf.Appendf ( " GROUP BY %s", q.m_sGroupBy.cstr() );
4746 FormatOrderBy ( &tBuf, "WITHIN GROUP ORDER BY", q.m_eSort, q.m_sSortBy );
4747 if ( q.m_sGroupSortBy!="@group desc" )
4748 FormatOrderBy ( &tBuf, "ORDER BY", SPH_SORT_EXTENDED, q.m_sGroupSortBy );
4749 }
4750
4751 // LIMIT clause
4752 if ( q.m_iOffset!=0 || q.m_iLimit!=20 )
4753 tBuf.Appendf ( " LIMIT %d,%d", q.m_iOffset, q.m_iLimit );
4754
4755 // OPTION clause
4756 int iOpts = 0;
4757
4758 if ( q.m_iMaxMatches!=1000 )
4759 {
4760 tBuf.Appendf ( iOpts++ ? ", " : " OPTION " );
4761 tBuf.Appendf ( "max_matches=%d", q.m_iMaxMatches );
4762 }
4763
4764 if ( !q.m_sComment.IsEmpty() )
4765 {
4766 tBuf.Appendf ( iOpts++ ? ", " : " OPTION " );
4767 tBuf.Appendf ( "comment='%s'", q.m_sComment.cstr() ); // FIXME! escape, replace newlines..
4768 }
4769
4770 if ( q.m_eRanker!=SPH_RANK_DEFAULT )
4771 {
4772 const char * sRanker = "proximity_bm25";
4773 switch ( q.m_eRanker )
4774 {
4775 case SPH_RANK_BM25: sRanker = "bm25"; break;
4776 case SPH_RANK_NONE: sRanker = "none"; break;
4777 case SPH_RANK_WORDCOUNT: sRanker = "wordcount"; break;
4778 case SPH_RANK_PROXIMITY: sRanker = "proximity"; break;
4779 case SPH_RANK_MATCHANY: sRanker = "matchany"; break;
4780 case SPH_RANK_FIELDMASK: sRanker = "fieldmask"; break;
4781 case SPH_RANK_SPH04: sRanker = "sph04"; break;
4782 case SPH_RANK_EXPR: sRanker = "expr"; break;
4783 default: break;
4784 }
4785
4786 tBuf.Appendf ( iOpts++ ? ", " : " OPTION " );
4787 tBuf.Appendf ( "ranker=%s", sRanker );
4788 }
4789
4790 // finish SQL statement
4791 tBuf += ";";
4792
4793 ///////////////
4794 // query stats
4795 ///////////////
4796
4797 if ( !tRes.m_sError.IsEmpty() )
4798 {
4799 // all we have is an error
4800 tBuf.Appendf ( " # error=%s", tRes.m_sError.cstr() );
4801
4802 } else if ( g_bIOStats || g_bCpuStats || dAgentTimes.GetLength() || !tRes.m_sWarning.IsEmpty() )
4803 {
4804 // got some extra data, add a comment
4805 tBuf += " #";
4806
4807 // performance counters
4808 if ( g_bIOStats || g_bCpuStats )
4809 {
4810 const CSphIOStats & IOStats = tRes.m_tIOStats;
4811
4812 if ( g_bIOStats )
4813 tBuf.Appendf ( " ios=%d kb=%d.%d ioms=%d.%d",
4814 IOStats.m_iReadOps, (int)( IOStats.m_iReadBytes/1024 ), (int)( IOStats.m_iReadBytes%1024 )*10/1024,
4815 (int)( IOStats.m_iReadTime/1000 ), (int)( IOStats.m_iReadTime%1000 )/100 );
4816
4817 if ( g_bCpuStats )
4818 tBuf.Appendf ( " cpums=%d.%d", (int)( tRes.m_iCpuTime/1000 ), (int)( tRes.m_iCpuTime%1000 )/100 );
4819 }
4820
4821 // per-agent times
4822 if ( dAgentTimes.GetLength() )
4823 {
4824 tBuf += " agents=(";
4825 ARRAY_FOREACH ( i, dAgentTimes )
4826 tBuf.Appendf ( i ? ", %d.%03d" : "%d.%03d",
4827 (int)(dAgentTimes[i]/1000000),
4828 (int)((dAgentTimes[i]/1000)%1000) );
4829
4830 tBuf += ")";
4831 }
4832
4833 // warning
4834 if ( !tRes.m_sWarning.IsEmpty() )
4835 tBuf.Appendf ( " warning=%s", tRes.m_sWarning.cstr() );
4836 }
4837
4838 // line feed
4839 tBuf += "\n";
4840
4841 lseek ( g_iQueryLogFile, 0, SEEK_END );
4842 sphWrite ( g_iQueryLogFile, tBuf.cstr(), tBuf.Length() );
4843 }
4844
4845
LogQuery(const CSphQuery & q,const CSphQueryResult & tRes,const CSphVector<int64_t> & dAgentTimes)4846 void LogQuery ( const CSphQuery & q, const CSphQueryResult & tRes, const CSphVector<int64_t> & dAgentTimes )
4847 {
4848 switch ( g_eLogFormat )
4849 {
4850 case LOG_FORMAT_PLAIN: LogQueryPlain ( q, tRes ); break;
4851 case LOG_FORMAT_SPHINXQL: LogQuerySphinxql ( q, tRes, dAgentTimes ); break;
4852 }
4853 }
4854
4855
LogSphinxqlError(const char * sStmt,const char * sError)4856 void LogSphinxqlError ( const char * sStmt, const char * sError )
4857 {
4858 if ( g_eLogFormat!=LOG_FORMAT_SPHINXQL || g_iQueryLogFile<0 || !sStmt || !sError )
4859 return;
4860
4861 // time, conn id, query, error
4862 CSphStringBuilder tBuf;
4863
4864 int iCid = ( g_eWorkers!=MPM_THREADS ) ? g_iConnID : *(int*) sphThreadGet ( g_tConnKey );
4865
4866 char sTimeBuf[SPH_TIME_PID_MAX_SIZE];
4867 sphFormatCurrentTime ( sTimeBuf, sizeof(sTimeBuf) );
4868
4869 tBuf += "/""* ";
4870 tBuf += sTimeBuf;
4871 tBuf.Appendf ( " conn %d *""/ %s # error=%s\n", iCid, sStmt, sError );
4872
4873 lseek ( g_iQueryLogFile, 0, SEEK_END );
4874 sphWrite ( g_iQueryLogFile, tBuf.cstr(), tBuf.Length() );
4875 }
4876
4877 //////////////////////////////////////////////////////////////////////////
4878
4879 // internals attributes are last no need to send them
SendGetAttrCount(const CSphSchema & tSchema)4880 static int SendGetAttrCount ( const CSphSchema & tSchema )
4881 {
4882 int iCount = tSchema.GetAttrsCount();
4883 if ( iCount
4884 && sphIsSortStringInternal ( tSchema.GetAttr ( iCount-1 ).m_sName.cstr() ) )
4885 {
4886 for ( int i=iCount-1; i>=0 && sphIsSortStringInternal ( tSchema.GetAttr(i).m_sName.cstr() ); i-- )
4887 {
4888 iCount = i;
4889 }
4890 }
4891
4892 return iCount;
4893 }
4894
4895
CalcResultLength(int iVer,const CSphQueryResult * pRes,const CSphVector<PoolPtrs_t> & dTag2Pools,bool bExtendedStat)4896 int CalcResultLength ( int iVer, const CSphQueryResult * pRes, const CSphVector<PoolPtrs_t> & dTag2Pools, bool bExtendedStat )
4897 {
4898 int iRespLen = 0;
4899
4900 // query status
4901 if ( iVer>=0x10D )
4902 {
4903 // multi-query status
4904 iRespLen += 4; // status code
4905
4906 if ( !pRes->m_sError.IsEmpty() )
4907 return iRespLen + 4 +strlen ( pRes->m_sError.cstr() );
4908
4909 if ( !pRes->m_sWarning.IsEmpty() )
4910 iRespLen += 4+strlen ( pRes->m_sWarning.cstr() );
4911
4912 } else if ( iVer>=0x106 )
4913 {
4914 // warning message
4915 if ( !pRes->m_sWarning.IsEmpty() )
4916 iRespLen += 4 + strlen ( pRes->m_sWarning.cstr() );
4917 }
4918
4919 // query stats
4920 iRespLen += 20;
4921
4922 int iAttrsCount = SendGetAttrCount ( pRes->m_tSchema );
4923
4924 // schema
4925 if ( iVer>=0x102 )
4926 {
4927 iRespLen += 8; // 4 for field count, 4 for attr count
4928 ARRAY_FOREACH ( i, pRes->m_tSchema.m_dFields )
4929 iRespLen += 4 + strlen ( pRes->m_tSchema.m_dFields[i].m_sName.cstr() ); // namelen, name
4930 for ( int i=0; i<iAttrsCount; i++ )
4931 iRespLen += 8 + strlen ( pRes->m_tSchema.GetAttr(i).m_sName.cstr() ); // namelen, name, type
4932 }
4933
4934 // matches
4935 if ( iVer<0x102 )
4936 iRespLen += 16*pRes->m_iCount; // matches
4937 else if ( iVer<0x108 )
4938 iRespLen += ( 8+4*iAttrsCount )*pRes->m_iCount; // matches
4939 else
4940 iRespLen += 4 + ( 8+4*USE_64BIT+4*iAttrsCount )*pRes->m_iCount; // id64 tag and matches
4941
4942 if ( iVer>=0x114 )
4943 {
4944 // 64bit matches
4945 int iWideAttrs = 0;
4946 for ( int i=0; i<iAttrsCount; i++ )
4947 if ( pRes->m_tSchema.GetAttr(i).m_eAttrType==SPH_ATTR_BIGINT )
4948 iWideAttrs++;
4949 iRespLen += 4*pRes->m_iCount*iWideAttrs; // extra 4 bytes per attr per match
4950 }
4951
4952 // agents send additional flag from words statistics
4953 if ( bExtendedStat )
4954 iRespLen += pRes->m_hWordStats.GetLength();
4955
4956 pRes->m_hWordStats.IterateStart();
4957 while ( pRes->m_hWordStats.IterateNext() ) // per-word stats
4958 iRespLen += 12 + strlen ( pRes->m_hWordStats.IterateGetKey().cstr() ); // wordlen, word, docs, hits
4959
4960 // MVA and string values
4961 CSphVector<CSphAttrLocator> dMvaItems;
4962 CSphVector<CSphAttrLocator> dStringItems;
4963 for ( int i=0; i<iAttrsCount; i++ )
4964 {
4965 const CSphColumnInfo & tCol = pRes->m_tSchema.GetAttr(i);
4966 if ( tCol.m_eAttrType==SPH_ATTR_UINT32SET || tCol.m_eAttrType==SPH_ATTR_INT64SET )
4967 dMvaItems.Add ( tCol.m_tLocator );
4968 if ( tCol.m_eAttrType==SPH_ATTR_STRING )
4969 dStringItems.Add ( tCol.m_tLocator );
4970 }
4971
4972 if ( iVer>=0x10C && dMvaItems.GetLength() )
4973 {
4974 for ( int i=0; i<pRes->m_iCount; i++ )
4975 {
4976 const CSphMatch & tMatch = pRes->m_dMatches [ pRes->m_iOffset+i ];
4977 const DWORD * pMvaPool = dTag2Pools [ tMatch.m_iTag ].m_pMva;
4978 ARRAY_FOREACH ( j, dMvaItems )
4979 {
4980 assert ( tMatch.GetAttr ( dMvaItems[j] )==0 || pMvaPool );
4981 const DWORD * pMva = tMatch.GetAttrMVA ( dMvaItems[j], pMvaPool );
4982 if ( pMva )
4983 iRespLen += pMva[0]*4; // FIXME? maybe add some sanity check here
4984 }
4985 }
4986 }
4987
4988 if ( iVer>=0x117 && dStringItems.GetLength() )
4989 {
4990 for ( int i=0; i<pRes->m_iCount; i++ )
4991 {
4992 const CSphMatch & tMatch = pRes->m_dMatches [ pRes->m_iOffset+i ];
4993 const BYTE * pStrings = dTag2Pools [ tMatch.m_iTag ].m_pStrings;
4994 ARRAY_FOREACH ( j, dStringItems )
4995 {
4996 DWORD uOffset = (DWORD) tMatch.GetAttr ( dStringItems[j] );
4997 assert ( !uOffset || pStrings );
4998 if ( uOffset ) // magic zero
4999 iRespLen += sphUnpackStr ( pStrings+uOffset, NULL );
5000 }
5001 }
5002 }
5003
5004 return iRespLen;
5005 }
5006
5007
SendResult(int iVer,NetOutputBuffer_c & tOut,const CSphQueryResult * pRes,const CSphVector<PoolPtrs_t> & dTag2Pools,bool bExtendedStat)5008 void SendResult ( int iVer, NetOutputBuffer_c & tOut, const CSphQueryResult * pRes, const CSphVector<PoolPtrs_t> & dTag2Pools, bool bExtendedStat )
5009 {
5010 // status
5011 if ( iVer>=0x10D )
5012 {
5013 // multi-query status
5014 bool bError = !pRes->m_sError.IsEmpty();
5015 bool bWarning = !bError && !pRes->m_sWarning.IsEmpty();
5016
5017 if ( bError )
5018 {
5019 tOut.SendInt ( SEARCHD_ERROR );
5020 tOut.SendString ( pRes->m_sError.cstr() );
5021 if ( g_bOptNoDetach && g_eLogFormat!=LOG_FORMAT_SPHINXQL )
5022 sphInfo ( "query error: %s", pRes->m_sError.cstr() );
5023 return;
5024
5025 } else if ( bWarning )
5026 {
5027 tOut.SendInt ( SEARCHD_WARNING );
5028 tOut.SendString ( pRes->m_sWarning.cstr() );
5029 if ( g_bOptNoDetach && g_eLogFormat!=LOG_FORMAT_SPHINXQL )
5030 sphInfo ( "query warning: %s", pRes->m_sWarning.cstr() );
5031 } else
5032 {
5033 tOut.SendInt ( SEARCHD_OK );
5034 }
5035
5036 } else
5037 {
5038 // single-query warning
5039 if ( iVer>=0x106 && !pRes->m_sWarning.IsEmpty() )
5040 tOut.SendString ( pRes->m_sWarning.cstr() );
5041 }
5042
5043 int iAttrsCount = SendGetAttrCount ( pRes->m_tSchema );
5044
5045 // send schema
5046 if ( iVer>=0x102 )
5047 {
5048 tOut.SendInt ( pRes->m_tSchema.m_dFields.GetLength() );
5049 ARRAY_FOREACH ( i, pRes->m_tSchema.m_dFields )
5050 tOut.SendString ( pRes->m_tSchema.m_dFields[i].m_sName.cstr() );
5051
5052 tOut.SendInt ( iAttrsCount );
5053 for ( int i=0; i<iAttrsCount; i++ )
5054 {
5055 const CSphColumnInfo & tCol = pRes->m_tSchema.GetAttr(i);
5056 tOut.SendString ( tCol.m_sName.cstr() );
5057 tOut.SendDword ( (DWORD)tCol.m_eAttrType );
5058 }
5059 }
5060
5061 // send matches
5062 CSphAttrLocator iGIDLoc, iTSLoc;
5063 if ( iVer<=0x101 )
5064 {
5065 for ( int i=0; i<pRes->m_tSchema.GetAttrsCount(); i++ )
5066 {
5067 const CSphColumnInfo & tAttr = pRes->m_tSchema.GetAttr(i);
5068
5069 if ( iTSLoc.m_iBitOffset<0 && tAttr.m_eAttrType==SPH_ATTR_TIMESTAMP )
5070 iTSLoc = tAttr.m_tLocator;
5071
5072 if ( iGIDLoc.m_iBitOffset<0 && tAttr.m_eAttrType==SPH_ATTR_INTEGER )
5073 iGIDLoc = tAttr.m_tLocator;
5074 }
5075 }
5076
5077 tOut.SendInt ( pRes->m_iCount );
5078 if ( iVer>=0x108 )
5079 tOut.SendInt ( USE_64BIT );
5080
5081 for ( int i=0; i<pRes->m_iCount; i++ )
5082 {
5083 const CSphMatch & tMatch = pRes->m_dMatches [ pRes->m_iOffset+i ];
5084 #if USE_64BIT
5085 if ( iVer>=0x108 )
5086 tOut.SendUint64 ( tMatch.m_iDocID );
5087 else
5088 #endif
5089 tOut.SendDword ( (DWORD)tMatch.m_iDocID );
5090
5091 if ( iVer<=0x101 )
5092 {
5093 tOut.SendDword ( iGIDLoc.m_iBitOffset>=0 ? (DWORD) tMatch.GetAttr ( iGIDLoc ) : 1 );
5094 tOut.SendDword ( iTSLoc.m_iBitOffset>=0 ? (DWORD) tMatch.GetAttr ( iTSLoc ) : 1 );
5095 tOut.SendInt ( tMatch.m_iWeight );
5096 } else
5097 {
5098 tOut.SendInt ( tMatch.m_iWeight );
5099
5100 const DWORD * pMvaPool = dTag2Pools [ tMatch.m_iTag ].m_pMva;
5101 const BYTE * pStrings = dTag2Pools [ tMatch.m_iTag ].m_pStrings;
5102
5103 assert ( tMatch.m_pStatic || !pRes->m_tSchema.GetStaticSize() );
5104 #if 0
5105 // not correct any more because of internal attrs (such as string sorting ptrs)
5106 assert ( tMatch.m_pDynamic || !pRes->m_tSchema.GetDynamicSize() );
5107 assert ( !tMatch.m_pDynamic || (int)tMatch.m_pDynamic[-1]==pRes->m_tSchema.GetDynamicSize() );
5108 #endif
5109
5110 for ( int j=0; j<iAttrsCount; j++ )
5111 {
5112 const CSphColumnInfo & tAttr = pRes->m_tSchema.GetAttr(j);
5113 if ( tAttr.m_eAttrType==SPH_ATTR_UINT32SET || tAttr.m_eAttrType==SPH_ATTR_INT64SET )
5114 {
5115 assert ( tMatch.GetAttr ( tAttr.m_tLocator )==0 || pMvaPool );
5116 const DWORD * pValues = tMatch.GetAttrMVA ( tAttr.m_tLocator, pMvaPool );
5117 if ( iVer<0x10C || !pValues )
5118 {
5119 // for older clients, fixups column value to 0
5120 // for newer clients, means that there are 0 values
5121 tOut.SendDword ( 0 );
5122 } else
5123 {
5124 // send MVA values
5125 int iValues = *pValues++;
5126 tOut.SendDword ( iValues );
5127 if ( tAttr.m_eAttrType==SPH_ATTR_INT64SET )
5128 {
5129 assert ( ( iValues%2 )==0 );
5130 while ( iValues )
5131 {
5132 uint64_t uVal = (uint64_t)MVA_UPSIZE ( pValues );
5133 tOut.SendUint64 ( uVal );
5134 pValues += 2;
5135 iValues -= 2;
5136 }
5137 } else
5138 {
5139 while ( iValues-- )
5140 tOut.SendDword ( *pValues++ );
5141 }
5142 }
5143
5144 } else if ( tAttr.m_eAttrType==SPH_ATTR_STRING )
5145 {
5146 // send string attr
5147 if ( iVer<0x117 )
5148 {
5149 // for older clients, just send int value of 0
5150 tOut.SendDword ( 0 );
5151 } else
5152 {
5153 // for newer clients, send binary string
5154 DWORD uOffset = (DWORD) tMatch.GetAttr ( tAttr.m_tLocator );
5155 if ( !uOffset ) // magic zero
5156 {
5157 tOut.SendDword ( 0 ); // null string
5158 } else
5159 {
5160 const BYTE * pStr;
5161 assert ( pStrings );
5162 int iLen = sphUnpackStr ( pStrings+uOffset, &pStr );
5163 tOut.SendDword ( iLen );
5164 tOut.SendBytes ( pStr, iLen );
5165 }
5166 }
5167
5168 } else
5169 {
5170 // send plain attr
5171 if ( tAttr.m_eAttrType==SPH_ATTR_FLOAT )
5172 tOut.SendFloat ( tMatch.GetAttrFloat ( tAttr.m_tLocator ) );
5173 else if ( iVer>=0x114 && tAttr.m_eAttrType==SPH_ATTR_BIGINT )
5174 tOut.SendUint64 ( tMatch.GetAttr ( tAttr.m_tLocator ) );
5175 else
5176 tOut.SendDword ( (DWORD)tMatch.GetAttr ( tAttr.m_tLocator ) );
5177 }
5178 }
5179 }
5180 }
5181 tOut.SendInt ( pRes->m_dMatches.GetLength() );
5182 tOut.SendAsDword ( pRes->m_iTotalMatches );
5183 tOut.SendInt ( Max ( pRes->m_iQueryTime, 0 ) );
5184 tOut.SendInt ( pRes->m_hWordStats.GetLength() );
5185
5186 pRes->m_hWordStats.IterateStart();
5187 while ( pRes->m_hWordStats.IterateNext() )
5188 {
5189 const CSphQueryResultMeta::WordStat_t & tStat = pRes->m_hWordStats.IterateGet();
5190 tOut.SendString ( pRes->m_hWordStats.IterateGetKey().cstr() );
5191 tOut.SendAsDword ( tStat.m_iDocs );
5192 tOut.SendAsDword ( tStat.m_iHits );
5193 if ( bExtendedStat )
5194 tOut.SendByte ( tStat.m_bExpanded );
5195 }
5196 }
5197
5198 /////////////////////////////////////////////////////////////////////////////
5199
5200 struct AggrResult_t : CSphQueryResult
5201 {
5202 int m_iTag; ///< current tag
5203 CSphVector<CSphSchema> m_dSchemas; ///< aggregated resultsets schemas (for schema minimization)
5204 CSphVector<int> m_dMatchCounts; ///< aggregated resultsets lengths (for schema minimization)
5205 CSphVector<const CSphIndex*> m_dLockedAttrs; ///< indexes which are hold in the memory untill sending result
5206 CSphVector<PoolPtrs_t> m_dTag2Pools; ///< tag to MVA and strings storage pools mapping
5207 };
5208
5209
5210 struct TaggedMatchSorter_fn : public SphAccessor_T<CSphMatch>
5211 {
CopyKeyTaggedMatchSorter_fn5212 void CopyKey ( CSphMatch * pMed, CSphMatch * pVal ) const
5213 {
5214 pMed->m_iDocID = pVal->m_iDocID;
5215 pMed->m_iTag = pVal->m_iTag;
5216 }
5217
IsLessTaggedMatchSorter_fn5218 bool IsLess ( const CSphMatch & a, const CSphMatch & b ) const
5219 {
5220 return ( a.m_iDocID < b.m_iDocID ) || ( a.m_iDocID==b.m_iDocID && a.m_iTag > b.m_iTag );
5221 }
5222
5223 // inherited swap does not work on gcc
SwapTaggedMatchSorter_fn5224 void Swap ( CSphMatch * a, CSphMatch * b ) const
5225 {
5226 ::Swap ( *a, *b );
5227 }
5228 };
5229
5230 // just to avoid the const_cast of the schema (i.e, return writable columns)
5231 // also to make possible several members refer to one and same locator.
5232 class CVirtualSchema : public CSphSchema
5233 {
5234 public:
LastColumn()5235 inline CSphColumnInfo & LastColumn() { return m_dAttrs.Last(); }
GetWAttr(int iIndex)5236 inline CSphColumnInfo & GetWAttr ( int iIndex ) { return m_dAttrs[iIndex]; }
GetWAttrs()5237 inline CSphVector<CSphColumnInfo> & GetWAttrs () { return m_dAttrs; }
AlignSizes(const CSphSchema & tProof)5238 inline void AlignSizes ( const CSphSchema& tProof )
5239 {
5240 m_dDynamicUsed.Resize ( tProof.GetDynamicSize() );
5241 m_iStaticSize = tProof.GetStaticSize();
5242 }
5243 };
5244
MkIdAttribute(CSphColumnInfo * pId)5245 void MkIdAttribute ( CSphColumnInfo * pId )
5246 {
5247 pId->m_tLocator.m_bDynamic = true;
5248 pId->m_sName = "id";
5249 pId->m_eAttrType = USE_64BIT ? SPH_ATTR_BIGINT : SPH_ATTR_INTEGER;
5250 pId->m_tLocator.m_iBitOffset = -8*(int)sizeof(SphDocID_t);
5251 pId->m_tLocator.m_iBitCount = 8*sizeof(SphDocID_t);
5252 }
5253
AddIDAttribute(CVirtualSchema * pSchema)5254 void AddIDAttribute ( CVirtualSchema * pSchema )
5255 {
5256 assert ( pSchema );
5257 if ( pSchema->GetAttrIndex("id")>=0 )
5258 return;
5259
5260 CSphColumnInfo tId;
5261 MkIdAttribute ( &tId );
5262 pSchema->GetWAttrs().Insert ( 0, tId );
5263 }
5264
IsIDAttribute(const CSphColumnInfo & tTarget)5265 inline bool IsIDAttribute ( const CSphColumnInfo & tTarget )
5266 {
5267 return tTarget.m_tLocator.IsID();
5268 }
5269
5270 // swap the schema into the new one
AdoptSchema(AggrResult_t * pRes,CSphSchema * pSchema)5271 void AdoptSchema ( AggrResult_t * pRes, CSphSchema * pSchema )
5272 {
5273 pSchema->m_dFields = pRes->m_tSchema.m_dFields;
5274 pRes->m_tSchema = *pSchema;
5275 }
5276
AdoptAliasedSchema(AggrResult_t & tRes,CVirtualSchema * pSchema)5277 void AdoptAliasedSchema ( AggrResult_t & tRes, CVirtualSchema * pSchema )
5278 {
5279 pSchema->AlignSizes ( tRes.m_tSchema );
5280 AdoptSchema ( &tRes, pSchema );
5281 }
5282
RemapResult(CSphSchema * pTarget,AggrResult_t * pRes,bool bMultiSchema=true)5283 void RemapResult ( CSphSchema * pTarget, AggrResult_t * pRes, bool bMultiSchema=true )
5284 {
5285 int iCur = 0;
5286 CSphVector<int> dMapFrom ( pTarget->GetAttrsCount() );
5287
5288 ARRAY_FOREACH ( iSchema, pRes->m_dSchemas )
5289 {
5290 dMapFrom.Resize ( 0 );
5291 CSphSchema & dSchema = ( bMultiSchema ? pRes->m_dSchemas[iSchema] : pRes->m_tSchema );
5292 for ( int i=0; i<pTarget->GetAttrsCount(); i++ )
5293 {
5294 dMapFrom.Add ( dSchema.GetAttrIndex ( pTarget->GetAttr(i).m_sName.cstr() ) );
5295 assert ( dMapFrom[i]>=0
5296 || IsIDAttribute ( pTarget->GetAttr(i) )
5297 || sphIsSortStringInternal ( pTarget->GetAttr(i).m_sName.cstr() )
5298 );
5299 }
5300 int iLimit = bMultiSchema
5301 ? (int)Min ( iCur + pRes->m_dMatchCounts[iSchema], pRes->m_dMatches.GetLength() )
5302 : (int)Min ( pRes->m_iTotalMatches, pRes->m_dMatches.GetLength() );
5303 for ( int i=iCur; i<iLimit; i++ )
5304 {
5305 CSphMatch & tMatch = pRes->m_dMatches[i];
5306
5307 // create new and shiny (and properly sized) match
5308 CSphMatch tRow;
5309 tRow.Reset ( pTarget->GetDynamicSize() );
5310 tRow.m_iDocID = tMatch.m_iDocID;
5311 tRow.m_iWeight = tMatch.m_iWeight;
5312 tRow.m_iTag = tMatch.m_iTag;
5313
5314 // remap attrs
5315 for ( int j=0; j<pTarget->GetAttrsCount(); j++ )
5316 {
5317 const CSphColumnInfo & tDst = pTarget->GetAttr(j);
5318 // we could keep some of the rows static
5319 // and so, avoid the duplication of the data.
5320 if ( !tDst.m_tLocator.m_bDynamic )
5321 {
5322 assert ( dMapFrom[j]<0 || !dSchema.GetAttr ( dMapFrom[j] ).m_tLocator.m_bDynamic );
5323 tRow.m_pStatic = tMatch.m_pStatic;
5324 } else if ( dMapFrom[j]>=0 )
5325 {
5326 const CSphColumnInfo & tSrc = dSchema.GetAttr ( dMapFrom[j] );
5327 if ( tDst.m_eAttrType==SPH_ATTR_FLOAT && tSrc.m_eAttrType==SPH_ATTR_BOOL )
5328 {
5329 tRow.SetAttrFloat ( tDst.m_tLocator, ( tMatch.GetAttr ( tSrc.m_tLocator )>0 ? 1.0f : 0.0f ) );
5330 } else
5331 {
5332 tRow.SetAttr ( tDst.m_tLocator, tMatch.GetAttr ( tSrc.m_tLocator ) );
5333 }
5334 }
5335 }
5336 // swap out old (most likely wrong sized) match
5337 Swap ( tMatch, tRow );
5338 }
5339
5340 if ( !bMultiSchema )
5341 break;
5342
5343 iCur = iLimit;
5344 }
5345
5346 assert ( !bMultiSchema || iCur==pRes->m_dMatches.GetLength() );
5347 if ( &pRes->m_tSchema!=pTarget )
5348 AdoptSchema ( pRes, pTarget );
5349 }
5350
5351 // rebuild the results itemlist expanding stars
ExpandAsterisk(const CSphSchema & tSchema,const CSphVector<CSphQueryItem> & tItems,CSphVector<CSphQueryItem> * pExpanded,bool bNoID=false)5352 const CSphVector<CSphQueryItem> * ExpandAsterisk ( const CSphSchema & tSchema, const CSphVector<CSphQueryItem> & tItems, CSphVector<CSphQueryItem> * pExpanded, bool bNoID=false )
5353 {
5354 // the result schema usually is the index schema + calculated items + @-items
5355 // we need to extract the index schema only - so, look at the items
5356 // and cutoff from calculated or @.
5357 int iSchemaBound = tSchema.GetAttrsCount();
5358 bool bStar = false;
5359 ARRAY_FOREACH ( i, tItems )
5360 {
5361 const CSphQueryItem & tItem = tItems[i];
5362 if ( tItem.m_sAlias.cstr() )
5363 {
5364 int j = tSchema.GetAttrIndex ( tItem.m_sAlias.cstr() );
5365 if ( j>=0 )
5366 iSchemaBound = Min ( iSchemaBound, j );
5367 }
5368 bStar = bStar || tItem.m_sExpr=="*";
5369 }
5370 // no stars? Nothing to do.
5371 if ( !bStar )
5372 return & tItems;
5373
5374 while ( iSchemaBound && tSchema.GetAttr ( iSchemaBound-1 ).m_sName.cstr()[0]=='@' )
5375 iSchemaBound--;
5376 ARRAY_FOREACH ( i, tItems )
5377 {
5378 if ( tItems[i].m_sExpr=="*" )
5379 { // asterisk expands to 'id' + all the items from the schema
5380 if ( tSchema.GetAttrIndex ( "id" )<0 && !bNoID )
5381 {
5382 CSphQueryItem& tItem = pExpanded->Add();
5383 tItem.m_sExpr = "id";
5384 }
5385 for ( int j=0; j<iSchemaBound; j++ )
5386 {
5387 if ( !j && bNoID && tSchema.GetAttr(j).m_sName=="id" )
5388 continue;
5389 CSphQueryItem& tItem = pExpanded->Add();
5390 tItem.m_sExpr = tSchema.GetAttr ( j ).m_sName;
5391 }
5392 } else
5393 pExpanded->Add ( tItems[i] );
5394 }
5395 return pExpanded;
5396 }
5397
5398
RemapStrings(ISphMatchSorter * pSorter,AggrResult_t & tRes)5399 static void RemapStrings ( ISphMatchSorter * pSorter, AggrResult_t & tRes )
5400 {
5401 // do match ptr pre-calc if its "order by string" case
5402 CSphVector<SphStringSorterRemap_t> dRemapAttr;
5403 if ( pSorter && pSorter->UsesAttrs() && sphSortGetStringRemap ( pSorter->GetSchema(), tRes.m_tSchema, dRemapAttr ) )
5404 {
5405 int iCur = 0;
5406 ARRAY_FOREACH ( iSchema, tRes.m_dSchemas )
5407 {
5408 for ( int i=iCur; i<iCur+tRes.m_dMatchCounts[iSchema]; i++ )
5409 {
5410 CSphMatch & tMatch = tRes.m_dMatches[i];
5411 const BYTE * pStringBase = tRes.m_dTag2Pools[tMatch.m_iTag].m_pStrings;
5412
5413 ARRAY_FOREACH ( iAttr, dRemapAttr )
5414 {
5415 SphAttr_t uOff = tMatch.GetAttr ( dRemapAttr[iAttr].m_tSrc );
5416 SphAttr_t uPtr = (SphAttr_t)( pStringBase && uOff ? pStringBase + uOff : 0 );
5417 tMatch.SetAttr ( dRemapAttr[iAttr].m_tDst, uPtr );
5418 }
5419 }
5420 iCur += tRes.m_dMatchCounts[iSchema];
5421 }
5422 }
5423 }
5424
5425
KillAllDupes(ISphMatchSorter * pSorter,AggrResult_t & tRes,const CSphQuery & tQuery)5426 static int KillAllDupes ( ISphMatchSorter * pSorter, AggrResult_t & tRes, const CSphQuery & tQuery )
5427 {
5428 assert ( pSorter );
5429 int iDupes = 0;
5430
5431 if ( pSorter->IsGroupby () )
5432 {
5433 // groupby sorter does that automagically
5434 pSorter->SetMVAPool ( NULL ); // because we must be able to group on @groupby anyway
5435 pSorter->SetStringPool ( NULL );
5436 ARRAY_FOREACH ( i, tRes.m_dMatches )
5437 {
5438 CSphMatch & tMatch = tRes.m_dMatches[i];
5439
5440 if ( !pSorter->PushGrouped ( tMatch ) )
5441 iDupes++;
5442 }
5443 } else
5444 {
5445 // normal sorter needs massasging
5446 // sort by docid and then by tag to guarantee the replacement order
5447 TaggedMatchSorter_fn fnSort;
5448 sphSort ( tRes.m_dMatches.Begin(), tRes.m_dMatches.GetLength(), fnSort, fnSort );
5449
5450 // fold them matches
5451 if ( tQuery.m_dIndexWeights.GetLength() )
5452 {
5453 // if there were per-index weights, compute weighted ranks sum
5454 int iCur = 0;
5455 int iMax = tRes.m_dMatches.GetLength();
5456
5457 while ( iCur<iMax )
5458 {
5459 CSphMatch & tMatch = tRes.m_dMatches[iCur++];
5460
5461 while ( iCur<iMax && tRes.m_dMatches[iCur].m_iDocID==tMatch.m_iDocID )
5462 {
5463 const CSphMatch & tDupe = tRes.m_dMatches[iCur];
5464 int iAddWeight = tDupe.m_iWeight;
5465 tMatch.m_iWeight += iAddWeight;
5466
5467 iDupes++;
5468 iCur++;
5469 }
5470
5471 pSorter->Push ( tMatch );
5472 }
5473
5474 } else
5475 {
5476 // by default, simply remove dupes (select first by tag)
5477 ARRAY_FOREACH ( i, tRes.m_dMatches )
5478 {
5479 if ( i==0 || tRes.m_dMatches[i].m_iDocID!=tRes.m_dMatches[i-1].m_iDocID )
5480 pSorter->Push ( tRes.m_dMatches[i] );
5481 else
5482 iDupes++;
5483 }
5484 }
5485 }
5486
5487 tRes.m_dMatches.Reset ();
5488 sphFlattenQueue ( pSorter, &tRes, -1 );
5489 SafeDelete ( pSorter );
5490
5491 return iDupes;
5492 }
5493
5494
RecoverAggregateFunctions(const CSphQuery & tQuery,const AggrResult_t & tRes)5495 static void RecoverAggregateFunctions ( const CSphQuery & tQuery, const AggrResult_t & tRes )
5496 {
5497 ARRAY_FOREACH ( i, tQuery.m_dItems )
5498 {
5499 const CSphQueryItem & tItem = tQuery.m_dItems[i];
5500 if ( tItem.m_eAggrFunc==SPH_AGGR_NONE )
5501 continue;
5502
5503 for ( int j=0; j<tRes.m_tSchema.GetAttrsCount(); j++ )
5504 {
5505 CSphColumnInfo & tCol = const_cast<CSphColumnInfo&> ( tRes.m_tSchema.GetAttr(j) );
5506 if ( tCol.m_sName==tItem.m_sAlias )
5507 {
5508 assert ( tCol.m_eAggrFunc==SPH_AGGR_NONE );
5509 tCol.m_eAggrFunc = tItem.m_eAggrFunc;
5510 }
5511 }
5512 }
5513 }
5514
5515
MinimizeAggrResult(AggrResult_t & tRes,const CSphQuery & tQuery,bool bHadLocalIndexes,CSphSchema * pExtraSchema,bool bFromSphinxql=false)5516 bool MinimizeAggrResult ( AggrResult_t & tRes, const CSphQuery & tQuery, bool bHadLocalIndexes, CSphSchema* pExtraSchema, bool bFromSphinxql=false )
5517 {
5518 // sanity check
5519 int iExpected = 0;
5520 ARRAY_FOREACH ( i, tRes.m_dMatchCounts )
5521 iExpected += tRes.m_dMatchCounts[i];
5522
5523 if ( iExpected!=tRes.m_dMatches.GetLength() )
5524 {
5525 tRes.m_sError.SetSprintf ( "INTERNAL ERROR: expected %d matches in combined result set, got %d",
5526 iExpected, tRes.m_dMatches.GetLength() );
5527 tRes.m_iSuccesses = 0;
5528 return false;
5529 }
5530
5531 if ( !( bFromSphinxql || tRes.m_dMatches.GetLength() ) )
5532 return true;
5533
5534 // build minimal schema
5535 if ( !tRes.m_dSchemas.GetLength() && bFromSphinxql )
5536 {
5537 AddIDAttribute ( (CVirtualSchema*) &tRes.m_tSchema );
5538 return true;
5539 }
5540 tRes.m_tSchema = tRes.m_dSchemas[0];
5541 bool bAllEqual = true;
5542 bool bAgent = tQuery.m_bAgent;
5543 bool bUsualApi = !( bAgent || bFromSphinxql );
5544
5545 for ( int i=1; i<tRes.m_dSchemas.GetLength(); i++ )
5546 {
5547 if ( !MinimizeSchema ( tRes.m_tSchema, tRes.m_dSchemas[i] ) )
5548 bAllEqual = false;
5549 }
5550
5551 CSphVector<CSphQueryItem> tExtItems;
5552 const CSphVector<CSphQueryItem> * pSelectItems = ExpandAsterisk ( tRes.m_tSchema, tQuery.m_dItems, &tExtItems, bUsualApi );
5553
5554 if ( !bUsualApi )
5555 {
5556 AddIDAttribute ( (CVirtualSchema*) &tRes.m_tSchema );
5557 ARRAY_FOREACH ( i, tRes.m_dSchemas )
5558 AddIDAttribute ( (CVirtualSchema*) &tRes.m_dSchemas[i] );
5559 }
5560
5561 // the final result schema - for collections, etc
5562 // we can't construct the random final schema right now, since
5563 // the final sorter needs the schema fields in specific order:
5564
5565 // shortcuts
5566 const char * sCount = "@count";
5567 const char * sWeight = "@weight";
5568
5569 // truly virtual schema which contains unique necessary fields.
5570 CVirtualSchema tInternalSchema;
5571 // truly virtual schema for final result returning
5572 CVirtualSchema tFrontendSchema;
5573 tFrontendSchema.GetWAttrs().Resize ( pSelectItems->GetLength() );
5574
5575 CSphVector<int> dKnownItems;
5576 int iKnownItems = 0;
5577 if ( pSelectItems->GetLength() )
5578 {
5579 for ( int i=0; i<tRes.m_tSchema.GetAttrsCount(); i++ )
5580 {
5581 const CSphColumnInfo & tCol = tRes.m_tSchema.GetAttr(i);
5582 if ( tCol.m_pExpr.Ptr() || ( bUsualApi && *tCol.m_sName.cstr()=='@' ) )
5583 {
5584 if ( *tCol.m_sName.cstr()=='@' )
5585 {
5586 ARRAY_FOREACH ( j, (*pSelectItems) )
5587 {
5588 const CSphQueryItem & tQueryItem = (*pSelectItems)[j];
5589 const char * sExpr = tQueryItem.m_sExpr.cstr();
5590 if ( tQueryItem.m_sExpr=="count(*)" )
5591 sExpr = sCount;
5592 else if ( tQueryItem.m_sExpr=="weight()" )
5593 sExpr = sWeight;
5594
5595 if ( tFrontendSchema.GetAttr(j).m_iIndex<0 && sExpr && tCol.m_sName==sExpr )
5596 {
5597 CSphColumnInfo & tItem = tFrontendSchema.GetWAttr(j);
5598 tItem.m_iIndex = tInternalSchema.GetAttrsCount();
5599 tItem.m_sName = tQueryItem.m_sAlias;
5600 dKnownItems.Add(j);
5601 ++iKnownItems;
5602 }
5603 }
5604 if ( tFrontendSchema.GetAttr ( tCol.m_sName.cstr() )==NULL )
5605 {
5606 CSphColumnInfo & tItem = tFrontendSchema.GetWAttrs().Add();
5607 tItem.m_iIndex = tInternalSchema.GetAttrsCount();
5608 tItem.m_sName = tCol.m_sName;
5609 }
5610 } else
5611 ARRAY_FOREACH ( j, (*pSelectItems) )
5612 if ( tFrontendSchema.GetAttr(j).m_iIndex<0
5613 && ( (*pSelectItems)[j].m_sAlias.cstr() && (*pSelectItems)[j].m_sAlias==tCol.m_sName ) )
5614 {
5615 CSphColumnInfo & tItem = tFrontendSchema.GetWAttr(j);
5616 tItem.m_iIndex = tInternalSchema.GetAttrsCount();
5617 tItem.m_sName = (*pSelectItems)[j].m_sAlias;
5618 dKnownItems.Add(j);
5619 ++iKnownItems;
5620 }
5621 } else
5622 {
5623 bool bAdd = false;
5624 ARRAY_FOREACH ( j, (*pSelectItems) )
5625 {
5626 const CSphQueryItem & tQueryItem = (*pSelectItems)[j];
5627 const char * sExpr = tQueryItem.m_sExpr.cstr();
5628 if ( tQueryItem.m_sExpr=="count(*)" )
5629 sExpr = sCount;
5630 else if ( tQueryItem.m_sExpr=="weight()" )
5631 sExpr = sWeight;
5632
5633 if ( tFrontendSchema.GetAttr(j).m_iIndex>=0 )
5634 continue;
5635
5636 if ( ( sExpr && tCol.m_sName==sExpr && tQueryItem.m_eAggrFunc==SPH_AGGR_NONE )
5637 || ( tQueryItem.m_sAlias.cstr() && tQueryItem.m_sAlias==tCol.m_sName
5638 // do not add attr2 to frontend schema in cases like this
5639 // attr1 AS attr2
5640 && ( tRes.m_tSchema.GetAttrIndex ( sExpr )==-1
5641 // but add attr2, not attr1 in cases like this
5642 // MIN(attr1) AS attr2
5643 || tQueryItem.m_eAggrFunc!=SPH_AGGR_NONE ) ) )
5644 {
5645 bAdd = true;
5646 dKnownItems.Add(j);
5647 ++iKnownItems;
5648 if ( !bAgent )
5649 {
5650 CSphColumnInfo & tItem = tFrontendSchema.GetWAttr(j);
5651 tItem.m_iIndex = tInternalSchema.GetAttrsCount(); // temporary idx, will change to locator by this index
5652 if ( tQueryItem.m_sAlias.cstr() )
5653 tItem.m_sName = tQueryItem.m_sAlias;
5654 else
5655 tItem.m_sName = tQueryItem.m_sExpr;
5656 }
5657 }
5658 }
5659 if ( !bAdd && pExtraSchema!=NULL )
5660 {
5661 if ( pExtraSchema->GetAttrsCount() )
5662 {
5663 for ( int j=0; j<pExtraSchema->GetAttrsCount(); j++ )
5664 {
5665 if ( pExtraSchema->GetAttr(j).m_sName==tCol.m_sName )
5666 bAdd = true;
5667 }
5668 // the extra schema is not null, but empty - and we have no local agents
5669 // so, the schema of result is already aligned to the extra, just add it
5670 } else if ( !bHadLocalIndexes )
5671 {
5672 bAdd = true;
5673 }
5674 }
5675 if ( !bAdd && bUsualApi && *tCol.m_sName.cstr()=='@' )
5676 bAdd = true;
5677
5678 if ( !bAdd )
5679 continue;
5680 }
5681
5682 // if before all schemas were proved as equal, and the tCol taken from current schema is static -
5683 // this is no reason now to make it dynamic.
5684 bool bDynamic = ( bAllEqual ? tCol.m_tLocator.m_bDynamic : true );
5685 tInternalSchema.AddAttr ( tCol, bDynamic );
5686 if ( !bDynamic )
5687 {
5688 // all schemas are equal, so all offsets and bitcounts also equal.
5689 // If we Add the static attribute which already exists in result, we need
5690 // not to corrupt it's locator. So, in this case let us force the locator
5691 // to the old data.
5692 CSphColumnInfo & tNewCol = tInternalSchema.LastColumn();
5693 assert ( !tNewCol.m_tLocator.m_bDynamic );
5694 tNewCol.m_tLocator = tCol.m_tLocator;
5695 }
5696 }
5697
5698 bAllEqual &= ( tRes.m_tSchema.GetAttrsCount()==tInternalSchema.GetAttrsCount() );
5699 }
5700
5701 // check if we actually have all required columns already
5702 if ( iKnownItems<pSelectItems->GetLength() )
5703 {
5704 tRes.m_iSuccesses = 0;
5705 dKnownItems.Sort();
5706 ARRAY_FOREACH ( j, dKnownItems )
5707 if ( j!=dKnownItems[j] )
5708 {
5709 tRes.m_sError.SetSprintf ( "INTERNAL ERROR: the column '%s/%s' does not present in result set schema",
5710 (*pSelectItems)[j].m_sExpr.cstr(), (*pSelectItems)[j].m_sAlias.cstr() );
5711 return false;
5712 }
5713 if ( dKnownItems.GetLength()==pSelectItems->GetLength()-1 )
5714 {
5715 tRes.m_sError.SetSprintf ( "INTERNAL ERROR: the column '%s/%s' does not present in result set schema",
5716 pSelectItems->Last().m_sExpr.cstr(), pSelectItems->Last().m_sAlias.cstr() );
5717 return false;
5718 }
5719 tRes.m_sError = "INTERNAL ERROR: some columns does not present in result set schema";
5720 return false;
5721 }
5722
5723 // finalize the tFrontendSchema - switch back m_iIndex field
5724 // and set up the locators for the fields
5725 if ( !bAgent )
5726 {
5727 ARRAY_FOREACH ( i, tFrontendSchema.GetWAttrs() )
5728 {
5729 CSphColumnInfo & tCol = tFrontendSchema.GetWAttr(i);
5730 const CSphColumnInfo & tSource = tInternalSchema.GetAttr ( tCol.m_iIndex );
5731 tCol.m_tLocator = tSource.m_tLocator;
5732 tCol.m_eAttrType = tSource.m_eAttrType;
5733 tCol.m_iIndex = -1;
5734 }
5735 }
5736
5737 // tricky bit
5738 // in purely distributed case, all schemas are received from the wire, and miss aggregate functions info
5739 // thus, we need to re-assign that info
5740 if ( !bHadLocalIndexes )
5741 RecoverAggregateFunctions ( tQuery, tRes );
5742
5743 // we do not need to re-sort if there's exactly one result set
5744 if ( tRes.m_iSuccesses==1 )
5745 {
5746 // convert all matches to minimal schema
5747 if ( !bAllEqual )
5748 RemapResult ( &tInternalSchema, &tRes );
5749 if ( !bAgent )
5750 AdoptAliasedSchema ( tRes, &tFrontendSchema );
5751 return true;
5752 }
5753
5754 // if there's more than one result set, we need to re-sort the matches
5755 // so we need to bring matches to the schema that the *sorter* wants
5756 // so we need to create the sorter before conversion
5757 //
5758 // create queue
5759 // at this point, we do not need to compute anything; it all must be here
5760 ISphMatchSorter * pSorter = sphCreateQueue ( &tQuery, tRes.m_tSchema, tRes.m_sError, false );
5761 if ( !pSorter )
5762 return false;
5763
5764 // reset bAllEqual flag if sorter makes new attributes
5765 if ( bAllEqual )
5766 {
5767 // at first we count already existed internal attributes
5768 // then check if sorter makes more
5769 CSphVector<SphStringSorterRemap_t> dRemapAttr;
5770 sphSortGetStringRemap ( tRes.m_tSchema, tRes.m_tSchema, dRemapAttr );
5771 int iRemapCount = dRemapAttr.GetLength();
5772 sphSortGetStringRemap ( pSorter->GetSchema(), tRes.m_tSchema, dRemapAttr );
5773
5774 bAllEqual = ( dRemapAttr.GetLength()<=iRemapCount );
5775 }
5776
5777 // sorter expects this
5778 tRes.m_tSchema = pSorter->GetSchema();
5779
5780 // convert all matches to sorter schema - at least to manage all static to dynamic
5781 if ( !bAllEqual )
5782 RemapResult ( &tRes.m_tSchema, &tRes );
5783
5784 RemapStrings ( pSorter, tRes );
5785 tRes.m_iTotalMatches -= KillAllDupes ( pSorter, tRes, tQuery );
5786
5787 if ( !bAllEqual )
5788 RemapResult ( &tInternalSchema, &tRes, false );
5789 if ( !bAgent )
5790 AdoptAliasedSchema ( tRes, &tFrontendSchema );
5791 return true;
5792 }
5793
5794
MinimizeAggrResultCompat(AggrResult_t & tRes,const CSphQuery & tQuery,bool bHadLocalIndexes)5795 bool MinimizeAggrResultCompat ( AggrResult_t & tRes, const CSphQuery & tQuery, bool bHadLocalIndexes )
5796 {
5797 // sanity check
5798 int iExpected = 0;
5799 ARRAY_FOREACH ( i, tRes.m_dMatchCounts )
5800 iExpected += tRes.m_dMatchCounts[i];
5801
5802 if ( iExpected!=tRes.m_dMatches.GetLength() )
5803 {
5804 tRes.m_sError.SetSprintf ( "INTERNAL ERROR: expected %d matches in combined result set, got %d",
5805 iExpected, tRes.m_dMatches.GetLength() );
5806 return false;
5807 }
5808
5809 if ( !tRes.m_dMatches.GetLength() )
5810 return true;
5811
5812 // build minimal schema
5813 bool bAllEqual = true;
5814 tRes.m_tSchema = tRes.m_dSchemas[0];
5815 for ( int i=1; i<tRes.m_dSchemas.GetLength(); i++ )
5816 {
5817 if ( !MinimizeSchema ( tRes.m_tSchema, tRes.m_dSchemas[i] ) )
5818 bAllEqual = false;
5819 }
5820
5821 // apply select-items on top of that
5822 bool bStar = false;
5823 int iStar = 0;
5824 for ( ; iStar<tQuery.m_dItems.GetLength(); iStar++ )
5825 {
5826 if ( tQuery.m_dItems[iStar].m_sExpr=="*" )
5827 {
5828 bStar = true;
5829 break;
5830 }
5831 }
5832
5833 // remove id attr which may be emerged by the new agents
5834 if ( bStar && !bHadLocalIndexes && tRes.m_tSchema.GetAttr(iStar).m_sName=="id" )
5835 {
5836 CVirtualSchema * pSchema = (CVirtualSchema *)&tRes.m_tSchema;
5837 pSchema->GetWAttrs().Remove(iStar);
5838 }
5839
5840 if ( !bStar && tQuery.m_dItems.GetLength() )
5841 {
5842 CSphSchema tItems;
5843 for ( int i=0; i<tRes.m_tSchema.GetAttrsCount(); i++ )
5844 {
5845 const CSphColumnInfo & tCol = tRes.m_tSchema.GetAttr(i);
5846 if ( !tCol.m_pExpr )
5847 {
5848 bool bAdd = false;
5849
5850 ARRAY_FOREACH ( j, tQuery.m_dItems )
5851 {
5852 const CSphQueryItem & tQueryItem = tQuery.m_dItems[j];
5853 if ( ( tQueryItem.m_sExpr.cstr() && tQueryItem.m_sExpr==tCol.m_sName )
5854 || ( tQueryItem.m_sAlias.cstr() && tQueryItem.m_sAlias==tCol.m_sName ) )
5855 {
5856 bAdd = true;
5857 break;
5858 }
5859 }
5860
5861 if ( !bAdd )
5862 continue;
5863 }
5864 tItems.AddAttr ( tCol, true );
5865 }
5866
5867 if ( tRes.m_tSchema.GetAttrsCount()!=tItems.GetAttrsCount() )
5868 {
5869 tRes.m_tSchema = tItems;
5870 bAllEqual = false;
5871 }
5872 }
5873
5874 // tricky bit
5875 // in purely distributed case, all schemas are received from the wire, and miss aggregate functions info
5876 // thus, we need to re-assign that info
5877 if ( !bHadLocalIndexes )
5878 RecoverAggregateFunctions ( tQuery, tRes );
5879
5880 // if there's more than one result set, we need to re-sort the matches
5881 // so we need to bring matches to the schema that the *sorter* wants
5882 // so we need to create the sorter before conversion
5883 ISphMatchSorter * pSorter = NULL;
5884 if ( tRes.m_iSuccesses!=1 )
5885 {
5886 // create queue
5887 // at this point, we do not need to compute anything; it all must be here
5888 pSorter = sphCreateQueue ( &tQuery, tRes.m_tSchema, tRes.m_sError, false );
5889 if ( !pSorter )
5890 return false;
5891
5892 // reset bAllEqual flag if sorter makes new attributes
5893 if ( bAllEqual )
5894 {
5895 // at first we count already existed internal attributes
5896 // then check if sorter makes more
5897 CSphVector<SphStringSorterRemap_t> dRemapAttr;
5898 sphSortGetStringRemap ( tRes.m_tSchema, tRes.m_tSchema, dRemapAttr );
5899 int iRemapCount = dRemapAttr.GetLength();
5900 sphSortGetStringRemap ( pSorter->GetSchema(), tRes.m_tSchema, dRemapAttr );
5901
5902 bAllEqual = ( dRemapAttr.GetLength()<=iRemapCount );
5903 }
5904
5905 // sorter expects this
5906 tRes.m_tSchema = pSorter->GetSchema();
5907 }
5908
5909 // convert all matches to minimal schema
5910 if ( !bAllEqual )
5911 RemapResult ( &tRes.m_tSchema, &tRes );
5912
5913 // we do not need to re-sort if there's exactly one result set
5914 if ( tRes.m_iSuccesses==1 )
5915 return true;
5916
5917 RemapStrings ( pSorter, tRes );
5918 tRes.m_iTotalMatches -= KillAllDupes ( pSorter, tRes, tQuery );
5919 return true;
5920 }
5921
5922
SetupKillListFilter(CSphFilterSettings & tFilter,const SphAttr_t * pKillList,int nEntries)5923 void SetupKillListFilter ( CSphFilterSettings & tFilter, const SphAttr_t * pKillList, int nEntries )
5924 {
5925 assert ( nEntries && pKillList );
5926
5927 tFilter.m_bExclude = true;
5928 tFilter.m_eType = SPH_FILTER_VALUES;
5929 tFilter.m_iMinValue = pKillList[0];
5930 tFilter.m_iMaxValue = pKillList[nEntries-1];
5931 tFilter.m_sAttrName = "@id";
5932 tFilter.SetExternalValues ( pKillList, nEntries );
5933 }
5934
5935 /////////////////////////////////////////////////////////////////////////////
5936
5937 class CSphSchemaMT : public CSphSchema
5938 {
5939 public:
CSphSchemaMT(const char * sName="(nameless)")5940 explicit CSphSchemaMT ( const char * sName="(nameless)" ) : CSphSchema ( sName ), m_pLock ( NULL )
5941 {}
5942
AwareMT()5943 void AwareMT()
5944 {
5945 if ( m_pLock )
5946 return;
5947 m_pLock = new CSphRwlock();
5948 m_pLock->Init();
5949 }
5950
~CSphSchemaMT()5951 ~CSphSchemaMT()
5952 {
5953 if ( m_pLock )
5954 Verify ( m_pLock->Done() );
5955 SafeDelete ( m_pLock )
5956 }
5957
5958 // get wlocked entry, only if it is not yet touched
GetVirgin()5959 inline CSphSchemaMT * GetVirgin ()
5960 {
5961 if ( !m_pLock )
5962 return this;
5963
5964 if ( m_pLock->WriteLock() )
5965 {
5966 if ( m_dAttrs.GetLength()!=0 ) // not already a virgin
5967 {
5968 m_pLock->Unlock();
5969 return NULL;
5970 }
5971 return this;
5972 } else
5973 {
5974 sphLogDebug ( "WriteLock %p failed", this );
5975 assert ( false );
5976 }
5977
5978 return NULL;
5979 }
5980
RLock()5981 inline CSphSchemaMT * RLock()
5982 {
5983 if ( !m_pLock )
5984 return this;
5985
5986 if ( !m_pLock->ReadLock() )
5987 {
5988 sphLogDebug ( "ReadLock %p failed", this );
5989 assert ( false );
5990 }
5991 return this;
5992 }
5993
UnLock() const5994 inline void UnLock() const
5995 {
5996 if ( m_pLock )
5997 m_pLock->Unlock();
5998 }
5999
6000 private:
6001 mutable CSphRwlock * m_pLock;
6002 };
6003
6004 class UnlockOnDestroy
6005 {
6006 public:
UnlockOnDestroy(const CSphSchemaMT * lock)6007 explicit UnlockOnDestroy ( const CSphSchemaMT * lock ) : m_pLock ( lock )
6008 {}
~UnlockOnDestroy()6009 inline ~UnlockOnDestroy()
6010 {
6011 if ( m_pLock )
6012 m_pLock->UnLock();
6013 }
6014 private:
6015 const CSphSchemaMT * m_pLock;
6016 };
6017
6018 class SearchHandler_c
6019 {
6020 friend void LocalSearchThreadFunc ( void * pArg );
6021
6022 public:
6023 explicit SearchHandler_c ( int iQueries, bool bSphinxql=false );
6024 ~SearchHandler_c();
6025 void RunQueries (); ///< run all queries, get all results
6026 void RunUpdates ( const CSphQuery & tQuery, const CSphString & sIndex, CSphAttrUpdateEx * pUpdates ); ///< run Update command instead of Search
6027
6028 public:
6029 CSphVector<CSphQuery> m_dQueries; ///< queries which i need to search
6030 CSphVector<AggrResult_t> m_dResults; ///< results which i obtained
6031 CSphVector<SearchFailuresLog_c> m_dFailuresSet; ///< failure logs for each query
6032 CSphVector < CSphVector<int64_t> > m_dAgentTimes; ///< per-agent time stats
6033
6034 protected:
6035 void RunSubset ( int iStart, int iEnd ); ///< run queries against index(es) from first query in the subset
6036 void RunLocalSearches ( ISphMatchSorter * pLocalSorter, const char * sDistName );
6037 void RunLocalSearchesMT ();
6038 bool RunLocalSearch ( int iLocal, ISphMatchSorter ** ppSorters, CSphQueryResult ** pResults, bool * pMulti ) const;
6039 bool HasExpresions ( int iStart, int iEnd ) const;
6040
6041 CSphVector<DWORD> m_dMvaStorage;
6042 CSphVector<BYTE> m_dStringsStorage;
6043
6044 int m_iStart; ///< subset start
6045 int m_iEnd; ///< subset end
6046 bool m_bMultiQueue; ///< whether current subset is subject to multi-queue optimization
6047 CSphVector<CSphString> m_dLocal; ///< local indexes for the current subset
6048 mutable CSphVector<CSphSchemaMT> m_dExtraSchemas; ///< the extra fields for agents
6049 bool m_bSphinxql; ///< if the query get from sphinxql - to avoid applying sphinxql magick for others
6050 CSphAttrUpdateEx * m_pUpdates; ///< holder for updates
6051
6052 mutable CSphMutex m_tLock;
6053 mutable SmallStringHash_T<int> m_hUsed;
6054
6055 const ServedIndex_t * UseIndex ( int iLocal ) const;
6056 void ReleaseIndex ( int iLocal ) const;
6057
6058 void OnRunFinished ();
6059 };
6060
6061
SearchHandler_c(int iQueries,bool bSphinxql)6062 SearchHandler_c::SearchHandler_c ( int iQueries, bool bSphinxql )
6063 {
6064 m_iStart = m_iEnd = 0;
6065 m_bMultiQueue = false;
6066
6067 m_dQueries.Resize ( iQueries );
6068 m_dResults.Resize ( iQueries );
6069 m_dFailuresSet.Resize ( iQueries );
6070 m_dExtraSchemas.Resize ( iQueries );
6071 m_dAgentTimes.Resize ( iQueries );
6072 m_tLock.Init();
6073 m_bSphinxql = bSphinxql;
6074 m_pUpdates = NULL;
6075
6076 m_dMvaStorage.Reserve ( 1024 );
6077 m_dMvaStorage.Add ( 0 ); // dummy value
6078 m_dStringsStorage.Reserve ( 1024 );
6079 m_dStringsStorage.Add ( 0 ); // dummy value
6080
6081 ARRAY_FOREACH ( i, m_dResults )
6082 {
6083 m_dResults[i].m_iTag = 1; // first avail tag for local storage ptrs
6084 m_dResults[i].m_dTag2Pools.Add (); // reserved index 0 for remote mva storage ptr; we'll fix this up later
6085 }
6086 }
6087
6088
~SearchHandler_c()6089 SearchHandler_c::~SearchHandler_c ()
6090 {
6091 m_tLock.Done();
6092 m_hUsed.IterateStart();
6093 while ( m_hUsed.IterateNext() )
6094 {
6095 if ( m_hUsed.IterateGet()>0 )
6096 g_pIndexes->GetUnlockedEntry ( m_hUsed.IterateGetKey() ).Unlock();
6097 }
6098 }
6099
6100
UseIndex(int iLocal) const6101 const ServedIndex_t * SearchHandler_c::UseIndex ( int iLocal ) const
6102 {
6103 assert ( iLocal>=0 && iLocal<m_dLocal.GetLength() );
6104 const CSphString & sName = m_dLocal[iLocal];
6105 if ( g_eWorkers!=MPM_THREADS )
6106 return g_pIndexes->GetRlockedEntry ( sName );
6107
6108 m_tLock.Lock();
6109 int * pUseCount = m_hUsed ( sName );
6110 assert ( ( m_pUpdates && pUseCount && *pUseCount>0 ) || !m_pUpdates );
6111
6112 const ServedIndex_t * pServed = NULL;
6113 if ( pUseCount && *pUseCount>0 )
6114 {
6115 pServed = &g_pIndexes->GetUnlockedEntry ( sName );
6116 *pUseCount += ( pServed!=NULL );
6117 } else
6118 {
6119 pServed = g_pIndexes->GetRlockedEntry ( sName );
6120 if ( pServed )
6121 {
6122 if ( pUseCount )
6123 (*pUseCount)++;
6124 else
6125 m_hUsed.Add ( 1, sName );
6126 }
6127 }
6128
6129 m_tLock.Unlock();
6130 return pServed;
6131 }
6132
6133
ReleaseIndex(int iLocal) const6134 void SearchHandler_c::ReleaseIndex ( int iLocal ) const
6135 {
6136 assert ( iLocal>=0 && iLocal<m_dLocal.GetLength() );
6137 if ( g_eWorkers!=MPM_THREADS )
6138 return;
6139
6140 const CSphString & sName = m_dLocal[iLocal];
6141 m_tLock.Lock();
6142
6143 int * pUseCount = m_hUsed ( sName );
6144 assert ( pUseCount && *pUseCount>=0 );
6145 (*pUseCount)--;
6146
6147 if ( !*pUseCount )
6148 g_pIndexes->GetUnlockedEntry ( sName ).Unlock();
6149
6150 assert ( ( m_pUpdates && pUseCount && *pUseCount ) || !m_pUpdates );
6151
6152 m_tLock.Unlock();
6153 }
6154
6155
RunUpdates(const CSphQuery & tQuery,const CSphString & sIndex,CSphAttrUpdateEx * pUpdates)6156 void SearchHandler_c::RunUpdates ( const CSphQuery & tQuery, const CSphString & sIndex, CSphAttrUpdateEx * pUpdates )
6157 {
6158 m_pUpdates = pUpdates;
6159
6160 m_dQueries[0] = tQuery;
6161 m_dQueries[0].m_sIndexes = sIndex;
6162
6163 // lets add index to prevent deadlock
6164 // as index already r-locker or w-locked at this point
6165 m_dLocal.Add ( sIndex );
6166 m_hUsed.Add ( 1, sIndex );
6167
6168 CheckQuery ( tQuery, *pUpdates->m_pError );
6169 if ( !pUpdates->m_pError->IsEmpty() )
6170 return;
6171
6172 int64_t tmLocal = -sphMicroTimer();
6173
6174 RunLocalSearches ( NULL, NULL );
6175 tmLocal += sphMicroTimer();
6176
6177 OnRunFinished();
6178
6179 CSphQueryResult & tRes = m_dResults[0];
6180
6181 tRes.m_iOffset = tQuery.m_iOffset;
6182 tRes.m_iCount = Max ( Min ( tQuery.m_iLimit, tRes.m_dMatches.GetLength()-tQuery.m_iOffset ), 0 );
6183
6184 tRes.m_iQueryTime += (int)(tmLocal/1000);
6185 tRes.m_iCpuTime += tmLocal;
6186
6187 if ( !tRes.m_iSuccesses )
6188 {
6189 CSphStringBuilder sFailures;
6190 m_dFailuresSet[0].BuildReport ( sFailures );
6191 *pUpdates->m_pError = sFailures.cstr();
6192
6193 } else if ( !tRes.m_sError.IsEmpty() )
6194 {
6195 CSphStringBuilder sFailures;
6196 m_dFailuresSet[0].BuildReport ( sFailures );
6197 tRes.m_sWarning = sFailures.cstr(); // FIXME!!! commint warnings too
6198 }
6199
6200 if ( g_pStats )
6201 {
6202 const CSphIOStats & tIO = tRes.m_tIOStats;
6203
6204 g_tStatsMutex.Lock();
6205 g_pStats->m_iQueries += 1;
6206 g_pStats->m_iQueryTime += tmLocal;
6207 g_pStats->m_iQueryCpuTime += tmLocal;
6208 g_pStats->m_iDiskReads += tIO.m_iReadOps;
6209 g_pStats->m_iDiskReadTime += tIO.m_iReadTime;
6210 g_pStats->m_iDiskReadBytes += tIO.m_iReadBytes;
6211 g_tStatsMutex.Unlock();
6212 }
6213
6214 LogQuery ( m_dQueries[0], m_dResults[0], m_dAgentTimes[0] );
6215 };
6216
RunQueries()6217 void SearchHandler_c::RunQueries ()
6218 {
6219 ///////////////////////////////
6220 // choose path and run queries
6221 ///////////////////////////////
6222
6223 // check if all queries are to the same index
6224 bool bSameIndex = false;
6225 if ( m_dQueries.GetLength()>1 )
6226 {
6227 bSameIndex = true;
6228 ARRAY_FOREACH ( i, m_dQueries )
6229 if ( m_dQueries[i].m_sIndexes!=m_dQueries[0].m_sIndexes )
6230 {
6231 bSameIndex = false;
6232 break;
6233 }
6234 }
6235
6236 if ( bSameIndex )
6237 {
6238 ///////////////////////////////
6239 // batch queries to same index
6240 ///////////////////////////////
6241
6242 RunSubset ( 0, m_dQueries.GetLength()-1 );
6243 ARRAY_FOREACH ( i, m_dQueries )
6244 LogQuery ( m_dQueries[i], m_dResults[i], m_dAgentTimes[i] );
6245
6246 } else
6247 {
6248 /////////////////////////////////////////////
6249 // fallback; just work each query separately
6250 /////////////////////////////////////////////
6251
6252 ARRAY_FOREACH ( i, m_dQueries )
6253 {
6254 RunSubset ( i, i );
6255 LogQuery ( m_dQueries[i], m_dResults[i], m_dAgentTimes[i] );
6256 }
6257 }
6258
6259 OnRunFinished();
6260 }
6261
6262
6263 // final fixup
OnRunFinished()6264 void SearchHandler_c::OnRunFinished()
6265 {
6266 ARRAY_FOREACH ( i, m_dResults )
6267 {
6268 m_dResults[i].m_dTag2Pools[0].m_pMva = m_dMvaStorage.Begin();
6269 m_dResults[i].m_dTag2Pools[0].m_pStrings = m_dStringsStorage.Begin();
6270 m_dResults[i].m_iMatches = m_dResults[i].m_dMatches.GetLength();
6271 }
6272 }
6273
6274
6275 /// return cpu time, in microseconds
sphCpuTimer()6276 int64_t sphCpuTimer ()
6277 {
6278 #ifdef HAVE_CLOCK_GETTIME
6279 if ( !g_bCpuStats )
6280 return 0;
6281
6282 #if defined(CLOCK_PROCESS_CPUTIME_ID)
6283 // CPU time (user+sys), Linux style
6284 #define LOC_CLOCK CLOCK_PROCESS_CPUTIME_ID
6285 #elif defined(CLOCK_PROF)
6286 // CPU time (user+sys), FreeBSD style
6287 #define LOC_CLOCK CLOCK_PROF
6288 #else
6289 // POSIX fallback (wall time)
6290 #define LOC_CLOCK CLOCK_REALTIME
6291 #endif
6292
6293 struct timespec tp;
6294 if ( clock_gettime ( LOC_CLOCK, &tp ) )
6295 return 0;
6296
6297 return tp.tv_sec*1000000 + tp.tv_nsec/1000;
6298 #else
6299 return 0;
6300 #endif
6301 }
6302
6303
6304 struct LocalSearch_t
6305 {
6306 int m_iLocal;
6307 ISphMatchSorter ** m_ppSorters;
6308 CSphQueryResult ** m_ppResults;
6309 bool m_bResult;
6310 };
6311
6312
6313 struct LocalSearchThreadContext_t
6314 {
6315 SphThread_t m_tThd;
6316 SearchHandler_c * m_pHandler;
6317 CSphVector<LocalSearch_t*> m_pSearches;
6318 CrashQuery_t m_tCrashQuery;
6319 };
6320
6321
LocalSearchThreadFunc(void * pArg)6322 void LocalSearchThreadFunc ( void * pArg )
6323 {
6324 LocalSearchThreadContext_t * pContext = (LocalSearchThreadContext_t*) pArg;
6325
6326 // setup query guard for thread
6327 SphCrashLogger_c tQueryTLS;
6328 tQueryTLS.SetupTLS ();
6329 SphCrashLogger_c::SetLastQuery ( pContext->m_tCrashQuery );
6330
6331 ARRAY_FOREACH ( i, pContext->m_pSearches )
6332 {
6333 LocalSearch_t * pCall = pContext->m_pSearches[i];
6334 pCall->m_bResult = pContext->m_pHandler->RunLocalSearch ( pCall->m_iLocal, pCall->m_ppSorters, pCall->m_ppResults, &pContext->m_pHandler->m_bMultiQueue );
6335 }
6336 }
6337
6338
MergeWordStats(CSphQueryResultMeta & tDstResult,const SmallStringHash_T<CSphQueryResultMeta::WordStat_t> & hSrc,SearchFailuresLog_c * pLog,const char * sIndex)6339 static void MergeWordStats ( CSphQueryResultMeta & tDstResult, const SmallStringHash_T<CSphQueryResultMeta::WordStat_t> & hSrc, SearchFailuresLog_c * pLog, const char * sIndex )
6340 {
6341 assert ( pLog );
6342
6343 if ( !tDstResult.m_hWordStats.GetLength() )
6344 {
6345 // nothing has been set yet; just copy
6346 tDstResult.m_hWordStats = hSrc;
6347 return;
6348 }
6349
6350 hSrc.IterateStart();
6351 CSphStringBuilder tDifferWords;
6352 while ( hSrc.IterateNext() )
6353 {
6354 const CSphQueryResultMeta::WordStat_t * pDstStat = tDstResult.m_hWordStats ( hSrc.IterateGetKey() );
6355 const CSphQueryResultMeta::WordStat_t & tSrcStat = hSrc.IterateGet();
6356
6357 // all indexes should produce same words from the query
6358 if ( !pDstStat && !tSrcStat.m_bExpanded )
6359 {
6360 if ( !tDifferWords.Length() )
6361 tDifferWords += hSrc.IterateGetKey().cstr();
6362 else
6363 tDifferWords.Appendf ( ", %s", hSrc.IterateGetKey().cstr() );
6364 }
6365
6366 tDstResult.AddStat ( hSrc.IterateGetKey(), tSrcStat.m_iDocs, tSrcStat.m_iHits, tSrcStat.m_bExpanded );
6367 }
6368
6369 if ( tDifferWords.Length() )
6370 pLog->SubmitEx ( sIndex, "query word(s) mismatch: %s", tDifferWords.cstr() );
6371 }
6372
6373
FlattenToRes(ISphMatchSorter * pSorter,AggrResult_t & tRes)6374 static void FlattenToRes ( ISphMatchSorter * pSorter, AggrResult_t & tRes )
6375 {
6376 assert ( pSorter );
6377
6378 if ( pSorter->GetLength() )
6379 {
6380 tRes.m_dMatchCounts.Add ( pSorter->GetLength() );
6381 tRes.m_dSchemas.Add ( tRes.m_tSchema );
6382 PoolPtrs_t & tPoolPtrs = tRes.m_dTag2Pools.Add ();
6383 tPoolPtrs.m_pMva = tRes.m_pMva;
6384 tPoolPtrs.m_pStrings = tRes.m_pStrings;
6385 sphFlattenQueue ( pSorter, &tRes, tRes.m_iTag++ );
6386
6387 // clean up for next index search
6388 tRes.m_pMva = NULL;
6389 tRes.m_pStrings = NULL;
6390 }
6391 }
6392
6393
RunLocalSearchesMT()6394 void SearchHandler_c::RunLocalSearchesMT ()
6395 {
6396 int64_t tmLocal = sphMicroTimer();
6397
6398 // setup local searches
6399 const int iQueries = m_iEnd-m_iStart+1;
6400 CSphVector<LocalSearch_t> dLocals ( m_dLocal.GetLength() );
6401 CSphVector<CSphQueryResult> dResults ( m_dLocal.GetLength()*iQueries );
6402 CSphVector<ISphMatchSorter*> pSorters ( m_dLocal.GetLength()*iQueries );
6403 CSphVector<CSphQueryResult*> pResults ( m_dLocal.GetLength()*iQueries );
6404
6405 ARRAY_FOREACH ( i, pResults )
6406 pResults[i] = &dResults[i];
6407
6408 ARRAY_FOREACH ( i, m_dLocal )
6409 {
6410 dLocals[i].m_iLocal = i;
6411 dLocals[i].m_ppSorters = &pSorters [ i*iQueries ];
6412 dLocals[i].m_ppResults = &pResults [ i*iQueries ];
6413 }
6414
6415 // setup threads
6416 // FIXME! implement better than naive index:thread mapping
6417 // FIXME! maybe implement a thread-shared jobs queue
6418 CSphVector<LocalSearchThreadContext_t> dThreads ( Min ( g_iDistThreads, dLocals.GetLength() ) );
6419 int iCurThread = 0;
6420
6421 ARRAY_FOREACH ( i, dLocals )
6422 {
6423 dThreads[iCurThread].m_pSearches.Add ( &dLocals[i] );
6424 iCurThread = ( iCurThread+1 ) % g_iDistThreads;
6425 }
6426
6427 // prepare for multithread extra schema processing
6428 for ( int iQuery=m_iStart; iQuery<=m_iEnd; iQuery++ )
6429 m_dExtraSchemas[iQuery].AwareMT();
6430
6431 CrashQuery_t tCrashQuery = SphCrashLogger_c::GetQuery(); // transfer query info for crash logger to new thread
6432 // fire searcher threads
6433 ARRAY_FOREACH ( i, dThreads )
6434 {
6435 dThreads[i].m_pHandler = this;
6436 dThreads[i].m_tCrashQuery = tCrashQuery;
6437 sphThreadCreate ( &dThreads[i].m_tThd, LocalSearchThreadFunc, (void*)&dThreads[i] ); // FIXME! check result
6438 }
6439
6440 // wait for them to complete
6441 ARRAY_FOREACH ( i, dThreads )
6442 sphThreadJoin ( &dThreads[i].m_tThd );
6443
6444 // now merge the results
6445 ARRAY_FOREACH ( iLocal, dLocals )
6446 {
6447 bool bResult = dLocals[iLocal].m_bResult;
6448 const char * sLocal = m_dLocal[iLocal].cstr();
6449
6450 if ( !bResult )
6451 {
6452 // failed
6453 for ( int iQuery=m_iStart; iQuery<=m_iEnd; iQuery++ )
6454 {
6455 int iResultIndex = iLocal*iQueries;
6456 if ( !m_bMultiQueue )
6457 iResultIndex += iQuery - m_iStart;
6458 m_dFailuresSet[iQuery].Submit ( sLocal, dResults[iResultIndex].m_sError.cstr() );
6459 }
6460 continue;
6461 }
6462
6463 // multi-query succeeded
6464 for ( int iQuery=m_iStart; iQuery<=m_iEnd; iQuery++ )
6465 {
6466 // base result set index
6467 // in multi-queue case, the only (!) result set actually filled with meta info
6468 // in non-multi-queue case, just a first index, we fix it below
6469 int iResultIndex = iLocal*iQueries;
6470
6471 // current sorter ALWAYS resides at this index, in all cases
6472 // (current as in sorter for iQuery-th query against iLocal-th index)
6473 int iSorterIndex = iLocal*iQueries + iQuery - m_iStart;
6474
6475 if ( !m_bMultiQueue )
6476 {
6477 // non-multi-queue case
6478 // means that we have mere 1:1 mapping between results and sorters
6479 // so let's adjust result set index
6480 iResultIndex = iSorterIndex;
6481
6482 } else if ( dResults[iResultIndex].m_iMultiplier==-1 )
6483 {
6484 // multi-queue case
6485 // need to additionally check per-query failures of MultiQueryEx
6486 // those are reported through multiplier
6487 // note that iSorterIndex just below is NOT a typo
6488 // separate errors still go into separate result sets
6489 // even though regular meta does not
6490 m_dFailuresSet[iQuery].Submit ( sLocal, dResults[iSorterIndex].m_sError.cstr() );
6491 continue;
6492 }
6493
6494 // no sorter, no fun
6495 ISphMatchSorter * pSorter = pSorters[iSorterIndex];
6496 if ( !pSorter )
6497 continue;
6498
6499 // this one seems OK
6500 AggrResult_t & tRes = m_dResults[iQuery];
6501 CSphQueryResult & tRaw = dResults[iResultIndex];
6502
6503 tRes.m_iSuccesses++;
6504 tRes.m_tSchema = pSorter->GetSchema();
6505 tRes.m_iTotalMatches += pSorter->GetTotalCount();
6506
6507 tRes.m_pMva = tRaw.m_pMva;
6508 tRes.m_pStrings = tRaw.m_pStrings;
6509 MergeWordStats ( tRes, tRaw.m_hWordStats, &m_dFailuresSet[iQuery], sLocal );
6510
6511 // move external attributes storage from tRaw to actual result
6512 tRaw.LeakStorages ( tRes );
6513
6514 tRes.m_iMultiplier = m_bMultiQueue ? iQueries : 1;
6515 tRes.m_iCpuTime += tRaw.m_iCpuTime / tRes.m_iMultiplier;
6516 tRes.m_tIOStats.Add ( tRaw.m_tIOStats );
6517
6518 // extract matches from sorter
6519 FlattenToRes ( pSorter, tRes );
6520
6521 if ( !tRaw.m_sWarning.IsEmpty() )
6522 m_dFailuresSet[iQuery].Submit ( sLocal, tRaw.m_sWarning.cstr() );
6523 }
6524 }
6525
6526 ARRAY_FOREACH ( i, pSorters )
6527 SafeDelete ( pSorters[i] );
6528
6529 // update our wall time for every result set
6530 tmLocal = sphMicroTimer() - tmLocal;
6531 for ( int iQuery=m_iStart; iQuery<=m_iEnd; iQuery++ )
6532 m_dResults[iQuery].m_iQueryTime += (int)( tmLocal/1000 );
6533 }
6534
6535 // invoked from MT searches. So, must be MT-aware!
RunLocalSearch(int iLocal,ISphMatchSorter ** ppSorters,CSphQueryResult ** ppResults,bool * pMulti) const6536 bool SearchHandler_c::RunLocalSearch ( int iLocal, ISphMatchSorter ** ppSorters, CSphQueryResult ** ppResults, bool * pMulti ) const
6537 {
6538 const int iQueries = m_iEnd-m_iStart+1;
6539 const ServedIndex_t * pServed = UseIndex ( iLocal );
6540 if ( !pServed )
6541 {
6542 // FIXME! submit a failure?
6543 return false;
6544 }
6545 assert ( pServed->m_pIndex );
6546 assert ( pServed->m_bEnabled );
6547 assert ( pMulti );
6548
6549 // create sorters
6550 int iValidSorters = 0;
6551 for ( int i=0; i<iQueries; i++ )
6552 {
6553 CSphString& sError = ppResults[i]->m_sError;
6554 const CSphQuery & tQuery = m_dQueries[i+m_iStart];
6555 CSphSchemaMT * pExtraSchemaMT = tQuery.m_bAgent?m_dExtraSchemas[i+m_iStart].GetVirgin():NULL;
6556 UnlockOnDestroy dSchemaLock ( pExtraSchemaMT );
6557
6558 assert ( !tQuery.m_iOldVersion || tQuery.m_iOldVersion>=0x102 );
6559 ppSorters[i] = sphCreateQueue ( &tQuery, pServed->m_pIndex->GetMatchSchema(), sError, true, pExtraSchemaMT, m_pUpdates );
6560
6561 if ( ppSorters[i] )
6562 iValidSorters++;
6563
6564 // can't use multi-query for sorter with string attribute at group by or sort
6565 if ( ppSorters[i] && *pMulti )
6566 *pMulti = ppSorters[i]->CanMulti();
6567 }
6568 if ( !iValidSorters )
6569 {
6570 ReleaseIndex ( iLocal );
6571 return false;
6572 }
6573
6574 CSphVector<int> dLocked;
6575
6576 // setup kill-lists
6577 CSphVector<CSphFilterSettings> dKlists;
6578 for ( int i=iLocal+1; i<m_dLocal.GetLength (); i++ )
6579 {
6580 const ServedIndex_t * pKlistIndex = UseIndex ( i );
6581 if ( !pKlistIndex )
6582 continue;
6583
6584 if ( pKlistIndex->m_pIndex->GetKillListSize() )
6585 {
6586 SetupKillListFilter ( dKlists.Add(), pKlistIndex->m_pIndex->GetKillList(), pKlistIndex->m_pIndex->GetKillListSize() );
6587 dLocked.Add ( i );
6588 } else
6589 {
6590 ReleaseIndex ( i );
6591 }
6592 }
6593
6594 // do the query
6595 bool bResult = false;
6596 pServed->m_pIndex->SetCacheSize ( g_iMaxCachedDocs, g_iMaxCachedHits );
6597 ppResults[0]->m_tIOStats.Start();
6598 if ( *pMulti )
6599 {
6600 bResult = pServed->m_pIndex->MultiQuery ( &m_dQueries[m_iStart], ppResults[0], iQueries, ppSorters, &dKlists );
6601 } else
6602 {
6603 bResult = pServed->m_pIndex->MultiQueryEx ( iQueries, &m_dQueries[m_iStart], ppResults, ppSorters, &dKlists );
6604 }
6605 ppResults[0]->m_tIOStats.Stop();
6606
6607 ARRAY_FOREACH ( i, dLocked )
6608 ReleaseIndex ( dLocked[i] );
6609
6610 return bResult;
6611 }
6612
6613
RunLocalSearches(ISphMatchSorter * pLocalSorter,const char * sDistName)6614 void SearchHandler_c::RunLocalSearches ( ISphMatchSorter * pLocalSorter, const char * sDistName )
6615 {
6616 if ( g_iDistThreads>1 && m_dLocal.GetLength()>1 )
6617 {
6618 RunLocalSearchesMT();
6619 return;
6620 }
6621
6622 CSphVector <int> dLocked;
6623 ARRAY_FOREACH ( iLocal, m_dLocal )
6624 {
6625 const char * sLocal = m_dLocal[iLocal].cstr();
6626
6627 const ServedIndex_t * pServed = UseIndex ( iLocal );
6628 if ( !pServed )
6629 {
6630 if ( sDistName )
6631 for ( int i=m_iStart; i<=m_iEnd; i++ )
6632 m_dFailuresSet[i].SubmitEx ( sDistName, "local index %s missing", sLocal );
6633 continue;
6634 }
6635
6636 assert ( pServed->m_pIndex );
6637 assert ( pServed->m_bEnabled );
6638
6639 // create sorters
6640 CSphVector<ISphMatchSorter*> dSorters ( m_iEnd-m_iStart+1 );
6641 ARRAY_FOREACH ( i, dSorters )
6642 dSorters[i] = NULL;
6643
6644 int iValidSorters = 0;
6645 for ( int iQuery=m_iStart; iQuery<=m_iEnd; iQuery++ )
6646 {
6647 CSphString sError;
6648 CSphQuery & tQuery = m_dQueries[iQuery];
6649 CSphSchemaMT * pExtraSchema = tQuery.m_bAgent?m_dExtraSchemas[iQuery].GetVirgin():NULL;
6650 UnlockOnDestroy dSchemaLock ( pExtraSchema );
6651
6652 // create sorter, if needed
6653 ISphMatchSorter * pSorter = pLocalSorter;
6654 if ( !pLocalSorter )
6655 {
6656 // fixup old queries
6657 if ( !FixupQuery ( &tQuery, &pServed->m_pIndex->GetMatchSchema(), sLocal, sError ) )
6658 {
6659 m_dFailuresSet[iQuery].Submit ( sLocal, sError.cstr() );
6660 continue;
6661 }
6662
6663 // create queue
6664 pSorter = sphCreateQueue ( &tQuery, pServed->m_pIndex->GetMatchSchema(), sError, true, pExtraSchema, m_pUpdates );
6665 if ( !pSorter )
6666 {
6667 m_dFailuresSet[iQuery].Submit ( sLocal, sError.cstr() );
6668 continue;
6669 } else if ( m_bMultiQueue )
6670 {
6671 // can't use multi-query for sorter with string attribute at group by or sort
6672 m_bMultiQueue = pSorter->CanMulti();
6673 }
6674 if ( !sError.IsEmpty() )
6675 m_dFailuresSet[iQuery].Submit ( sLocal, sError.cstr() );
6676 }
6677
6678 dSorters[iQuery-m_iStart] = pSorter;
6679 iValidSorters++;
6680 }
6681 if ( !iValidSorters )
6682 {
6683 ReleaseIndex ( iLocal );
6684 continue;
6685 }
6686
6687 // me shortcuts
6688 AggrResult_t tStats;
6689 CSphQuery * pQuery = &m_dQueries[m_iStart];
6690
6691 // set kill-list
6692 int iNumFilters = pQuery->m_dFilters.GetLength ();
6693 for ( int i=iLocal+1; i<m_dLocal.GetLength (); i++ )
6694 {
6695 const ServedIndex_t * pLocServed = UseIndex ( i );
6696 if ( !pLocServed )
6697 continue;
6698
6699 if ( pLocServed->m_pIndex->GetKillListSize () )
6700 {
6701 CSphFilterSettings tKillListFilter;
6702 SetupKillListFilter ( tKillListFilter, pLocServed->m_pIndex->GetKillList (), pLocServed->m_pIndex->GetKillListSize () );
6703 pQuery->m_dFilters.Add ( tKillListFilter );
6704 dLocked.Add ( i );
6705 } else
6706 {
6707 ReleaseIndex ( i );
6708 }
6709 }
6710
6711 // do the query
6712 bool bResult = false;
6713 pServed->m_pIndex->SetCacheSize ( g_iMaxCachedDocs, g_iMaxCachedHits );
6714 if ( m_bMultiQueue )
6715 {
6716 tStats.m_tIOStats.Start();
6717 bResult = pServed->m_pIndex->MultiQuery ( &m_dQueries[m_iStart], &tStats,
6718 dSorters.GetLength(), dSorters.Begin(), NULL );
6719 tStats.m_tIOStats.Stop();
6720 } else
6721 {
6722 CSphVector<CSphQueryResult*> dResults ( m_dResults.GetLength() );
6723 ARRAY_FOREACH ( i, m_dResults )
6724 dResults[i] = &m_dResults[i];
6725
6726 dResults[m_iStart]->m_tIOStats.Start();
6727
6728 bResult = pServed->m_pIndex->MultiQueryEx ( dSorters.GetLength(),
6729 &m_dQueries[m_iStart], &dResults[m_iStart], &dSorters[0], NULL );
6730
6731 dResults[m_iStart]->m_tIOStats.Stop();
6732 }
6733
6734 // handle results
6735 if ( !bResult )
6736 {
6737 // failed
6738 for ( int iQuery=m_iStart; iQuery<=m_iEnd; iQuery++ )
6739 m_dFailuresSet[iQuery].Submit ( sLocal,
6740 m_dResults [ m_bMultiQueue ? m_iStart : iQuery ].m_sError.cstr() );
6741 } else
6742 {
6743 // multi-query succeeded
6744 for ( int iQuery=m_iStart; iQuery<=m_iEnd; iQuery++ )
6745 {
6746 // but some of the sorters could had failed at "create sorter" stage
6747 ISphMatchSorter * pSorter = dSorters [ iQuery-m_iStart ];
6748 if ( !pSorter )
6749 continue;
6750
6751 // this one seems OK
6752 AggrResult_t & tRes = m_dResults[iQuery];
6753 // multi-queue only returned one result set meta, so we need to replicate it
6754 if ( m_bMultiQueue )
6755 {
6756 // these times will be overridden below, but let's be clean
6757 tRes.m_iQueryTime += tStats.m_iQueryTime / ( m_iEnd-m_iStart+1 );
6758 tRes.m_iCpuTime += tStats.m_iCpuTime / ( m_iEnd-m_iStart+1 );
6759 tRes.m_tIOStats.Add ( tStats.m_tIOStats );
6760 tRes.m_pMva = tStats.m_pMva;
6761 tRes.m_pStrings = tStats.m_pStrings;
6762 MergeWordStats ( tRes, tStats.m_hWordStats, &m_dFailuresSet[iQuery], sLocal );
6763 tRes.m_iMultiplier = m_iEnd-m_iStart+1;
6764 } else if ( tRes.m_iMultiplier==-1 )
6765 {
6766 m_dFailuresSet[iQuery].Submit ( sLocal, tRes.m_sError.cstr() );
6767 continue;
6768 }
6769
6770 tRes.m_iSuccesses++;
6771 tRes.m_tSchema = pSorter->GetSchema();
6772 tRes.m_iTotalMatches += pSorter->GetTotalCount();
6773
6774 // extract matches from sorter
6775 FlattenToRes ( pSorter, tRes );
6776
6777 // move external attributes storage from tStats to actual result
6778 tStats.LeakStorages ( tRes );
6779 }
6780 }
6781
6782 // cleanup kill-list
6783 pQuery->m_dFilters.Resize ( iNumFilters );
6784
6785 ARRAY_FOREACH ( i, dLocked )
6786 ReleaseIndex ( dLocked[i] );
6787
6788 dLocked.Resize ( 0 );
6789
6790 // cleanup sorters
6791 if ( !pLocalSorter )
6792 ARRAY_FOREACH ( i, dSorters )
6793 SafeDelete ( dSorters[i] );
6794 }
6795 }
6796
6797
6798 // check expressions into a query to make sure that it's ready for multi query optimization
HasExpresions(int iStart,int iEnd) const6799 bool SearchHandler_c::HasExpresions ( int iStart, int iEnd ) const
6800 {
6801 ARRAY_FOREACH ( i, m_dLocal )
6802 {
6803 const ServedIndex_t * pServedIndex = UseIndex ( i );
6804
6805 // check that it exists
6806 if ( !pServedIndex || !pServedIndex->m_bEnabled )
6807 {
6808 if ( pServedIndex )
6809 ReleaseIndex ( i );
6810 continue;
6811 }
6812
6813 bool bHasExpression = false;
6814 const CSphSchema & tSchema = pServedIndex->m_pIndex->GetMatchSchema();
6815 for ( int iCheck=iStart; iCheck<=iEnd && !bHasExpression; iCheck++ )
6816 bHasExpression = sphHasExpressions ( m_dQueries[iCheck], tSchema );
6817
6818 ReleaseIndex ( i );
6819
6820 if ( bHasExpression )
6821 return true;
6822 }
6823 return false;
6824 }
6825
6826
RunSubset(int iStart,int iEnd)6827 void SearchHandler_c::RunSubset ( int iStart, int iEnd )
6828 {
6829 m_iStart = iStart;
6830 m_iEnd = iEnd;
6831 m_dLocal.Reset();
6832
6833 // all my stats
6834 int64_t tmSubset = sphMicroTimer();
6835 int64_t tmLocal = 0;
6836 int64_t tmWait = 0;
6837 int64_t tmCpu = sphCpuTimer ();
6838
6839 // prepare for descent
6840 CSphQuery & tFirst = m_dQueries[iStart];
6841
6842 for ( int iRes=iStart; iRes<=iEnd; iRes++ )
6843 m_dResults[iRes].m_iSuccesses = 0;
6844
6845 ////////////////////////////////////////////////////////////////
6846 // check for single-query, multi-queue optimization possibility
6847 ////////////////////////////////////////////////////////////////
6848
6849 m_bMultiQueue = ( iStart<iEnd );
6850 for ( int iCheck=iStart+1; iCheck<=iEnd && m_bMultiQueue; iCheck++ )
6851 {
6852 const CSphQuery & qFirst = m_dQueries[iStart];
6853 const CSphQuery & qCheck = m_dQueries[iCheck];
6854
6855 // these parameters must be the same
6856 if (
6857 ( qCheck.m_sRawQuery!=qFirst.m_sRawQuery ) || // query string
6858 ( qCheck.m_iWeights!=qFirst.m_iWeights ) || // weights count
6859 ( qCheck.m_pWeights && memcmp ( qCheck.m_pWeights, qFirst.m_pWeights, sizeof(int)*qCheck.m_iWeights ) ) || // weights; NOLINT
6860 ( qCheck.m_eMode!=qFirst.m_eMode ) || // search mode
6861 ( qCheck.m_eRanker!=qFirst.m_eRanker ) || // ranking mode
6862 ( qCheck.m_dFilters.GetLength()!=qFirst.m_dFilters.GetLength() ) || // attr filters count
6863 ( qCheck.m_iCutoff!=qFirst.m_iCutoff ) || // cutoff
6864 ( qCheck.m_eSort==SPH_SORT_EXPR && qFirst.m_eSort==SPH_SORT_EXPR && qCheck.m_sSortBy!=qFirst.m_sSortBy ) || // sort expressions
6865 ( qCheck.m_bGeoAnchor!=qFirst.m_bGeoAnchor ) || // geodist expression
6866 ( qCheck.m_bGeoAnchor && qFirst.m_bGeoAnchor && ( qCheck.m_fGeoLatitude!=qFirst.m_fGeoLatitude || qCheck.m_fGeoLongitude!=qFirst.m_fGeoLongitude ) ) ) // some geodist cases
6867 {
6868 m_bMultiQueue = false;
6869 break;
6870 }
6871
6872 // filters must be the same too
6873 assert ( qCheck.m_dFilters.GetLength()==qFirst.m_dFilters.GetLength() );
6874 ARRAY_FOREACH ( i, qCheck.m_dFilters )
6875 if ( qCheck.m_dFilters[i]!=qFirst.m_dFilters[i] )
6876 {
6877 m_bMultiQueue = false;
6878 break;
6879 }
6880 }
6881
6882 ////////////////////////////
6883 // build local indexes list
6884 ////////////////////////////
6885
6886 CSphVector<AgentConn_t> dAgents;
6887 CSphVector<CSphString> dDistLocal;
6888 bool bDist = false;
6889 int iAgentConnectTimeout = 0, iAgentQueryTimeout = 0;
6890
6891 {
6892 g_tDistLock.Lock();
6893 DistributedIndex_t * pDist = g_hDistIndexes ( tFirst.m_sIndexes );
6894 if ( pDist )
6895 {
6896 bDist = true;
6897 iAgentConnectTimeout = pDist->m_iAgentConnectTimeout;
6898 iAgentQueryTimeout = pDist->m_iAgentQueryTimeout;
6899
6900 dDistLocal = pDist->m_dLocal;
6901
6902 dAgents.Resize ( pDist->m_dAgents.GetLength() );
6903 ARRAY_FOREACH ( i, pDist->m_dAgents )
6904 dAgents[i] = pDist->m_dAgents[i];
6905 }
6906 g_tDistLock.Unlock();
6907 }
6908
6909 if ( !bDist )
6910 {
6911 // they're all local, build the list
6912 if ( tFirst.m_sIndexes=="*" )
6913 {
6914 // search through all local indexes
6915 for ( IndexHashIterator_c it ( g_pIndexes ); it.Next(); )
6916 if ( it.Get ().m_bEnabled )
6917 m_dLocal.Add ( it.GetKey() );
6918 } else
6919 {
6920 // search through specified local indexes
6921 ParseIndexList ( tFirst.m_sIndexes, m_dLocal );
6922
6923 // there should be no distributed indexes in multi-index query
6924 int iDistFound = -1;
6925 g_tDistLock.Lock();
6926
6927 ARRAY_FOREACH ( i, m_dLocal )
6928 if ( g_hDistIndexes.Exists ( m_dLocal[i] ) )
6929 {
6930 iDistFound = i;
6931 break;
6932 }
6933
6934 g_tDistLock.Unlock();
6935
6936 if ( iDistFound!=-1 )
6937 {
6938 for ( int iRes=iStart; iRes<=iEnd; iRes++ )
6939 m_dResults[iRes].m_sError.SetSprintf ( "distributed index '%s' in multi-index query found", m_dLocal[iDistFound].cstr() );
6940 return;
6941 }
6942
6943 ARRAY_FOREACH ( i, m_dLocal )
6944 {
6945 const ServedIndex_t * pServedIndex = UseIndex ( i );
6946
6947 // check that it exists
6948 if ( !pServedIndex )
6949 {
6950 for ( int iRes=iStart; iRes<=iEnd; iRes++ )
6951 m_dResults[iRes].m_sError.SetSprintf ( "unknown local index '%s' in search request", m_dLocal[i].cstr() );
6952 return;
6953 }
6954
6955 bool bEnabled = pServedIndex->m_bEnabled;
6956 ReleaseIndex ( i );
6957 // if it exists but is not enabled, remove it from the list and force recheck
6958 if ( !bEnabled )
6959 m_dLocal.Remove ( i-- );
6960 }
6961 }
6962
6963 // sanity check
6964 if ( !m_dLocal.GetLength() )
6965 {
6966 for ( int iRes=iStart; iRes<=iEnd; iRes++ )
6967 m_dResults[iRes].m_sError.SetSprintf ( "no enabled local indexes to search" );
6968 return;
6969 }
6970
6971 } else
6972 {
6973 // copy local indexes list from distributed definition, but filter out disabled ones
6974 ARRAY_FOREACH ( i, dDistLocal )
6975 {
6976 int iDistLocal = m_dLocal.GetLength();
6977 m_dLocal.Add ( dDistLocal[i] );
6978
6979 const ServedIndex_t * pServedIndex = UseIndex ( iDistLocal );
6980 bool bValidLocalIndex = pServedIndex && pServedIndex->m_bEnabled;
6981 if ( pServedIndex )
6982 ReleaseIndex ( iDistLocal );
6983
6984 if ( !bValidLocalIndex )
6985 m_dLocal.Pop();
6986 }
6987 }
6988
6989 /////////////////////////////////////////////////////
6990 // optimize single-query, same-schema local searches
6991 /////////////////////////////////////////////////////
6992
6993 ISphMatchSorter * pLocalSorter = NULL;
6994 while ( iStart==iEnd && m_dLocal.GetLength()>1 )
6995 {
6996 CSphString sError;
6997
6998 // check if all schemes are equal
6999 bool bAllEqual = true;
7000
7001 const ServedIndex_t * pFirstIndex = UseIndex ( 0 );
7002 if ( !pFirstIndex )
7003 break;
7004
7005 const CSphSchema & tFirstSchema = pFirstIndex->m_pIndex->GetMatchSchema();
7006 for ( int i=1; i<m_dLocal.GetLength() && bAllEqual; i++ )
7007 {
7008 const ServedIndex_t * pNextIndex = UseIndex ( i );
7009 if ( !pNextIndex )
7010 {
7011 bAllEqual = false;
7012 break;
7013 }
7014
7015 if ( !tFirstSchema.CompareTo ( pNextIndex->m_pIndex->GetMatchSchema(), sError ) )
7016 bAllEqual = false;
7017
7018 ReleaseIndex ( i );
7019 }
7020
7021 // we can reuse the very same sorter
7022 if ( bAllEqual && FixupQuery ( &m_dQueries[iStart], &tFirstSchema, "local-sorter", sError ) )
7023 {
7024 CSphSchemaMT * pExtraSchemaMT = m_dQueries[iStart].m_bAgent?m_dExtraSchemas[iStart].GetVirgin():NULL;
7025 UnlockOnDestroy ExtraLocker ( pExtraSchemaMT );
7026 pLocalSorter = sphCreateQueue ( &m_dQueries[iStart], tFirstSchema, sError, true, pExtraSchemaMT );
7027 }
7028
7029 ReleaseIndex ( 0 );
7030 break;
7031 }
7032
7033 // select lists must have no expressions
7034 if ( m_bMultiQueue )
7035 {
7036 m_bMultiQueue = !HasExpresions ( iStart, iEnd );
7037 }
7038
7039 // these are mutual exclusive
7040 assert ( !( m_bMultiQueue && pLocalSorter ) );
7041
7042 ///////////////////////////////////////////////////////////
7043 // main query loop (with multiple retries for distributed)
7044 ///////////////////////////////////////////////////////////
7045
7046 tFirst.m_iRetryCount = Min ( Max ( tFirst.m_iRetryCount, 0 ), MAX_RETRY_COUNT ); // paranoid clamp
7047 if ( !bDist )
7048 tFirst.m_iRetryCount = 0;
7049
7050 for ( int iRetry=0; iRetry<=tFirst.m_iRetryCount; iRetry++ )
7051 {
7052 ////////////////////////
7053 // issue remote queries
7054 ////////////////////////
7055
7056 // delay between retries
7057 if ( iRetry>0 )
7058 sphSleepMsec ( tFirst.m_iRetryDelay );
7059
7060 // connect to remote agents and query them, if required
7061 int iRemote = 0;
7062 if ( bDist )
7063 {
7064 ConnectToRemoteAgents ( dAgents, iRetry!=0 );
7065
7066 SearchRequestBuilder_t tReqBuilder ( m_dQueries, iStart, iEnd );
7067 iRemote = QueryRemoteAgents ( dAgents, iAgentConnectTimeout, tReqBuilder, &tmWait );
7068 }
7069
7070 /////////////////////
7071 // run local queries
7072 //////////////////////
7073
7074 // while the remote queries are running, do local searches
7075 // FIXME! what if the remote agents finish early, could they timeout?
7076 if ( iRetry==0 )
7077 {
7078 if ( bDist && !iRemote && !m_dLocal.GetLength() )
7079 {
7080 for ( int iRes=iStart; iRes<=iEnd; iRes++ )
7081 m_dResults[iRes].m_sError = "all remote agents unreachable and no available local indexes found";
7082
7083 SafeDelete ( pLocalSorter );
7084 return;
7085 }
7086
7087 tmLocal = -sphMicroTimer();
7088 RunLocalSearches ( pLocalSorter, bDist ? tFirst.m_sIndexes.cstr() : NULL );
7089 tmLocal += sphMicroTimer();
7090 }
7091
7092 ///////////////////////
7093 // poll remote queries
7094 ///////////////////////
7095
7096 // wait for remote queries to complete
7097 if ( iRemote )
7098 {
7099 SearchReplyParser_t tParser ( iStart, iEnd, m_dMvaStorage, m_dStringsStorage );
7100 int iMsecLeft = iAgentQueryTimeout - (int)( tmLocal/1000 );
7101 int iReplys = WaitForRemoteAgents ( dAgents, Max ( iMsecLeft, 0 ), tParser, &tmWait );
7102
7103 // check if there were valid (though might be 0-matches) replys, and merge them
7104 if ( iReplys )
7105 ARRAY_FOREACH ( iAgent, dAgents )
7106 {
7107 AgentConn_t & tAgent = dAgents[iAgent];
7108 if ( !tAgent.m_bSuccess )
7109 continue;
7110
7111 // merge this agent's results
7112 for ( int iRes=iStart; iRes<=iEnd; iRes++ )
7113 {
7114 const CSphQueryResult & tRemoteResult = tAgent.m_dResults[iRes-iStart];
7115
7116 // copy errors or warnings
7117 if ( !tRemoteResult.m_sError.IsEmpty() )
7118 m_dFailuresSet[iRes].SubmitEx ( tFirst.m_sIndexes.cstr(), "agent %s: remote query error: %s", tAgent.GetName().cstr(), tRemoteResult.m_sError.cstr() );
7119 if ( !tRemoteResult.m_sWarning.IsEmpty() )
7120 m_dFailuresSet[iRes].SubmitEx ( tFirst.m_sIndexes.cstr(), "agent %s: remote query warning: %s", tAgent.GetName().cstr(), tRemoteResult.m_sWarning.cstr() );
7121
7122 if ( tRemoteResult.m_iSuccesses<=0 )
7123 continue;
7124
7125 AggrResult_t & tRes = m_dResults[iRes];
7126 tRes.m_iSuccesses++;
7127
7128 ARRAY_FOREACH ( i, tRemoteResult.m_dMatches )
7129 {
7130 tRes.m_dMatches.Add();
7131 tRes.m_dMatches.Last().Clone ( tRemoteResult.m_dMatches[i], tRemoteResult.m_tSchema.GetRowSize() );
7132 tRes.m_dMatches.Last().m_iTag = 0; // all remote MVA values go to special pool which is at index 0
7133 }
7134
7135 tRes.m_dMatchCounts.Add ( tRemoteResult.m_dMatches.GetLength() );
7136 tRes.m_dSchemas.Add ( tRemoteResult.m_tSchema );
7137 // note how we do NOT add per-index weight here; remote agents are all tagged 0 (which contains weight 1)
7138
7139 // merge this agent's stats
7140 tRes.m_iTotalMatches += tRemoteResult.m_iTotalMatches;
7141 tRes.m_iQueryTime += tRemoteResult.m_iQueryTime;
7142
7143 // merge this agent's words
7144 MergeWordStats ( tRes, tRemoteResult.m_hWordStats, &m_dFailuresSet[iRes], tFirst.m_sIndexes.cstr() );
7145 }
7146
7147 // dismissed
7148 tAgent.m_dResults.Reset ();
7149 tAgent.m_bSuccess = false;
7150 tAgent.m_sFailure = "";
7151 }
7152 }
7153
7154 // check if we need to retry again
7155 int iToRetry = 0;
7156 if ( bDist )
7157 ARRAY_FOREACH ( i, dAgents )
7158 if ( dAgents[i].m_eState==AGENT_RETRY )
7159 iToRetry++;
7160 if ( !iToRetry )
7161 break;
7162 }
7163
7164 // submit failures from failed agents
7165 // copy timings from all agents
7166 if ( bDist )
7167 {
7168 ARRAY_FOREACH ( i, dAgents )
7169 {
7170 const AgentConn_t & tAgent = dAgents[i];
7171
7172 for ( int j=iStart; j<=iEnd; j++ )
7173 m_dAgentTimes[j].Add ( tAgent.m_iWall / ( iEnd-iStart+1 ) );
7174
7175 if ( !tAgent.m_bSuccess && !tAgent.m_sFailure.IsEmpty() )
7176 for ( int j=iStart; j<=iEnd; j++ )
7177 m_dFailuresSet[j].SubmitEx ( tFirst.m_sIndexes.cstr(), tAgent.m_bBlackhole ? "blackhole %s: %s" : "agent %s: %s",
7178 tAgent.GetName().cstr(), tAgent.m_sFailure.cstr() );
7179 }
7180 }
7181
7182 ARRAY_FOREACH ( i, m_dResults )
7183 assert ( m_dResults[i].m_iTag==m_dResults[i].m_dTag2Pools.GetLength() );
7184
7185 // cleanup
7186 bool bWasLocalSorter = pLocalSorter!=NULL;
7187 SafeDelete ( pLocalSorter );
7188
7189 /////////////////////
7190 // merge all results
7191 /////////////////////
7192
7193 CSphIOStats tIO;
7194
7195 for ( int iRes=iStart; iRes<=iEnd; iRes++ )
7196 {
7197 AggrResult_t & tRes = m_dResults[iRes];
7198 CSphQuery & tQuery = m_dQueries[iRes];
7199 CSphSchemaMT * pExtraSchema = tQuery.m_bAgent ? m_dExtraSchemas.Begin() + ( bWasLocalSorter ? 0 : iRes ) : NULL;
7200
7201 // minimize sorters needs these pointers
7202 tRes.m_dTag2Pools[0].m_pMva = m_dMvaStorage.Begin();
7203 tRes.m_dTag2Pools[0].m_pStrings = m_dStringsStorage.Begin();
7204 tIO.Add ( tRes.m_tIOStats );
7205
7206 // if there were no successful searches at all, this is an error
7207 if ( !tRes.m_iSuccesses )
7208 {
7209 CSphStringBuilder sFailures;
7210 m_dFailuresSet[iRes].BuildReport ( sFailures );
7211
7212 tRes.m_sError = sFailures.cstr();
7213 continue;
7214 }
7215
7216 // minimize schema and remove dupes
7217 if ( tRes.m_dSchemas.GetLength() )
7218 tRes.m_tSchema = tRes.m_dSchemas[0];
7219 if ( tRes.m_iSuccesses>1 || tQuery.m_dItems.GetLength() )
7220 {
7221 if ( g_bCompatResults && !tQuery.m_bAgent )
7222 {
7223 if ( !MinimizeAggrResultCompat ( tRes, tQuery, m_dLocal.GetLength()!=0 ) )
7224 {
7225 tRes.m_iSuccesses = 0;
7226 return;
7227 }
7228 } else
7229 {
7230 if ( pExtraSchema )
7231 pExtraSchema->RLock();
7232 UnlockOnDestroy SchemaLocker ( pExtraSchema );
7233 if ( !MinimizeAggrResult ( tRes, tQuery, m_dLocal.GetLength()!=0, pExtraSchema, m_bSphinxql ) )
7234 {
7235 tRes.m_iSuccesses = 0;
7236 return;
7237 }
7238 }
7239 }
7240
7241 if ( !m_dFailuresSet[iRes].IsEmpty() )
7242 {
7243 CSphStringBuilder sFailures;
7244 m_dFailuresSet[iRes].BuildReport ( sFailures );
7245 tRes.m_sWarning = sFailures.cstr();
7246 }
7247
7248 ////////////
7249 // finalize
7250 ////////////
7251
7252 tRes.m_iOffset = tQuery.m_iOffset;
7253 tRes.m_iCount = Max ( Min ( tQuery.m_iLimit, tRes.m_dMatches.GetLength()-tQuery.m_iOffset ), 0 );
7254 }
7255
7256 // stats
7257 tmSubset = sphMicroTimer() - tmSubset;
7258 tmCpu = sphCpuTimer() - tmCpu;
7259
7260 // in multi-queue case (1 actual call per N queries), just divide overall query time evenly
7261 // otherwise (N calls per N queries), divide common query time overheads evenly
7262 const int iQueries = iEnd-iStart+1;
7263 if ( m_bMultiQueue )
7264 {
7265 for ( int iRes=iStart; iRes<=iEnd; iRes++ )
7266 {
7267 m_dResults[iRes].m_iQueryTime = (int)( tmSubset/1000/iQueries );
7268 m_dResults[iRes].m_iCpuTime = tmCpu/iQueries;
7269 }
7270 } else
7271 {
7272 int64_t tmAccountedWall = 0;
7273 int64_t tmAccountedCpu = 0;
7274 for ( int iRes=iStart; iRes<=iEnd; iRes++ )
7275 {
7276 tmAccountedWall += m_dResults[iRes].m_iQueryTime*1000;
7277 tmAccountedCpu += m_dResults[iRes].m_iCpuTime;
7278 }
7279
7280 int64_t tmDeltaWall = ( tmSubset - tmAccountedWall ) / iQueries;
7281 int64_t tmDeltaCpu = ( tmCpu - tmAccountedCpu ) / iQueries;
7282
7283 for ( int iRes=iStart; iRes<=iEnd; iRes++ )
7284 {
7285 m_dResults[iRes].m_iQueryTime += (int)(tmDeltaWall/1000);
7286 m_dResults[iRes].m_iCpuTime += tmDeltaCpu;
7287 }
7288 }
7289
7290 if ( g_pStats )
7291 {
7292 g_tStatsMutex.Lock();
7293 g_pStats->m_iQueries += iQueries;
7294 g_pStats->m_iQueryTime += tmSubset;
7295 g_pStats->m_iQueryCpuTime += tmCpu;
7296 if ( bDist && dAgents.GetLength() )
7297 {
7298 // do *not* count queries to dist indexes w/o actual remote agents
7299 g_pStats->m_iDistQueries++;
7300 g_pStats->m_iDistWallTime += tmSubset;
7301 g_pStats->m_iDistLocalTime += tmLocal;
7302 g_pStats->m_iDistWaitTime += tmWait;
7303 }
7304 g_pStats->m_iDiskReads += tIO.m_iReadOps;
7305 g_pStats->m_iDiskReadTime += tIO.m_iReadTime;
7306 g_pStats->m_iDiskReadBytes += tIO.m_iReadBytes;
7307 g_tStatsMutex.Unlock();
7308 }
7309 }
7310
7311
CheckCommandVersion(int iVer,int iDaemonVersion,InputBuffer_c & tReq)7312 bool CheckCommandVersion ( int iVer, int iDaemonVersion, InputBuffer_c & tReq )
7313 {
7314 if ( (iVer>>8)!=(iDaemonVersion>>8) )
7315 {
7316 tReq.SendErrorReply ( "major command version mismatch (expected v.%d.x, got v.%d.%d)",
7317 iDaemonVersion>>8, iVer>>8, iVer&0xff );
7318 return false;
7319 }
7320 if ( iVer>iDaemonVersion )
7321 {
7322 tReq.SendErrorReply ( "client version is higher than daemon version (client is v.%d.%d, daemon is v.%d.%d)",
7323 iVer>>8, iVer&0xff, iDaemonVersion>>8, iDaemonVersion&0xff );
7324 return false;
7325 }
7326 return true;
7327 }
7328
7329
SendSearchResponse(SearchHandler_c & tHandler,InputBuffer_c & tReq,int iSock,int iVer,int iMasterVer)7330 void SendSearchResponse ( SearchHandler_c & tHandler, InputBuffer_c & tReq, int iSock, int iVer, int iMasterVer )
7331 {
7332 // serve the response
7333 NetOutputBuffer_c tOut ( iSock );
7334 int iReplyLen = 0;
7335 bool bExtendedStat = ( iMasterVer>0 );
7336
7337 if ( iVer<=0x10C )
7338 {
7339 assert ( tHandler.m_dQueries.GetLength()==1 );
7340 assert ( tHandler.m_dResults.GetLength()==1 );
7341 const AggrResult_t & tRes = tHandler.m_dResults[0];
7342
7343 if ( !tRes.m_sError.IsEmpty() )
7344 {
7345 tReq.SendErrorReply ( "%s", tRes.m_sError.cstr() );
7346 return;
7347 }
7348
7349 iReplyLen = CalcResultLength ( iVer, &tRes, tRes.m_dTag2Pools, bExtendedStat );
7350 bool bWarning = ( iVer>=0x106 && !tRes.m_sWarning.IsEmpty() );
7351
7352 // send it
7353 tOut.SendWord ( (WORD)( bWarning ? SEARCHD_WARNING : SEARCHD_OK ) );
7354 tOut.SendWord ( VER_COMMAND_SEARCH );
7355 tOut.SendInt ( iReplyLen );
7356
7357 SendResult ( iVer, tOut, &tRes, tRes.m_dTag2Pools, bExtendedStat );
7358
7359 } else
7360 {
7361 ARRAY_FOREACH ( i, tHandler.m_dQueries )
7362 iReplyLen += CalcResultLength ( iVer, &tHandler.m_dResults[i], tHandler.m_dResults[i].m_dTag2Pools, bExtendedStat );
7363
7364 // send it
7365 tOut.SendWord ( (WORD)SEARCHD_OK );
7366 tOut.SendWord ( VER_COMMAND_SEARCH );
7367 tOut.SendInt ( iReplyLen );
7368
7369 ARRAY_FOREACH ( i, tHandler.m_dQueries )
7370 SendResult ( iVer, tOut, &tHandler.m_dResults[i], tHandler.m_dResults[i].m_dTag2Pools, bExtendedStat );
7371 }
7372
7373 tOut.Flush ();
7374 assert ( tOut.GetError()==true || tOut.GetSentCount()==iReplyLen+8 );
7375
7376 // clean up
7377 ARRAY_FOREACH ( i, tHandler.m_dQueries )
7378 SafeDeleteArray ( tHandler.m_dQueries[i].m_pWeights );
7379 }
7380
7381
HandleCommandSearch(int iSock,int iVer,InputBuffer_c & tReq)7382 void HandleCommandSearch ( int iSock, int iVer, InputBuffer_c & tReq )
7383 {
7384 MEMORY ( SPH_MEM_SEARCH_NONSQL );
7385
7386 if ( !CheckCommandVersion ( iVer, VER_COMMAND_SEARCH, tReq ) )
7387 return;
7388
7389 int iMasterVer = 0;
7390 if ( iVer>=0x118 )
7391 iMasterVer = tReq.GetInt();
7392
7393 // parse request
7394 int iQueries = 1;
7395 if ( iVer>=0x10D )
7396 iQueries = tReq.GetDword ();
7397
7398 if ( g_iMaxBatchQueries>0 && ( iQueries<=0 || iQueries>g_iMaxBatchQueries ) )
7399 {
7400 tReq.SendErrorReply ( "bad multi-query count %d (must be in 1..%d range)", iQueries, g_iMaxBatchQueries );
7401 return;
7402 }
7403
7404 SearchHandler_c tHandler ( iQueries );
7405 ARRAY_FOREACH ( i, tHandler.m_dQueries )
7406 if ( !ParseSearchQuery ( tReq, tHandler.m_dQueries[i], iVer, iMasterVer ) )
7407 return;
7408
7409 // run queries, send response
7410 tHandler.RunQueries ();
7411 SendSearchResponse ( tHandler, tReq, iSock, iVer, iMasterVer );
7412 }
7413
7414 //////////////////////////////////////////////////////////////////////////
7415 // SQL PARSER
7416 //////////////////////////////////////////////////////////////////////////
7417
7418 enum SqlStmt_e
7419 {
7420 STMT_PARSE_ERROR = 0,
7421 STMT_DUMMY,
7422
7423 STMT_SELECT,
7424 STMT_INSERT,
7425 STMT_REPLACE,
7426 STMT_DELETE,
7427 STMT_SHOW_WARNINGS,
7428 STMT_SHOW_STATUS,
7429 STMT_SHOW_META,
7430 STMT_SET,
7431 STMT_BEGIN,
7432 STMT_COMMIT,
7433 STMT_ROLLBACK,
7434 STMT_CALL,
7435 STMT_DESC,
7436 STMT_SHOW_TABLES,
7437 STMT_UPDATE,
7438 STMT_CREATE_FUNC,
7439 STMT_DROP_FUNC,
7440 STMT_ATTACH_INDEX,
7441 STMT_FLUSH_RTINDEX,
7442 STMT_SHOW_VARIABLES,
7443
7444 STMT_TOTAL
7445 };
7446
7447
7448 static const char * g_dSqlStmts[STMT_TOTAL] =
7449 {
7450 "parse_error", "dummy", "select", "insert", "replace", "delete", "show_warnings",
7451 "show_status", "show_meta", "set", "begin", "commit", "rollback", "call",
7452 "desc", "show_tables", "update", "create_func", "drop_func", "attach_index", "flush_rtindex", "show_variables"
7453 };
7454
7455
7456 /// refcounted vector
7457 template < typename T >
7458 class RefcountedVector_c : public CSphVector<T>, public ISphRefcounted
7459 {
7460 };
7461
7462 typedef CSphRefcountedPtr < RefcountedVector_c<SphAttr_t> > AttrValues_p;
7463
7464 /// insert value
7465 struct SqlInsert_t
7466 {
7467 int m_iType;
7468 CSphString m_sVal; // OPTIMIZE? use char* and point to node?
7469 int64_t m_iVal;
7470 float m_fVal;
7471 AttrValues_p m_pVals;
7472
SqlInsert_tSqlInsert_t7473 SqlInsert_t ()
7474 : m_pVals ( NULL )
7475 {}
7476 };
7477
7478
7479 /// parser view on a generic node
7480 /// CAUTION, nodes get copied in the parser all the time, must keep assignment slim
7481 struct SqlNode_t
7482 {
7483 int m_iStart;
7484 int m_iEnd;
7485 CSphString m_sValue;
7486 int64_t m_iValue;
7487 float m_fValue;
7488 int m_iInstype; // REMOVE? should not we know this somehow else?
7489 AttrValues_p m_pValues; // FIXME? replace with numeric handles into parser state?
7490
SqlNode_tSqlNode_t7491 SqlNode_t()
7492 : m_iValue ( 0 )
7493 , m_pValues ( NULL )
7494 {}
7495 };
7496 #define YYSTYPE SqlNode_t
7497
7498
7499 enum SqlSet_e
7500 {
7501 SET_LOCAL,
7502 SET_GLOBAL_UVAR,
7503 SET_GLOBAL_SVAR
7504 };
7505
7506 /// parsing result
7507 /// one day, we will start subclassing this
7508 struct SqlStmt_t
7509 {
7510 SqlStmt_e m_eStmt;
7511 int m_iRowsAffected;
7512 const char * m_sStmt; // for error reporting
7513
7514 // SELECT specific
7515 CSphQuery m_tQuery;
7516 CSphVector < CSphRefcountedPtr<UservarIntSet_c> > m_dRefs;
7517
7518 // used by INSERT, DELETE, CALL, DESC, ATTACH
7519 CSphString m_sIndex;
7520
7521 // INSERT (and CALL) specific
7522 CSphVector<SqlInsert_t> m_dInsertValues; // reused by CALL
7523 CSphVector<CSphString> m_dInsertSchema;
7524 int m_iSchemaSz;
7525
7526 // DELETE specific
7527 CSphVector<SphDocID_t> m_dDeleteIds;
7528
7529 // SET specific
7530 CSphString m_sSetName; // reused by ATTACH
7531 SqlSet_e m_eSet;
7532 int m_iSetValue;
7533 CSphString m_sSetValue;
7534 CSphVector<SphAttr_t> m_dSetValues;
7535 bool m_bSetNull;
7536
7537 // CALL specific
7538 CSphString m_sCallProc;
7539 CSphVector<CSphString> m_dCallOptNames;
7540 CSphVector<SqlInsert_t> m_dCallOptValues;
7541 CSphVector<CSphString> m_dCallStrings;
7542
7543 // UPDATE specific
7544 CSphAttrUpdate m_tUpdate;
7545 int m_iListStart; // < the position of start and end of index's definition in original query.
7546 int m_iListEnd;
7547
7548 // CREATE/DROP FUNCTION specific
7549 CSphString m_sUdfName;
7550 CSphString m_sUdfLib;
7551 ESphAttr m_eUdfType;
7552
SqlStmt_tSqlStmt_t7553 SqlStmt_t ()
7554 : m_eStmt ( STMT_PARSE_ERROR )
7555 , m_iRowsAffected ( 0 )
7556 , m_sStmt ( NULL )
7557 , m_iSchemaSz ( 0 )
7558 , m_eSet ( SET_LOCAL )
7559 , m_iSetValue ( 0 )
7560 , m_bSetNull ( false )
7561 , m_iListStart ( -1 )
7562 , m_iListEnd ( -1 )
7563 , m_eUdfType ( SPH_ATTR_NONE )
7564 {
7565 m_tQuery.m_eMode = SPH_MATCH_EXTENDED2; // only new and shiny matching and sorting
7566 m_tQuery.m_eSort = SPH_SORT_EXTENDED;
7567 m_tQuery.m_sSortBy = "@weight desc"; // default order
7568 m_tQuery.m_sOrderBy = "@weight desc";
7569 }
7570
AddSchemaItemSqlStmt_t7571 bool AddSchemaItem ( const char * psName )
7572 {
7573 m_dInsertSchema.Add ( psName );
7574 m_dInsertSchema.Last().ToLower();
7575 m_iSchemaSz = m_dInsertSchema.GetLength();
7576 return true; // stub; check if the given field actually exists in the schema
7577 }
7578
7579 // check if the number of fields which would be inserted is in accordance to the given schema
CheckInsertIntegritySqlStmt_t7580 bool CheckInsertIntegrity()
7581 {
7582 // cheat: if no schema assigned, assume the size of schema as the size of the first row.
7583 // (if it is wrong, it will be revealed later)
7584 if ( !m_iSchemaSz )
7585 m_iSchemaSz = m_dInsertValues.GetLength();
7586
7587 m_iRowsAffected++;
7588 return m_dInsertValues.GetLength()==m_iRowsAffected*m_iSchemaSz;
7589 }
7590 };
7591
7592
7593 struct SqlParser_c : ISphNoncopyable
7594 {
7595 public:
7596 void * m_pScanner;
7597 const char * m_pBuf;
7598 const char * m_pLastTokenStart;
7599 CSphString * m_pParseError;
7600 CSphQuery * m_pQuery;
7601 bool m_bGotQuery;
7602 SqlStmt_t * m_pStmt;
7603 CSphVector<SqlStmt_t> & m_dStmt;
7604 ESphCollation m_eCollation;
7605 BYTE m_uSyntaxFlags;
7606
7607 public:
7608 explicit SqlParser_c ( CSphVector<SqlStmt_t> & dStmt, ESphCollation eCollation );
7609
7610 void PushQuery ();
7611
7612 bool AddOption ( const SqlNode_t & tIdent, const SqlNode_t & tValue );
7613 bool AddOption ( const SqlNode_t & tIdent, const SqlNode_t & tValue, const CSphString & sArg );
7614 bool AddOption ( const SqlNode_t & tIdent, CSphVector<CSphNamedInt> & dNamed );
7615 void AddItem ( SqlNode_t * pExpr, ESphAggrFunc eFunc=SPH_AGGR_NONE, SqlNode_t * pStart=NULL, SqlNode_t * pEnd=NULL );
7616 bool AddItem ( const char * pToken, SqlNode_t * pStart=NULL, SqlNode_t * pEnd=NULL );
7617 void AliasLastItem ( SqlNode_t * pAlias );
SetSelectSqlParser_c7618 void SetSelect ( SqlNode_t * pStart, SqlNode_t * pEnd=NULL )
7619 {
7620 if ( m_pQuery )
7621 {
7622 if ( pStart && ( m_pQuery->m_iSQLSelectStart<0 || m_pQuery->m_iSQLSelectStart>pStart->m_iStart ) )
7623 m_pQuery->m_iSQLSelectStart = pStart->m_iStart;
7624 if ( !pEnd )
7625 pEnd = pStart;
7626 if ( pEnd && ( m_pQuery->m_iSQLSelectEnd<0 || m_pQuery->m_iSQLSelectEnd<pEnd->m_iEnd ) )
7627 m_pQuery->m_iSQLSelectEnd = pEnd->m_iEnd;
7628 }
7629 }
7630 bool AddSchemaItem ( SqlNode_t * pNode );
7631 void SetValue ( const char * sName, const SqlNode_t& tValue );
7632 bool SetMatch ( const SqlNode_t& tValue );
7633 void AddConst ( int iList, const SqlNode_t& tValue );
7634 void SetStatement ( const SqlNode_t& tName, SqlSet_e eSet );
7635 bool AddFloatRangeFilter ( const CSphString & sAttr, float fMin, float fMax );
7636 bool AddIntRangeFilter ( const CSphString & sAttr, int64_t iMin, int64_t iMax );
7637 bool AddIntFilterGTE ( const CSphString & sAttr, int64_t iVal );
7638 bool AddIntFilterLTE ( const CSphString & sAttr, int64_t iVal );
7639 bool AddUservarFilter ( const CSphString & sCol, const CSphString & sVar, bool bExclude );
7640 void SetGroupBy ( const CSphString & sGroupBy );
7641 bool AddDistinct ( SqlNode_t * pNewExpr, SqlNode_t * pStart, SqlNode_t * pEnd );
7642 CSphFilterSettings * AddFilter ( const CSphString & sCol, ESphFilter eType );
AddValuesFilterSqlParser_c7643 inline CSphFilterSettings * AddValuesFilter ( const SqlNode_t& sCol )
7644 {
7645 return AddFilter ( sCol.m_sValue, SPH_FILTER_VALUES );
7646 }
7647
SetOldSyntaxSqlParser_c7648 inline bool SetOldSyntax()
7649 {
7650 m_uSyntaxFlags |= 1;
7651 return IsGoodSyntax ();
7652 }
7653
SetNewSyntaxSqlParser_c7654 inline bool SetNewSyntax()
7655 {
7656 m_uSyntaxFlags |= 2;
7657 return IsGoodSyntax ();
7658 }
7659 bool IsGoodSyntax ();
IsDeprecatedSyntaxSqlParser_c7660 inline bool IsDeprecatedSyntax () const
7661 {
7662 return m_uSyntaxFlags & 1;
7663 }
7664
7665 int AllocNamedVec ();
7666 CSphVector<CSphNamedInt> & GetNamedVec ( int iIndex );
7667 void FreeNamedVec ( int iIndex );
7668 bool UpdateStatement ( SqlNode_t * pNode );
7669 void UpdateAttr ( const CSphString&, const SqlNode_t * pValue, ESphAttr eType = SPH_ATTR_INTEGER );
7670 void UpdateMVAAttr ( const CSphString& sName, const SqlNode_t& dValues );
7671 private:
7672 void AutoAlias ( CSphQueryItem & tItem, SqlNode_t * pStart, SqlNode_t * pEnd );
7673 void AddUpdatedAttr ( const CSphString&, ESphAttr eType );
7674
7675 protected:
7676 bool m_bNamedVecBusy;
7677 CSphVector<CSphNamedInt> m_dNamedVec;
7678 };
7679
AddInsval(CSphVector<SqlInsert_t> & dVec,const SqlNode_t & tNode)7680 static void AddInsval ( CSphVector<SqlInsert_t> & dVec, const SqlNode_t & tNode )
7681 {
7682 SqlInsert_t & tIns = dVec.Add();
7683 tIns.m_iType = tNode.m_iInstype;
7684 tIns.m_iVal = tNode.m_iValue; // OPTIMIZE? copy conditionally based on type?
7685 tIns.m_fVal = tNode.m_fValue;
7686 tIns.m_sVal = tNode.m_sValue;
7687 tIns.m_pVals = tNode.m_pValues;
7688 }
7689
7690 //////////////////////////////////////////////////////////////////////////
7691
7692 // unused parameter, simply to avoid type clash between all my yylex() functions
7693 #define YYLEX_PARAM pParser->m_pScanner, pParser
7694 #ifdef NDEBUG
7695 #define YY_DECL int yylex ( YYSTYPE * lvalp, void * yyscanner, SqlParser_c * pParser )
7696 #else
7697 #define YY_DECL int yylexd ( YYSTYPE * lvalp, void * yyscanner, SqlParser_c * pParser )
7698 #endif
7699 #include "llsphinxql.c"
7700
7701
yyerror(SqlParser_c * pParser,const char * sMessage)7702 void yyerror ( SqlParser_c * pParser, const char * sMessage )
7703 {
7704 // flex put a zero at last token boundary; make it undo that
7705 yylex_unhold ( pParser->m_pScanner );
7706
7707 // create our error message
7708 pParser->m_pParseError->SetSprintf ( "sphinxql: %s near '%s'", sMessage, pParser->m_pLastTokenStart ? pParser->m_pLastTokenStart : "(null)" );
7709
7710 // fixup TOK_xxx thingies
7711 char * s = const_cast<char*> ( pParser->m_pParseError->cstr() );
7712 char * d = s;
7713 while ( *s )
7714 {
7715 if ( strncmp ( s, "TOK_", 4 )==0 )
7716 s += 4;
7717 else
7718 *d++ = *s++;
7719 }
7720 *d = '\0';
7721 }
7722
7723
7724 #ifndef NDEBUG
7725 // using a proxy to be possible to debug inside yylex
yylex(YYSTYPE * lvalp,void * yyscanner,SqlParser_c * pParser)7726 int yylex ( YYSTYPE * lvalp, void * yyscanner, SqlParser_c * pParser )
7727 {
7728 int res = yylexd ( lvalp, yyscanner, pParser );
7729 return res;
7730 }
7731 #endif
7732
7733 #include "yysphinxql.c"
7734
7735 //////////////////////////////////////////////////////////////////////////
7736
7737 class CSphMatchVariant : public CSphMatch
7738 {
7739 public:
ToInt(const SqlInsert_t & tVal)7740 inline static SphAttr_t ToInt ( const SqlInsert_t & tVal )
7741 {
7742 switch ( tVal.m_iType )
7743 {
7744 case TOK_QUOTED_STRING : return strtoul ( tVal.m_sVal.cstr(), NULL, 10 ); // FIXME? report conversion error?
7745 case TOK_CONST_INT: return int(tVal.m_iVal);
7746 case TOK_CONST_FLOAT: return int(tVal.m_fVal); // FIXME? report conversion error
7747 }
7748 return 0;
7749 }
ToBigInt(const SqlInsert_t & tVal)7750 inline static SphAttr_t ToBigInt ( const SqlInsert_t & tVal )
7751 {
7752 switch ( tVal.m_iType )
7753 {
7754 case TOK_QUOTED_STRING : return strtoll ( tVal.m_sVal.cstr(), NULL, 10 ); // FIXME? report conversion error?
7755 case TOK_CONST_INT: return tVal.m_iVal;
7756 case TOK_CONST_FLOAT: return int(tVal.m_fVal); // FIXME? report conversion error?
7757 }
7758 return 0;
7759 }
7760 #if USE_64BIT
7761 #define ToDocid ToBigInt
7762 #else
7763 #define ToDocid ToInt
7764 #endif // USE_64BIT
7765
SetAttr(const CSphAttrLocator & tLoc,const SqlInsert_t & tVal,ESphAttr eTargetType)7766 bool SetAttr ( const CSphAttrLocator & tLoc, const SqlInsert_t & tVal, ESphAttr eTargetType )
7767 {
7768 switch ( eTargetType )
7769 {
7770 case SPH_ATTR_INTEGER:
7771 case SPH_ATTR_TIMESTAMP:
7772 CSphMatch::SetAttr ( tLoc, ToInt(tVal) );
7773 break;
7774 case SPH_ATTR_BIGINT:
7775 CSphMatch::SetAttr ( tLoc, ToBigInt(tVal) );
7776 break;
7777 case SPH_ATTR_FLOAT:
7778 if ( tVal.m_iType==TOK_QUOTED_STRING )
7779 SetAttrFloat ( tLoc, (float)strtod ( tVal.m_sVal.cstr(), NULL ) ); // FIXME? report conversion error?
7780 else if ( tVal.m_iType==TOK_CONST_INT )
7781 SetAttrFloat ( tLoc, float(tVal.m_iVal) ); // FIXME? report conversion error?
7782 else if ( tVal.m_iType==TOK_CONST_FLOAT )
7783 SetAttrFloat ( tLoc, tVal.m_fVal );
7784 break;
7785 case SPH_ATTR_STRING:
7786 case SPH_ATTR_UINT32SET:
7787 case SPH_ATTR_INT64SET:
7788 CSphMatch::SetAttr ( tLoc, 0 );
7789 break;
7790 default:
7791 return false;
7792 };
7793 return true;
7794 }
7795
SetDefaultAttr(const CSphAttrLocator & tLoc,ESphAttr eTargetType)7796 inline bool SetDefaultAttr ( const CSphAttrLocator & tLoc, ESphAttr eTargetType )
7797 {
7798 SqlInsert_t tVal;
7799 tVal.m_iType = TOK_CONST_INT;
7800 tVal.m_iVal = 0;
7801 return SetAttr ( tLoc, tVal, eTargetType );
7802 }
7803 };
7804
SqlParser_c(CSphVector<SqlStmt_t> & dStmt,ESphCollation eCollation)7805 SqlParser_c::SqlParser_c ( CSphVector<SqlStmt_t> & dStmt, ESphCollation eCollation )
7806 : m_pQuery ( NULL )
7807 , m_pStmt ( NULL )
7808 , m_dStmt ( dStmt )
7809 , m_eCollation ( eCollation )
7810 , m_uSyntaxFlags ( 0 )
7811 , m_bNamedVecBusy ( false )
7812 {
7813 assert ( !m_dStmt.GetLength() );
7814 PushQuery ();
7815 }
7816
PushQuery()7817 void SqlParser_c::PushQuery ()
7818 {
7819 assert ( m_dStmt.GetLength() || ( !m_pQuery && !m_pStmt ) );
7820
7821 // post set proper result-set order
7822 if ( m_dStmt.GetLength() )
7823 {
7824 if ( m_pQuery->m_sGroupBy.IsEmpty() )
7825 m_pQuery->m_sSortBy = m_pQuery->m_sOrderBy;
7826 else
7827 m_pQuery->m_sGroupSortBy = m_pQuery->m_sOrderBy;
7828 }
7829
7830 // add new
7831 m_dStmt.Add ( SqlStmt_t() );
7832 m_pStmt = &m_dStmt.Last();
7833 m_pQuery = &m_pStmt->m_tQuery;
7834 m_pQuery->m_eCollation = m_eCollation;
7835
7836 m_bGotQuery = false;
7837 }
7838
AddOption(const SqlNode_t & tIdent,const SqlNode_t & tValue)7839 bool SqlParser_c::AddOption ( const SqlNode_t& tIdent, const SqlNode_t& tValue )
7840 {
7841 CSphString sOpt = tIdent.m_sValue;
7842 CSphString sVal = tValue.m_sValue;
7843 sOpt.ToLower ();
7844 sVal.ToLower ();
7845
7846 if ( sOpt=="ranker" )
7847 {
7848 if ( sVal=="proximity_bm25" ) m_pQuery->m_eRanker = SPH_RANK_PROXIMITY_BM25;
7849 else if ( sVal=="bm25" ) m_pQuery->m_eRanker = SPH_RANK_BM25;
7850 else if ( sVal=="none" ) m_pQuery->m_eRanker = SPH_RANK_NONE;
7851 else if ( sVal=="wordcount" ) m_pQuery->m_eRanker = SPH_RANK_WORDCOUNT;
7852 else if ( sVal=="proximity" ) m_pQuery->m_eRanker = SPH_RANK_PROXIMITY;
7853 else if ( sVal=="matchany" ) m_pQuery->m_eRanker = SPH_RANK_MATCHANY;
7854 else if ( sVal=="fieldmask" ) m_pQuery->m_eRanker = SPH_RANK_FIELDMASK;
7855 else if ( sVal=="sph04" ) m_pQuery->m_eRanker = SPH_RANK_SPH04;
7856 else if ( sVal=="expr" )
7857 {
7858 m_pParseError->SetSprintf ( "missing ranker expression (use OPTION ranker=expr('1+2') for example)" );
7859 return false;
7860 } else
7861 {
7862 m_pParseError->SetSprintf ( "unknown ranker '%s'", sVal.cstr() );
7863 return false;
7864 }
7865
7866 } else if ( sOpt=="max_matches" )
7867 {
7868 m_pQuery->m_iMaxMatches = (int)tValue.m_iValue;
7869
7870 } else if ( sOpt=="cutoff" )
7871 {
7872 m_pQuery->m_iCutoff = (int)tValue.m_iValue;
7873
7874 } else if ( sOpt=="max_query_time" )
7875 {
7876 m_pQuery->m_uMaxQueryMsec = (int)tValue.m_iValue;
7877
7878 } else if ( sOpt=="retry_count" )
7879 {
7880 m_pQuery->m_iRetryCount = (int)tValue.m_iValue;
7881
7882 } else if ( sOpt=="retry_delay" )
7883 {
7884 m_pQuery->m_iRetryDelay = (int)tValue.m_iValue;
7885
7886 } else if ( sOpt=="reverse_scan" )
7887 {
7888 m_pQuery->m_bReverseScan = ( tValue.m_iValue!=0 );
7889
7890 } else if ( sOpt=="comment" )
7891 {
7892 m_pQuery->m_sComment = tValue.m_sValue;
7893
7894 } else
7895 {
7896 m_pParseError->SetSprintf ( "unknown option '%s' (or bad argument type)", tIdent.m_sValue.cstr() );
7897 return false;
7898 }
7899
7900 return true;
7901 }
7902
7903
AddOption(const SqlNode_t & tIdent,const SqlNode_t & tValue,const CSphString & sArg)7904 bool SqlParser_c::AddOption ( const SqlNode_t & tIdent, const SqlNode_t & tValue, const CSphString & sArg )
7905 {
7906 CSphString sOpt = tIdent.m_sValue;
7907 CSphString sVal = tValue.m_sValue;
7908 sOpt.ToLower ();
7909 sVal.ToLower ();
7910
7911 if ( sOpt=="ranker" && sVal=="expr" )
7912 {
7913 m_pQuery->m_eRanker = SPH_RANK_EXPR;
7914 m_pQuery->m_sRankerExpr = sArg;
7915 return true;
7916 } else
7917 {
7918 m_pParseError->SetSprintf ( "unknown option or extra argument to '%s=%s'", tIdent.m_sValue.cstr(), tValue.m_sValue.cstr() );
7919 return false;
7920 }
7921 }
7922
7923
AddOption(const SqlNode_t & tIdent,CSphVector<CSphNamedInt> & dNamed)7924 bool SqlParser_c::AddOption ( const SqlNode_t & tIdent, CSphVector<CSphNamedInt> & dNamed )
7925 {
7926 CSphString sOpt = tIdent.m_sValue;
7927 sOpt.ToLower ();
7928
7929 if ( sOpt=="field_weights" )
7930 {
7931 m_pQuery->m_dFieldWeights.SwapData ( dNamed );
7932
7933 } else if ( sOpt=="index_weights" )
7934 {
7935 m_pQuery->m_dIndexWeights.SwapData ( dNamed );
7936
7937 } else
7938 {
7939 m_pParseError->SetSprintf ( "unknown option '%s' (or bad argument type)", tIdent.m_sValue.cstr() );
7940 return false;
7941 }
7942
7943 return true;
7944 }
7945
AliasLastItem(SqlNode_t * pAlias)7946 void SqlParser_c::AliasLastItem ( SqlNode_t * pAlias )
7947 {
7948 if ( pAlias )
7949 {
7950 CSphQueryItem & tItem = m_pQuery->m_dItems.Last();
7951 tItem.m_sAlias.SetBinary ( m_pBuf + pAlias->m_iStart, pAlias->m_iEnd - pAlias->m_iStart );
7952 tItem.m_sAlias.ToLower();
7953 SetSelect ( pAlias );
7954 }
7955 }
7956
AutoAlias(CSphQueryItem & tItem,SqlNode_t * pStart,SqlNode_t * pEnd)7957 void SqlParser_c::AutoAlias ( CSphQueryItem & tItem, SqlNode_t * pStart, SqlNode_t * pEnd )
7958 {
7959 if ( pStart && pEnd )
7960 {
7961 tItem.m_sAlias.SetBinary ( m_pBuf + pStart->m_iStart, pEnd->m_iEnd - pStart->m_iStart );
7962 tItem.m_sAlias.ToLower();
7963 } else
7964 tItem.m_sAlias = tItem.m_sExpr;
7965 SetSelect ( pStart, pEnd );
7966 }
7967
AddItem(SqlNode_t * pExpr,ESphAggrFunc eAggrFunc,SqlNode_t * pStart,SqlNode_t * pEnd)7968 void SqlParser_c::AddItem ( SqlNode_t * pExpr, ESphAggrFunc eAggrFunc, SqlNode_t * pStart, SqlNode_t * pEnd )
7969 {
7970 CSphQueryItem & tItem = m_pQuery->m_dItems.Add();
7971 tItem.m_sExpr.SetBinary ( m_pBuf + pExpr->m_iStart, pExpr->m_iEnd - pExpr->m_iStart );
7972 tItem.m_sExpr.ToLower();
7973 tItem.m_eAggrFunc = eAggrFunc;
7974 AutoAlias ( tItem, pStart?pStart:pExpr, pEnd?pEnd:pExpr );
7975 }
7976
AddItem(const char * pToken,SqlNode_t * pStart,SqlNode_t * pEnd)7977 bool SqlParser_c::AddItem ( const char * pToken, SqlNode_t * pStart, SqlNode_t * pEnd )
7978 {
7979 CSphQueryItem & tItem = m_pQuery->m_dItems.Add();
7980 tItem.m_sExpr = pToken;
7981 tItem.m_eAggrFunc = SPH_AGGR_NONE;
7982 tItem.m_sExpr.ToLower();
7983 AutoAlias ( tItem, pStart, pEnd );
7984 return SetNewSyntax();
7985 }
7986
SetGroupBy(const CSphString & sGroupBy)7987 void SqlParser_c::SetGroupBy ( const CSphString & sGroupBy )
7988 {
7989 m_pQuery->m_eGroupFunc = SPH_GROUPBY_ATTR;
7990 m_pQuery->m_sGroupBy = sGroupBy;
7991 m_pQuery->m_sGroupBy.ToLower();
7992 }
7993
AddDistinct(SqlNode_t * pNewExpr,SqlNode_t * pStart,SqlNode_t * pEnd)7994 bool SqlParser_c::AddDistinct ( SqlNode_t * pNewExpr, SqlNode_t * pStart, SqlNode_t * pEnd )
7995 {
7996 if ( !m_pQuery->m_sGroupDistinct.IsEmpty() )
7997 {
7998 yyerror ( this, "too many COUNT(DISTINCT) clauses" );
7999 return false;
8000 }
8001
8002 m_pQuery->m_sGroupDistinct = pNewExpr->m_sValue;
8003 return AddItem ( "@distinct", pStart, pEnd );
8004 }
8005
AddSchemaItem(YYSTYPE * pNode)8006 bool SqlParser_c::AddSchemaItem ( YYSTYPE * pNode )
8007 {
8008 assert ( m_pStmt );
8009 CSphString sItem;
8010 sItem.SetBinary ( m_pBuf + pNode->m_iStart, pNode->m_iEnd - pNode->m_iStart );
8011 return m_pStmt->AddSchemaItem ( sItem.cstr() );
8012 }
8013
SetMatch(const YYSTYPE & tValue)8014 bool SqlParser_c::SetMatch ( const YYSTYPE& tValue )
8015 {
8016 if ( m_bGotQuery )
8017 {
8018 yyerror ( this, "too many MATCH() clauses" );
8019 return false;
8020 };
8021
8022 m_pQuery->m_sQuery = tValue.m_sValue;
8023 m_pQuery->m_sRawQuery = tValue.m_sValue;
8024 return m_bGotQuery = true;
8025 }
8026
AddConst(int iList,const YYSTYPE & tValue)8027 void SqlParser_c::AddConst ( int iList, const YYSTYPE& tValue )
8028 {
8029 CSphVector<CSphNamedInt> & dVec = GetNamedVec ( iList );
8030
8031 dVec.Add();
8032 dVec.Last().m_sName = tValue.m_sValue;
8033 dVec.Last().m_sName.ToLower();
8034 dVec.Last().m_iValue = (int) tValue.m_iValue;
8035 }
8036
SetStatement(const YYSTYPE & tName,SqlSet_e eSet)8037 void SqlParser_c::SetStatement ( const YYSTYPE& tName, SqlSet_e eSet )
8038 {
8039 m_pStmt->m_eStmt = STMT_SET;
8040 m_pStmt->m_eSet = eSet;
8041 m_pStmt->m_sSetName = tName.m_sValue;
8042 }
8043
UpdateStatement(SqlNode_t * pNode)8044 bool SqlParser_c::UpdateStatement ( SqlNode_t * pNode )
8045 {
8046 m_pStmt->m_eStmt = STMT_UPDATE;
8047 m_pStmt->m_iListStart = pNode->m_iStart;
8048 m_pStmt->m_iListEnd = pNode->m_iEnd;
8049 m_pStmt->m_sIndex.SetBinary ( m_pBuf + pNode->m_iStart, pNode->m_iEnd - pNode->m_iStart );
8050 m_pStmt->m_tUpdate.m_dRowOffset.Add ( 0 );
8051 return true;
8052 }
8053
AddUpdatedAttr(const CSphString & sName,ESphAttr eType)8054 void SqlParser_c::AddUpdatedAttr ( const CSphString& sName, ESphAttr eType )
8055 {
8056 CSphAttrUpdate & tUpd = m_pStmt->m_tUpdate;
8057 CSphColumnInfo & tAttr = tUpd.m_dAttrs.Add();
8058 tAttr.m_sName = sName;
8059 tAttr.m_sName.ToLower();
8060 tAttr.m_eAttrType = eType; // sorry, ints only for now, riding on legacy shit!
8061 }
8062
UpdateAttr(const CSphString & sName,const SqlNode_t * pValue,ESphAttr eType)8063 void SqlParser_c::UpdateAttr ( const CSphString& sName, const SqlNode_t * pValue, ESphAttr eType )
8064 {
8065 assert ( eType==SPH_ATTR_FLOAT || eType==SPH_ATTR_INTEGER || eType==SPH_ATTR_BIGINT );
8066 if ( eType==SPH_ATTR_FLOAT )
8067 {
8068 m_pStmt->m_tUpdate.m_dPool.Add ( *(const DWORD*)( &pValue->m_fValue ) );
8069
8070 } else if ( eType==SPH_ATTR_INTEGER || eType==SPH_ATTR_BIGINT )
8071 {
8072 m_pStmt->m_tUpdate.m_dPool.Add ( (DWORD) pValue->m_iValue );
8073 DWORD uHi = (DWORD) ( pValue->m_iValue>>32 );
8074 if ( uHi )
8075 {
8076 m_pStmt->m_tUpdate.m_dPool.Add ( uHi );
8077 eType = SPH_ATTR_BIGINT;
8078 }
8079 }
8080 AddUpdatedAttr ( sName, eType );
8081 }
8082
UpdateMVAAttr(const CSphString & sName,const SqlNode_t & dValues)8083 void SqlParser_c::UpdateMVAAttr ( const CSphString & sName, const SqlNode_t & dValues )
8084 {
8085 CSphAttrUpdate & tUpd = m_pStmt->m_tUpdate;
8086 ESphAttr eType = SPH_ATTR_UINT32SET;
8087
8088 if ( dValues.m_pValues.Ptr() && dValues.m_pValues->GetLength()>0 )
8089 {
8090 // got MVA values, let's process them
8091 dValues.m_pValues->Uniq(); // don't need dupes within MVA
8092 tUpd.m_dPool.Add ( dValues.m_pValues->GetLength()*2 );
8093 SphAttr_t * pVal = dValues.m_pValues.Ptr()->Begin();
8094 SphAttr_t * pValMax = pVal + dValues.m_pValues->GetLength();
8095 for ( ;pVal<pValMax; pVal++ )
8096 {
8097 SphAttr_t uVal = *pVal;
8098 if ( uVal>UINT_MAX )
8099 {
8100 eType = SPH_ATTR_INT64SET;
8101 }
8102 tUpd.m_dPool.Add ( (DWORD)uVal );
8103 tUpd.m_dPool.Add ( (DWORD)( uVal>>32 ) );
8104 }
8105 } else
8106 {
8107 // no values, means we should delete the attribute
8108 // we signal that to the update code by putting a single zero
8109 // to the values pool (meaning a zero-length MVA values list)
8110 tUpd.m_dPool.Add ( 0 );
8111 }
8112
8113 AddUpdatedAttr ( sName, eType );
8114 }
8115
AddFilter(const CSphString & sCol,ESphFilter eType)8116 CSphFilterSettings * SqlParser_c::AddFilter ( const CSphString & sCol, ESphFilter eType )
8117 {
8118 if ( sCol=="@count" || sCol=="count(*)" )
8119 {
8120 yyerror ( this, "Aggregates in 'where' clause prohibited" );
8121 return NULL;
8122 }
8123 CSphFilterSettings * pFilter = &m_pQuery->m_dFilters.Add();
8124 pFilter->m_sAttrName = ( sCol=="id" ) ? "@id" : sCol;
8125 pFilter->m_eType = eType;
8126 pFilter->m_sAttrName.ToLower();
8127 return pFilter;
8128 }
8129
AddFloatRangeFilter(const CSphString & sAttr,float fMin,float fMax)8130 bool SqlParser_c::AddFloatRangeFilter ( const CSphString & sAttr, float fMin, float fMax )
8131 {
8132 CSphFilterSettings * pFilter = AddFilter ( sAttr, SPH_FILTER_FLOATRANGE );
8133 if ( !pFilter )
8134 return false;
8135 pFilter->m_fMinValue = fMin;
8136 pFilter->m_fMaxValue = fMax;
8137 return true;
8138 }
8139
AddIntRangeFilter(const CSphString & sAttr,int64_t iMin,int64_t iMax)8140 bool SqlParser_c::AddIntRangeFilter ( const CSphString & sAttr, int64_t iMin, int64_t iMax )
8141 {
8142 CSphFilterSettings * pFilter = AddFilter ( sAttr, SPH_FILTER_RANGE );
8143 if ( !pFilter )
8144 return false;
8145 pFilter->m_iMinValue = iMin;
8146 pFilter->m_iMaxValue = iMax;
8147 return true;
8148 }
8149
AddIntFilterGTE(const CSphString & sAttr,int64_t iVal)8150 bool SqlParser_c::AddIntFilterGTE ( const CSphString & sAttr, int64_t iVal )
8151 {
8152 CSphFilterSettings * pFilter = AddFilter ( sAttr, SPH_FILTER_RANGE );
8153 if ( !pFilter )
8154 return false;
8155 bool bId = ( sAttr=="@id" ) || ( sAttr=="id" );
8156 pFilter->m_iMinValue = iVal;
8157 pFilter->m_iMaxValue = bId ? (SphAttr_t)ULLONG_MAX : LLONG_MAX;
8158 return true;
8159 }
8160
AddIntFilterLTE(const CSphString & sAttr,int64_t iVal)8161 bool SqlParser_c::AddIntFilterLTE ( const CSphString & sAttr, int64_t iVal )
8162 {
8163 CSphFilterSettings * pFilter = AddFilter ( sAttr, SPH_FILTER_RANGE );
8164 if ( !pFilter )
8165 return false;
8166 bool bId = ( sAttr=="@id" ) || ( sAttr=="id" );
8167 pFilter->m_iMinValue = bId ? 0 : LLONG_MIN;
8168 pFilter->m_iMaxValue = iVal;
8169 return true;
8170 }
8171
AddUservarFilter(const CSphString & sCol,const CSphString & sVar,bool bExclude)8172 bool SqlParser_c::AddUservarFilter ( const CSphString & sCol, const CSphString & sVar, bool bExclude )
8173 {
8174 CSphScopedLock<StaticThreadsOnlyMutex_t> tLock ( g_tUservarsMutex );
8175 Uservar_t * pVar = g_hUservars ( sVar );
8176 if ( !pVar )
8177 {
8178 yyerror ( this, "undefined global variable in IN clause" );
8179 return false;
8180 }
8181
8182 assert ( pVar->m_eType==USERVAR_INT_SET );
8183 CSphFilterSettings * pFilter = AddFilter ( sCol, SPH_FILTER_VALUES );
8184 if ( !pFilter )
8185 return false;
8186 pFilter->m_bExclude = bExclude;
8187
8188 // tricky black magic
8189 // we want to avoid copying the data, hence external values in the filter
8190 // we need to guarantee the data (uservar value) lifetime, then
8191 // suddenly, enter mutex-protected refcounted value objects
8192 // suddenly, we need to track those values in the statement object, too
8193 assert ( pVar->m_pVal );
8194 CSphRefcountedPtr<UservarIntSet_c> & tRef = m_pStmt->m_dRefs.Add();
8195 tRef = pVar->m_pVal; // take over semantics, and thus NO (!) automatic addref
8196 pVar->m_pVal->AddRef(); // so do that addref manually
8197 pFilter->SetExternalValues ( pVar->m_pVal->Begin(), pVar->m_pVal->GetLength() );
8198 return true;
8199 }
8200
IsGoodSyntax()8201 bool SqlParser_c::IsGoodSyntax ()
8202 {
8203 if ( ( m_uSyntaxFlags & 3 )!=3 )
8204 return true;
8205 yyerror ( this, "Mixing the old-fashion internal vars (@id, @count, @weight) with new acronyms like count(*), weight() is prohibited" );
8206 return false;
8207 }
8208
8209
AllocNamedVec()8210 int SqlParser_c::AllocNamedVec ()
8211 {
8212 // we only allow one such vector at a time, right now
8213 assert ( !m_bNamedVecBusy );
8214 m_bNamedVecBusy = true;
8215 m_dNamedVec.Resize ( 0 );
8216 return 0;
8217 }
8218
8219 #ifndef NDEBUG
GetNamedVec(int iIndex)8220 CSphVector<CSphNamedInt> & SqlParser_c::GetNamedVec ( int iIndex )
8221 #else
8222 CSphVector<CSphNamedInt> & SqlParser_c::GetNamedVec ( int )
8223 #endif
8224 {
8225 assert ( m_bNamedVecBusy && iIndex==0 );
8226 return m_dNamedVec;
8227 }
8228
8229 #ifndef NDEBUG
FreeNamedVec(int iIndex)8230 void SqlParser_c::FreeNamedVec ( int iIndex )
8231 #else
8232 void SqlParser_c::FreeNamedVec ( int )
8233 #endif
8234 {
8235 assert ( m_bNamedVecBusy && iIndex==0 );
8236 m_bNamedVecBusy = false;
8237 m_dNamedVec.Resize ( 0 );
8238 }
8239
ParseSqlQuery(const CSphString & sQuery,CSphVector<SqlStmt_t> & dStmt,CSphString & sError,ESphCollation eCollation)8240 bool ParseSqlQuery ( const CSphString & sQuery, CSphVector<SqlStmt_t> & dStmt, CSphString & sError, ESphCollation eCollation )
8241 {
8242 SqlParser_c tParser ( dStmt, eCollation );
8243 tParser.m_pBuf = sQuery.cstr();
8244 tParser.m_pLastTokenStart = NULL;
8245 tParser.m_pParseError = &sError;
8246 tParser.m_eCollation = eCollation;
8247
8248 int iLen = strlen ( sQuery.cstr() );
8249 char * sEnd = (char*)sQuery.cstr() + iLen;
8250 sEnd[0] = 0; // prepare for yy_scan_buffer
8251 sEnd[1] = 0; // this is ok because string allocates a small gap
8252
8253 yylex_init ( &tParser.m_pScanner );
8254 YY_BUFFER_STATE tLexerBuffer = yy_scan_buffer ( (char*)sQuery.cstr(), iLen+2, tParser.m_pScanner );
8255 if ( !tLexerBuffer )
8256 {
8257 sError = "internal error: yy_scan_buffer() failed";
8258 return false;
8259 }
8260
8261 int iRes = yyparse ( &tParser );
8262 yy_delete_buffer ( tLexerBuffer, tParser.m_pScanner );
8263 yylex_destroy ( tParser.m_pScanner );
8264
8265 dStmt.Pop(); // last query is always dummy
8266
8267 ARRAY_FOREACH ( i, dStmt )
8268 {
8269 CSphQuery & tQuery = dStmt[i].m_tQuery;
8270 if ( tQuery.m_iSQLSelectStart>=0 )
8271 {
8272 tQuery.m_sSelect.SetBinary ( tParser.m_pBuf + tQuery.m_iSQLSelectStart,
8273 tQuery.m_iSQLSelectEnd - tQuery.m_iSQLSelectStart );
8274 }
8275 }
8276
8277 if ( iRes!=0 || !dStmt.GetLength() )
8278 return false;
8279
8280 if ( tParser.IsDeprecatedSyntax() )
8281 sError = "Using the old-fashion @variables (@count, @weight, etc.) is deprecated";
8282
8283 return true;
8284 }
8285
8286
8287 /////////////////////////////////////////////////////////////////////////////
8288
sphGetPassageBoundary(const CSphString & sPassageBoundaryMode)8289 ESphSpz sphGetPassageBoundary ( const CSphString & sPassageBoundaryMode )
8290 {
8291 if ( sPassageBoundaryMode.IsEmpty() )
8292 return SPH_SPZ_NONE;
8293
8294 ESphSpz eSPZ = SPH_SPZ_NONE;
8295 if ( sPassageBoundaryMode=="sentence" )
8296 eSPZ = SPH_SPZ_SENTENCE;
8297 else if ( sPassageBoundaryMode=="paragraph" )
8298 eSPZ = SPH_SPZ_PARAGRAPH;
8299 else if ( sPassageBoundaryMode=="zone" )
8300 eSPZ = SPH_SPZ_ZONE;
8301
8302 return eSPZ;
8303 }
8304
sphCheckOptionsSPZ(const ExcerptQuery_t & q,const CSphString & sPassageBoundaryMode,CSphString & sError)8305 bool sphCheckOptionsSPZ ( const ExcerptQuery_t & q, const CSphString & sPassageBoundaryMode, CSphString & sError )
8306 {
8307 if ( q.m_ePassageSPZ )
8308 {
8309 if ( q.m_iAround==0 )
8310 {
8311 sError.SetSprintf ( "invalid combination of passage_boundary=%s and around=%d", sPassageBoundaryMode.cstr(), q.m_iAround );
8312 return false;
8313 } else if ( q.m_bUseBoundaries )
8314 {
8315 sError.SetSprintf ( "invalid combination of passage_boundary=%s and use_boundaries", sPassageBoundaryMode.cstr() );
8316 return false;
8317 }
8318 }
8319
8320 if ( q.m_bEmitZones )
8321 {
8322 if ( q.m_ePassageSPZ!=SPH_SPZ_ZONE )
8323 {
8324 sError.SetSprintf ( "invalid combination of passage_boundary=%s and emit_zones", sPassageBoundaryMode.cstr() );
8325 return false;
8326 }
8327 if ( !( q.m_sStripMode=="strip" || q.m_sStripMode=="index" ) )
8328 {
8329 sError.SetSprintf ( "invalid combination of strip=%s and emit_zones", q.m_sStripMode.cstr() );
8330 return false;
8331 }
8332 }
8333
8334 return true;
8335 }
8336
8337 /////////////////////////////////////////////////////////////////////////////
8338 // EXCERPTS HANDLER
8339 /////////////////////////////////////////////////////////////////////////////
8340
8341 enum eExcerpt_Flags
8342 {
8343 EXCERPT_FLAG_REMOVESPACES = 1,
8344 EXCERPT_FLAG_EXACTPHRASE = 2,
8345 EXCERPT_FLAG_SINGLEPASSAGE = 4,
8346 EXCERPT_FLAG_USEBOUNDARIES = 8,
8347 EXCERPT_FLAG_WEIGHTORDER = 16,
8348 EXCERPT_FLAG_QUERY = 32,
8349 EXCERPT_FLAG_FORCE_ALL_WORDS = 64,
8350 EXCERPT_FLAG_LOAD_FILES = 128,
8351 EXCERPT_FLAG_ALLOW_EMPTY = 256,
8352 EXCERPT_FLAG_EMIT_ZONES = 512,
8353 EXCERPT_FLAG_FILES_SCATTERED = 1024
8354 };
8355
8356 enum
8357 {
8358 PROCESSED_ITEM = -2,
8359 EOF_ITEM = -1
8360 };
8361 struct SnippetWorker_t
8362 {
8363 int64_t m_iTotal;
8364 int m_iHead;
8365 bool m_bLocal;
8366
SnippetWorker_tSnippetWorker_t8367 SnippetWorker_t()
8368 : m_iTotal ( 0 )
8369 , m_iHead ( EOF_ITEM )
8370 , m_bLocal ( false )
8371 {}
8372 };
8373
8374 struct SnippetsRemote_t : ISphNoncopyable
8375 {
8376 CSphVector<AgentConn_t> m_dAgents;
8377 CSphVector<SnippetWorker_t> m_dWorkers;
8378 CSphVector<ExcerptQuery_t> & m_dQueries;
8379 int m_iAgentConnectTimeout;
8380 int m_iAgentQueryTimeout;
8381
SnippetsRemote_tSnippetsRemote_t8382 explicit SnippetsRemote_t ( CSphVector<ExcerptQuery_t> & dQueries )
8383 : m_dQueries ( dQueries )
8384 , m_iAgentConnectTimeout ( 0 )
8385 , m_iAgentQueryTimeout ( 0 )
8386 {}
8387 };
8388
8389 struct SnippetThread_t
8390 {
8391 SphThread_t m_tThd;
8392 CSphMutex * m_pLock;
8393 int m_iQueries;
8394 ExcerptQuery_t * m_pQueries;
8395 volatile int * m_pCurQuery;
8396 CSphIndex * m_pIndex;
8397 CrashQuery_t m_tCrashQuery;
8398
SnippetThread_tSnippetThread_t8399 SnippetThread_t()
8400 : m_pLock ( NULL )
8401 , m_iQueries ( 0 )
8402 , m_pQueries ( NULL )
8403 , m_pCurQuery ( NULL )
8404 , m_pIndex ( NULL )
8405 {}
8406 };
8407
8408 struct SnippetRequestBuilder_t : public IRequestBuilder_t
8409 {
SnippetRequestBuilder_tSnippetRequestBuilder_t8410 explicit SnippetRequestBuilder_t ( const SnippetsRemote_t * pWorker )
8411 : m_pWorker ( pWorker )
8412 , m_iLastAgent ( -1 )
8413 , m_iLastWorker ( -1 )
8414 , m_iNumDocs ( -1 )
8415 , m_iReqLen ( -1 )
8416 , m_bScattered ( false )
8417 {}
8418 virtual void BuildRequest ( const char * sIndexes, NetOutputBuffer_c & tOut, int iNumAgent ) const;
8419
8420 private:
8421 const SnippetsRemote_t * m_pWorker;
8422 mutable int m_iLastAgent; ///< just a helper to optimize consequental linear search
8423 mutable int m_iLastWorker; ///< just a helper to optimize consequental linear search
8424 mutable int m_iNumDocs; ///< optimize numdocs/length calculation in scattered case
8425 mutable int m_iReqLen;
8426 mutable bool m_bScattered;
8427 };
8428
8429
8430 struct SnippetReplyParser_t : public IReplyParser_t
8431 {
SnippetReplyParser_tSnippetReplyParser_t8432 explicit SnippetReplyParser_t ( SnippetsRemote_t * pWorker )
8433 : m_pWorker ( pWorker )
8434 , m_iLastAgent ( -1 )
8435 , m_iLastWorker ( -1 )
8436 {}
8437
8438 virtual bool ParseReply ( MemInputBuffer_c & tReq, AgentConn_t &, int iNumAgent ) const;
8439
8440 private:
8441 SnippetsRemote_t * m_pWorker;
8442 mutable int m_iLastAgent; ///< just a helper to optimize consequental linear search
8443 mutable int m_iLastWorker; ///< just a helper to optimize consequental linear search
8444 };
8445
8446
SnippetGetCurrentWorker(const int iNumAgent,const int m_iLastAgent,const int m_iLastWorker,const SnippetsRemote_t * m_pWorker)8447 static int SnippetGetCurrentWorker ( const int iNumAgent, const int m_iLastAgent, const int m_iLastWorker, const SnippetsRemote_t * m_pWorker )
8448 {
8449 int iCurrentWorker = 0;
8450 if ( iNumAgent==m_iLastAgent+1 )
8451 {
8452 for ( int i=m_iLastWorker+1; i<m_pWorker->m_dWorkers.GetLength(); i++ )
8453 if ( !m_pWorker->m_dWorkers[i].m_bLocal )
8454 {
8455 iCurrentWorker = i;
8456 break;
8457 }
8458 } else
8459 {
8460 int j = iNumAgent;
8461 ARRAY_FOREACH ( i, m_pWorker->m_dWorkers )
8462 if ( !m_pWorker->m_dWorkers[i].m_bLocal )
8463 {
8464 if ( j-- )
8465 continue;
8466 iCurrentWorker = i;
8467 break;
8468 }
8469 }
8470 return iCurrentWorker;
8471 }
8472
8473
BuildRequest(const char * sIndex,NetOutputBuffer_c & tOut,int iNumAgent) const8474 void SnippetRequestBuilder_t::BuildRequest ( const char * sIndex, NetOutputBuffer_c & tOut, int iNumAgent ) const
8475 {
8476 m_iLastWorker = SnippetGetCurrentWorker ( iNumAgent, m_iLastAgent, m_iLastWorker, m_pWorker );
8477 m_iLastAgent = iNumAgent;
8478
8479 const CSphVector<ExcerptQuery_t> & dQueries = m_pWorker->m_dQueries;
8480 const ExcerptQuery_t & q = dQueries[0];
8481 const SnippetWorker_t & tWorker = m_pWorker->m_dWorkers[m_iLastWorker];
8482
8483 if ( m_iNumDocs < 0 )
8484 m_bScattered = ( q.m_iLoadFiles & 2 )!=0;
8485
8486 if ( !m_bScattered || ( m_bScattered && m_iNumDocs<0 ) )
8487 {
8488 m_iReqLen = 60 // 15 ints/dwords - params, strlens, etc.
8489 + strlen ( sIndex )
8490 + q.m_sWords.Length()
8491 + q.m_sBeforeMatch.Length()
8492 + q.m_sAfterMatch.Length()
8493 + q.m_sChunkSeparator.Length()
8494 + q.m_sStripMode.Length()
8495 + q.m_sRawPassageBoundary.Length();
8496
8497 m_iNumDocs = 0;
8498 for ( int iDoc = tWorker.m_iHead; iDoc!=EOF_ITEM; iDoc=dQueries[iDoc].m_iNext )
8499 {
8500 ++m_iNumDocs;
8501 m_iReqLen += 4 + dQueries[iDoc].m_sSource.Length();
8502 }
8503 }
8504
8505
8506 tOut.SendDword ( SPHINX_SEARCHD_PROTO );
8507 tOut.SendWord ( SEARCHD_COMMAND_EXCERPT );
8508 tOut.SendWord ( VER_COMMAND_EXCERPT );
8509
8510 tOut.SendInt ( m_iReqLen );
8511
8512 tOut.SendInt ( 0 );
8513
8514 if ( m_bScattered )
8515 tOut.SendInt ( q.m_iRawFlags & ~EXCERPT_FLAG_LOAD_FILES );
8516 else
8517 tOut.SendInt ( q.m_iRawFlags );
8518
8519 tOut.SendString ( sIndex );
8520 tOut.SendString ( q.m_sWords.cstr() );
8521 tOut.SendString ( q.m_sBeforeMatch.cstr() );
8522 tOut.SendString ( q.m_sAfterMatch.cstr() );
8523 tOut.SendString ( q.m_sChunkSeparator.cstr() );
8524 tOut.SendInt ( q.m_iLimit );
8525 tOut.SendInt ( q.m_iAround );
8526
8527 tOut.SendInt ( q.m_iLimitPassages );
8528 tOut.SendInt ( q.m_iLimitWords );
8529 tOut.SendInt ( q.m_iPassageId );
8530 tOut.SendString ( q.m_sStripMode.cstr() );
8531 tOut.SendString ( q.m_sRawPassageBoundary.cstr() );
8532
8533 tOut.SendInt ( m_iNumDocs );
8534 for ( int iDoc = tWorker.m_iHead; iDoc!=EOF_ITEM; iDoc=dQueries[iDoc].m_iNext )
8535 tOut.SendString ( dQueries[iDoc].m_sSource.cstr() );
8536 }
8537
ParseReply(MemInputBuffer_c & tReq,AgentConn_t &,int iNumAgent) const8538 bool SnippetReplyParser_t::ParseReply ( MemInputBuffer_c & tReq, AgentConn_t &, int iNumAgent ) const
8539 {
8540 m_iLastWorker = SnippetGetCurrentWorker ( iNumAgent, m_iLastAgent, m_iLastWorker, m_pWorker );
8541 m_iLastAgent = iNumAgent;
8542
8543 CSphVector<ExcerptQuery_t> & dQueries = m_pWorker->m_dQueries;
8544 const SnippetWorker_t & tWorker = m_pWorker->m_dWorkers[m_iLastWorker];
8545
8546 int iDoc = tWorker.m_iHead;
8547 bool bOk = true;
8548 while ( iDoc!=EOF_ITEM )
8549 {
8550 if ( ( dQueries[iDoc].m_iLoadFiles&2 )!=0 ) // NOLINT
8551 {
8552 char * sRes = tReq.GetString().Leak();
8553 if ( sRes && !strlen(sRes) )
8554 SafeDelete ( sRes );
8555 if ( sRes )
8556 {
8557 if ( dQueries[iDoc].m_sRes && strlen ( dQueries[iDoc].m_sRes )!=0 )
8558 {
8559 if ( strcmp ( sRes, dQueries[iDoc].m_sRes )!=0 )
8560 bOk = false;
8561 SafeDelete ( dQueries[iDoc].m_sRes );
8562 } else
8563 dQueries[iDoc].m_sError = "";
8564 dQueries[iDoc].m_sRes = sRes;
8565 }
8566
8567 iDoc = dQueries[iDoc].m_iNext;
8568 continue;
8569 }
8570 dQueries[iDoc].m_sRes = tReq.GetString().Leak();
8571 int iNextDoc = dQueries[iDoc].m_iNext;
8572 dQueries[iDoc].m_iNext = PROCESSED_ITEM;
8573 iDoc = iNextDoc;
8574 }
8575
8576 return bOk;
8577 }
8578
8579
SnippetTransformPassageMacros(CSphString & sSrc,CSphString & sPost)8580 static bool SnippetTransformPassageMacros ( CSphString & sSrc, CSphString & sPost )
8581 {
8582 const char sPassageMacro[] = "%PASSAGE_ID%";
8583
8584 const char * sPass = NULL;
8585 if ( !sSrc.IsEmpty() )
8586 sPass = strstr ( sSrc.cstr(), sPassageMacro );
8587
8588 if ( !sPass )
8589 return false;
8590
8591 int iSrcLen = sSrc.Length();
8592 int iPassLen = sizeof ( sPassageMacro ) - 1;
8593 int iTailLen = iSrcLen - iPassLen - ( sPass - sSrc.cstr() );
8594
8595 // copy tail
8596 if ( iTailLen )
8597 sPost.SetBinary ( sPass+iPassLen, iTailLen );
8598
8599 CSphString sPre;
8600 sPre.SetBinary ( sSrc.cstr(), sPass - sSrc.cstr() );
8601 sSrc.Swap ( sPre );
8602
8603 return true;
8604 }
8605
8606
SetupStripperSPZ(const CSphIndexSettings & tSettings,const ExcerptQuery_t & q,bool bSetupSPZ,CSphScopedPtr<CSphHTMLStripper> & tStripper,ISphTokenizer * pTokenizer,CSphString & sError)8607 static bool SetupStripperSPZ ( const CSphIndexSettings & tSettings, const ExcerptQuery_t & q, bool bSetupSPZ, CSphScopedPtr<CSphHTMLStripper> & tStripper, ISphTokenizer * pTokenizer, CSphString & sError )
8608 {
8609 if ( bSetupSPZ &&
8610 ( !pTokenizer->EnableSentenceIndexing ( sError ) || !pTokenizer->EnableZoneIndexing ( sError ) ) )
8611 {
8612 return false;
8613 }
8614
8615
8616 if ( q.m_sStripMode=="strip" || q.m_sStripMode=="retain"
8617 || ( q.m_sStripMode=="index" && tSettings.m_bHtmlStrip ) )
8618 {
8619 // don't strip HTML markup in 'retain' mode - proceed zones only
8620 tStripper = new CSphHTMLStripper ( q.m_sStripMode!="retain" );
8621
8622 if ( q.m_sStripMode=="index" )
8623 {
8624 if (
8625 !tStripper->SetIndexedAttrs ( tSettings.m_sHtmlIndexAttrs.cstr (), sError ) ||
8626 !tStripper->SetRemovedElements ( tSettings.m_sHtmlRemoveElements.cstr (), sError ) )
8627 {
8628 sError.SetSprintf ( "HTML stripper config error: %s", sError.cstr() );
8629 return false;
8630 }
8631 }
8632
8633 if ( bSetupSPZ )
8634 {
8635 tStripper->EnableParagraphs();
8636 }
8637
8638 // handle zone(s) in special mode only when passage_boundary enabled
8639 if ( bSetupSPZ && !tStripper->SetZones ( tSettings.m_sZones.cstr (), sError ) )
8640 {
8641 sError.SetSprintf ( "HTML stripper config error: %s", sError.cstr() );
8642 return false;
8643 }
8644 }
8645
8646 return true;
8647 }
8648
8649
SetupExactDict(const CSphIndexSettings & tSettings,const ExcerptQuery_t & q,CSphScopedPtr<CSphDict> & tExact,CSphDict * pDict,ISphTokenizer * pTokenizer)8650 static CSphDict * SetupExactDict ( const CSphIndexSettings & tSettings, const ExcerptQuery_t & q, CSphScopedPtr<CSphDict> & tExact, CSphDict * pDict, ISphTokenizer * pTokenizer )
8651 {
8652 // handle index_exact_words
8653 if ( !( q.m_bHighlightQuery && tSettings.m_bIndexExactWords ) )
8654 return pDict;
8655
8656 CSphRemapRange tEq ( '=', '=', '=' ); // FIXME? check and warn if star was already there
8657 pTokenizer->AddCaseFolding ( tEq );
8658
8659 tExact = new CSphDictExact ( pDict );
8660 return tExact.Ptr();
8661 }
8662
8663
CollectQuerySPZ(const XQNode_t * pNode)8664 static DWORD CollectQuerySPZ ( const XQNode_t * pNode )
8665 {
8666 if ( !pNode )
8667 return SPH_SPZ_NONE;
8668
8669 DWORD eSPZ = SPH_SPZ_NONE;
8670 if ( pNode->GetOp()==SPH_QUERY_SENTENCE )
8671 eSPZ |= SPH_SPZ_SENTENCE;
8672 else if ( pNode->GetOp()==SPH_QUERY_PARAGRAPH )
8673 eSPZ |= SPH_SPZ_PARAGRAPH;
8674
8675 ARRAY_FOREACH ( i, pNode->m_dChildren )
8676 eSPZ |= CollectQuerySPZ ( pNode->m_dChildren[i] );
8677
8678 return eSPZ;
8679 }
8680
8681
8682 class SnippetContext_t : ISphNoncopyable
8683 {
8684 private:
8685 CSphScopedPtr<CSphDict> m_tDictCloned;
8686 CSphScopedPtr<CSphDict> m_tExactDict;
8687 CSphScopedPtr<ISphTokenizer> m_tQueryTokenizer;
8688
8689 public:
8690 CSphDict * m_pDict;
8691 CSphScopedPtr<ISphTokenizer> m_tTokenizer;
8692 CSphScopedPtr<CSphHTMLStripper> m_tStripper;
8693 ISphTokenizer * m_pQueryTokenizer;
8694 XQQuery_t m_tExtQuery;
8695 DWORD m_eExtQuerySPZ;
8696
SnippetContext_t()8697 SnippetContext_t()
8698 : m_tDictCloned ( NULL )
8699 , m_tExactDict ( NULL )
8700 , m_tQueryTokenizer ( NULL )
8701 , m_pDict ( NULL )
8702 , m_tTokenizer ( NULL )
8703 , m_tStripper ( NULL )
8704 , m_pQueryTokenizer ( NULL )
8705 , m_eExtQuerySPZ ( SPH_SPZ_NONE )
8706 {
8707 }
8708
Setup(const CSphIndex * pIndex,const ExcerptQuery_t & tSettings,CSphString & sError)8709 bool Setup ( const CSphIndex * pIndex, const ExcerptQuery_t & tSettings, CSphString & sError )
8710 {
8711 CSphScopedPtr<CSphDict> tDictCloned ( NULL );
8712 m_pDict = pIndex->GetDictionary();
8713 if ( m_pDict->HasState() )
8714 {
8715 m_tDictCloned = m_pDict = m_pDict->Clone();
8716 }
8717
8718 m_tTokenizer = pIndex->GetTokenizer()->Clone ( true );
8719 m_pQueryTokenizer = m_tTokenizer.Ptr();
8720
8721 // setup exact dictionary if needed
8722 m_pDict = SetupExactDict ( pIndex->GetSettings(), tSettings, m_tExactDict, m_pDict, m_tTokenizer.Ptr() );
8723 // TODO!!! check star dict too
8724
8725 if ( tSettings.m_bHighlightQuery )
8726 {
8727 if ( !sphParseExtendedQuery ( m_tExtQuery, tSettings.m_sWords.cstr(), m_pQueryTokenizer, &pIndex->GetMatchSchema(), m_pDict, pIndex->GetSettings().m_iStopwordStep ) )
8728 {
8729 sError = m_tExtQuery.m_sParseError;
8730 return false;
8731 }
8732 if ( m_tExtQuery.m_pRoot )
8733 m_tExtQuery.m_pRoot->ClearFieldMask();
8734
8735 m_eExtQuerySPZ = SPH_SPZ_NONE;
8736 m_eExtQuerySPZ |= CollectQuerySPZ ( m_tExtQuery.m_pRoot );
8737 if ( m_tExtQuery.m_dZones.GetLength() )
8738 m_eExtQuerySPZ |= SPH_SPZ_ZONE;
8739 }
8740
8741 bool bSetupSPZ = ( tSettings.m_ePassageSPZ!=SPH_SPZ_NONE || m_eExtQuerySPZ!=SPH_SPZ_NONE ||
8742 ( tSettings.m_sStripMode=="retain" && tSettings.m_bHighlightQuery ) );
8743
8744 if ( !SetupStripperSPZ ( pIndex->GetSettings(), tSettings, bSetupSPZ, m_tStripper, m_tTokenizer.Ptr(), sError ) )
8745 return false;
8746
8747 if ( bSetupSPZ )
8748 {
8749 m_tQueryTokenizer = pIndex->GetTokenizer()->Clone ( true );
8750 m_pQueryTokenizer = m_tQueryTokenizer.Ptr();
8751 }
8752
8753 return true;
8754 }
8755 };
8756
8757
SnippetThreadFunc(void * pArg)8758 void SnippetThreadFunc ( void * pArg )
8759 {
8760 SnippetThread_t * pDesc = (SnippetThread_t*) pArg;
8761
8762 // setup query guard for thread
8763 SphCrashLogger_c tQueryTLS;
8764 tQueryTLS.SetupTLS ();
8765 SphCrashLogger_c::SetLastQuery ( pDesc->m_tCrashQuery );
8766
8767 SnippetContext_t tCtx;
8768 tCtx.Setup ( pDesc->m_pIndex, *pDesc->m_pQueries, pDesc->m_pQueries->m_sError );
8769
8770 for ( ;; )
8771 {
8772 pDesc->m_pLock->Lock();
8773 if ( *pDesc->m_pCurQuery==pDesc->m_iQueries )
8774 {
8775 pDesc->m_pLock->Unlock();
8776 return;
8777 }
8778
8779 ExcerptQuery_t * pQuery = pDesc->m_pQueries + (*pDesc->m_pCurQuery);
8780 (*pDesc->m_pCurQuery)++;
8781 bool bDone = ( *pDesc->m_pCurQuery==pDesc->m_iQueries );
8782 pDesc->m_pLock->Unlock();
8783
8784 if ( pQuery->m_iNext!=PROCESSED_ITEM )
8785 continue;
8786
8787 pQuery->m_sRes = sphBuildExcerpt ( *pQuery, pDesc->m_pIndex, tCtx.m_tStripper.Ptr(), tCtx.m_tExtQuery, tCtx.m_eExtQuerySPZ,
8788 pQuery->m_sError, tCtx.m_pDict, tCtx.m_tTokenizer.Ptr(), tCtx.m_pQueryTokenizer );
8789
8790 if ( bDone )
8791 return;
8792 }
8793 }
8794
GetRawSnippetFlags(const ExcerptQuery_t & q)8795 int GetRawSnippetFlags ( const ExcerptQuery_t& q )
8796 {
8797 int iRawFlags = 0;
8798
8799 iRawFlags |= q.m_bRemoveSpaces ? EXCERPT_FLAG_REMOVESPACES : 0;
8800 iRawFlags |= q.m_bUseBoundaries ? EXCERPT_FLAG_USEBOUNDARIES : 0;
8801 iRawFlags |= q.m_bWeightOrder ? EXCERPT_FLAG_WEIGHTORDER : 0;
8802 iRawFlags |= q.m_bHighlightQuery ? EXCERPT_FLAG_QUERY : 0;
8803 iRawFlags |= q.m_bForceAllWords ? EXCERPT_FLAG_FORCE_ALL_WORDS : 0;
8804 iRawFlags |= q.m_iLimitPassages ? EXCERPT_FLAG_SINGLEPASSAGE : 0;
8805 iRawFlags |= ( q.m_iLoadFiles & 1 ) ? EXCERPT_FLAG_LOAD_FILES : 0;
8806 iRawFlags |= ( q.m_iLoadFiles & 2 ) ? EXCERPT_FLAG_FILES_SCATTERED : 0;
8807 iRawFlags |= q.m_bAllowEmpty ? EXCERPT_FLAG_ALLOW_EMPTY : 0;
8808 iRawFlags |= q.m_bEmitZones ? EXCERPT_FLAG_EMIT_ZONES : 0;
8809
8810 return iRawFlags;
8811 }
8812
MakeSnippets(CSphString sIndex,CSphVector<ExcerptQuery_t> & dQueries,CSphString & sError)8813 bool MakeSnippets ( CSphString sIndex, CSphVector<ExcerptQuery_t> & dQueries, CSphString & sError )
8814 {
8815 SnippetsRemote_t dRemoteSnippets ( dQueries );
8816 CSphVector<CSphString> dDistLocal;
8817 ExcerptQuery_t & q = dQueries[0];
8818
8819 g_tDistLock.Lock();
8820 DistributedIndex_t * pDist = g_hDistIndexes ( sIndex );
8821 bool bRemote = pDist!=NULL;
8822
8823 // hack! load_files && load_files_scattered is the 'final' call. It will report the absent files as errors.
8824 // simple load_files_scattered without load_files just omits the absent files (returns empty strings).
8825 bool bScattered = ( q.m_iLoadFiles & 2 )!=0;
8826 bool bSkipAbsentFiles = !( q.m_iLoadFiles & 1 );
8827
8828 if ( bRemote )
8829 {
8830 dRemoteSnippets.m_iAgentConnectTimeout = pDist->m_iAgentConnectTimeout;
8831 dRemoteSnippets.m_iAgentQueryTimeout = pDist->m_iAgentQueryTimeout;
8832 dDistLocal = pDist->m_dLocal;
8833 dRemoteSnippets.m_dAgents.Resize ( pDist->m_dAgents.GetLength() );
8834 ARRAY_FOREACH ( i, pDist->m_dAgents )
8835 dRemoteSnippets.m_dAgents[i] = pDist->m_dAgents[i];
8836 }
8837 g_tDistLock.Unlock();
8838
8839 if ( pDist )
8840 {
8841 if ( pDist->m_dLocal.GetLength()!=1 )
8842 {
8843 sError.SetSprintf ( "%s", "The distributed index for snippets must have exactly one local agent" );
8844 return false;
8845 }
8846
8847 if ( !q.m_iLoadFiles )
8848 {
8849 sError.SetSprintf ( "%s", "The distributed index for snippets available only when using external files" );
8850 return false;
8851 }
8852 sIndex = dDistLocal[0];
8853
8854 // no remote - roll back to simple local query
8855 if ( dRemoteSnippets.m_dAgents.GetLength()==0 )
8856 bRemote = false;
8857 }
8858
8859 const ServedIndex_t * pServed = g_pIndexes->GetRlockedEntry ( sIndex );
8860
8861 if ( !pServed || !pServed->m_bEnabled || !pServed->m_pIndex )
8862 {
8863 sError.SetSprintf ( "unknown local index '%s' in search request", sIndex.cstr() );
8864 if ( pServed )
8865 pServed->Unlock();
8866 return false;
8867 }
8868
8869 CSphIndex * pIndex = pServed->m_pIndex;
8870
8871 SnippetContext_t tCtx;
8872 if ( !tCtx.Setup ( pIndex, q, sError ) ) // same path for single - threaded snippets, bail out here on error
8873 {
8874 sError.SetSprintf ( "%s", sError.cstr() );
8875 pServed->Unlock();
8876 return false;
8877 }
8878
8879 ///////////////////
8880 // do highlighting
8881 ///////////////////
8882
8883 bool bOk = true;
8884 int iAbsentHead = EOF_ITEM;
8885 if ( g_iDistThreads<=1 || dQueries.GetLength()<2 )
8886 {
8887 // boring single threaded loop
8888 ARRAY_FOREACH ( i, dQueries )
8889 {
8890 dQueries[i].m_sRes = sphBuildExcerpt ( dQueries[i], pIndex, tCtx.m_tStripper.Ptr(), tCtx.m_tExtQuery, tCtx.m_eExtQuerySPZ,
8891 sError, tCtx.m_pDict, tCtx.m_tTokenizer.Ptr(), tCtx.m_pQueryTokenizer );
8892 if ( !dQueries[i].m_sRes )
8893 {
8894 bOk = false;
8895 break;
8896 }
8897 }
8898 } else
8899 {
8900 // get file sizes
8901 ARRAY_FOREACH ( i, dQueries )
8902 {
8903 dQueries[i].m_iNext = PROCESSED_ITEM;
8904 if ( dQueries[i].m_iLoadFiles )
8905 {
8906 struct stat st;
8907 if ( ::stat ( dQueries[i].m_sSource.cstr(), &st )<0 )
8908 {
8909 if ( !bScattered )
8910 {
8911 sError.SetSprintf ( "failed to stat %s: %s", dQueries[i].m_sSource.cstr(), strerror(errno) );
8912 pServed->Unlock();
8913 return false;
8914 }
8915 dQueries[i].m_iNext = EOF_ITEM;
8916 }
8917 dQueries[i].m_iSize = -st.st_size; // so that sort would put bigger ones first
8918 } else
8919 {
8920 dQueries[i].m_iSize = -dQueries[i].m_sSource.Length();
8921 }
8922 dQueries[i].m_iSeq = i;
8923 }
8924
8925 // tough jobs first
8926 if ( !bScattered )
8927 dQueries.Sort ( bind ( &ExcerptQuery_t::m_iSize ) );
8928
8929 ARRAY_FOREACH ( i, dQueries )
8930 if ( dQueries[i].m_iNext==EOF_ITEM )
8931 {
8932 dQueries[i].m_iNext = iAbsentHead;
8933 iAbsentHead = i;
8934 if ( !bSkipAbsentFiles )
8935 dQueries[i].m_sError.SetSprintf ( "failed to stat %s: %s", dQueries[i].m_sSource.cstr(), strerror(errno) );
8936 }
8937
8938
8939
8940 // check if all files are available locally.
8941 if ( bScattered && iAbsentHead==EOF_ITEM )
8942 {
8943 bRemote = false;
8944 dRemoteSnippets.m_dAgents.Reset();
8945 }
8946
8947 if ( bRemote )
8948 {
8949 // schedule jobs across workers (the worker is either local thread or instance, either remote agent).
8950 // simple LPT (Least Processing Time) scheduling for now
8951 // might add dynamic programming or something later if needed
8952 int iLocalPart = 1; // one instance = one worker. Or set to = g_iDistThreads, one local thread = one worker.
8953 int iRemoteAgents = dRemoteSnippets.m_dAgents.GetLength();
8954
8955 dRemoteSnippets.m_dWorkers.Resize ( iLocalPart + iRemoteAgents );
8956 for ( int i=0; i<iLocalPart; i++ )
8957 dRemoteSnippets.m_dWorkers[i].m_bLocal = true;
8958
8959 if ( bScattered )
8960 {
8961 // on scattered case - the queries with m_iNext==PROCESSED_ITEM are here, and has to be scheduled to local agent
8962 // the rest has to be sent to remotes, all of them!
8963 for ( int i=0; i<iRemoteAgents; i++ )
8964 dRemoteSnippets.m_dWorkers[iLocalPart+i].m_iHead = iAbsentHead;
8965 } else
8966 {
8967 ARRAY_FOREACH ( i, dQueries )
8968 {
8969 dRemoteSnippets.m_dWorkers[0].m_iTotal -= dQueries[i].m_iSize;
8970 if ( !dRemoteSnippets.m_dWorkers[0].m_bLocal )
8971 {
8972 // queries sheduled for local still have iNext==PROCESSED_ITEM
8973 dQueries[i].m_iNext = dRemoteSnippets.m_dWorkers[0].m_iHead;
8974 dRemoteSnippets.m_dWorkers[0].m_iHead = i;
8975 }
8976 dRemoteSnippets.m_dWorkers.Sort ( bind ( &SnippetWorker_t::m_iTotal ) );
8977 }
8978 }
8979 }
8980
8981 // do MT searching
8982 CSphMutex tLock;
8983 tLock.Init();
8984
8985 CrashQuery_t tCrashQuery = SphCrashLogger_c::GetQuery(); // transfer query info for crash logger to new thread
8986 int iCurQuery = 0;
8987 CSphVector<SnippetThread_t> dThreads ( g_iDistThreads );
8988 for ( int i=0; i<g_iDistThreads; i++ )
8989 {
8990 SnippetThread_t & t = dThreads[i];
8991 t.m_pLock = &tLock;
8992 t.m_iQueries = dQueries.GetLength();
8993 t.m_pQueries = dQueries.Begin();
8994 t.m_pCurQuery = &iCurQuery;
8995 t.m_pIndex = pIndex;
8996 t.m_tCrashQuery = tCrashQuery;
8997 if ( i )
8998 sphThreadCreate ( &dThreads[i].m_tThd, SnippetThreadFunc, &dThreads[i] );
8999 }
9000
9001 int iRemote = 0;
9002 if ( bRemote )
9003 {
9004 // connect to remote agents and query them
9005 ConnectToRemoteAgents ( dRemoteSnippets.m_dAgents, false );
9006
9007 SnippetRequestBuilder_t tReqBuilder ( &dRemoteSnippets );
9008 iRemote = QueryRemoteAgents ( dRemoteSnippets.m_dAgents, dRemoteSnippets.m_iAgentConnectTimeout, tReqBuilder, NULL ); // FIXME? profile update time too?
9009 }
9010
9011 SnippetThreadFunc ( &dThreads[0] );
9012
9013 int iSuccesses = 0;
9014
9015 if ( iRemote )
9016 {
9017 SnippetReplyParser_t tParser ( &dRemoteSnippets );
9018 iSuccesses = WaitForRemoteAgents ( dRemoteSnippets.m_dAgents, dRemoteSnippets.m_iAgentQueryTimeout, tParser, NULL ); // FIXME? profile update time too?
9019 }
9020
9021 for ( int i=1; i<dThreads.GetLength(); i++ )
9022 sphThreadJoin ( &dThreads[i].m_tThd );
9023
9024 if ( iSuccesses!=dRemoteSnippets.m_dAgents.GetLength() )
9025 {
9026 sphWarning ( "Remote snippets: some of the agents didn't answered: %d queried, %d available, %d answered",
9027 dRemoteSnippets.m_dAgents.GetLength(),
9028 iRemote,
9029 iSuccesses );
9030
9031 if ( !bScattered )
9032 {
9033 // inverse the success/failed state - so that the queries with negative m_iNext are treated as failed
9034 ARRAY_FOREACH ( i, dQueries )
9035 dQueries[i].m_iNext = (dQueries[i].m_iNext==PROCESSED_ITEM)?0:PROCESSED_ITEM;
9036
9037 // failsafe - one more turn for failed queries on local agent
9038 SnippetThread_t & t = dThreads[0];
9039 t.m_pQueries = dQueries.Begin();
9040 iCurQuery = 0;
9041 SnippetThreadFunc ( &dThreads[0] );
9042 }
9043 }
9044 tLock.Done();
9045
9046 // back in query order
9047 dQueries.Sort ( bind ( &ExcerptQuery_t::m_iSeq ) );
9048
9049 ARRAY_FOREACH ( i, dQueries )
9050 {
9051 if ( !dQueries[i].m_sError.IsEmpty() )
9052 {
9053 bOk = false;
9054 if ( sError.IsEmpty() )
9055 sError.SetSprintf ( "%s", dQueries[i].m_sError.cstr() );
9056 else
9057 sError.SetSprintf ( "%s; %s", sError.cstr(), dQueries[i].m_sError.cstr() );
9058 }
9059 }
9060 }
9061
9062 pServed->Unlock();
9063 return bOk;
9064 }
9065
HandleCommandExcerpt(int iSock,int iVer,InputBuffer_c & tReq)9066 void HandleCommandExcerpt ( int iSock, int iVer, InputBuffer_c & tReq )
9067 {
9068 if ( !CheckCommandVersion ( iVer, VER_COMMAND_EXCERPT, tReq ) )
9069 return;
9070
9071 /////////////////////////////
9072 // parse and process request
9073 /////////////////////////////
9074
9075 const int EXCERPT_MAX_ENTRIES = 1024;
9076
9077 // v.1.1
9078 ExcerptQuery_t q;
9079
9080 tReq.GetInt (); // mode field is for now reserved and ignored
9081 int iFlags = tReq.GetInt ();
9082 q.m_iRawFlags = iFlags;
9083 CSphString sIndex = tReq.GetString ();
9084
9085 q.m_sWords = tReq.GetString ();
9086 q.m_sBeforeMatch = tReq.GetString ();
9087 q.m_sAfterMatch = tReq.GetString ();
9088 q.m_sChunkSeparator = tReq.GetString ();
9089 q.m_iLimit = tReq.GetInt ();
9090 q.m_iAround = tReq.GetInt ();
9091
9092 if ( iVer>=0x102 )
9093 {
9094 q.m_iLimitPassages = tReq.GetInt();
9095 q.m_iLimitWords = tReq.GetInt();
9096 q.m_iPassageId = tReq.GetInt();
9097 q.m_sStripMode = tReq.GetString();
9098 if ( q.m_sStripMode!="none" && q.m_sStripMode!="index" && q.m_sStripMode!="strip" && q.m_sStripMode!="retain" )
9099 {
9100 tReq.SendErrorReply ( "unknown html_strip_mode=%s", q.m_sStripMode.cstr() );
9101 return;
9102 }
9103 }
9104
9105 q.m_bHasBeforePassageMacro = SnippetTransformPassageMacros ( q.m_sBeforeMatch, q.m_sBeforeMatchPassage );
9106 q.m_bHasAfterPassageMacro = SnippetTransformPassageMacros ( q.m_sAfterMatch, q.m_sAfterMatchPassage );
9107
9108 CSphString sPassageBoundaryMode;
9109 if ( iVer>=0x103 )
9110 q.m_sRawPassageBoundary = tReq.GetString();
9111
9112 q.m_bRemoveSpaces = ( iFlags & EXCERPT_FLAG_REMOVESPACES )!=0;
9113 q.m_bExactPhrase = ( iFlags & EXCERPT_FLAG_EXACTPHRASE )!=0;
9114 q.m_bUseBoundaries = ( iFlags & EXCERPT_FLAG_USEBOUNDARIES )!=0;
9115 q.m_bWeightOrder = ( iFlags & EXCERPT_FLAG_WEIGHTORDER )!=0;
9116 q.m_bHighlightQuery = ( iFlags & EXCERPT_FLAG_QUERY )!=0;
9117 q.m_bForceAllWords = ( iFlags & EXCERPT_FLAG_FORCE_ALL_WORDS )!=0;
9118 if ( iFlags & EXCERPT_FLAG_SINGLEPASSAGE )
9119 q.m_iLimitPassages = 1;
9120 q.m_iLoadFiles = (( iFlags & EXCERPT_FLAG_LOAD_FILES )!=0)?1:0;
9121 bool bScattered = ( iFlags & EXCERPT_FLAG_FILES_SCATTERED )!=0;
9122 q.m_iLoadFiles |= bScattered?2:0;
9123 q.m_bAllowEmpty = ( iFlags & EXCERPT_FLAG_ALLOW_EMPTY )!=0;
9124 q.m_bEmitZones = ( iFlags & EXCERPT_FLAG_EMIT_ZONES )!=0;
9125
9126 int iCount = tReq.GetInt ();
9127 if ( iCount<=0 || iCount>EXCERPT_MAX_ENTRIES )
9128 {
9129 tReq.SendErrorReply ( "invalid entries count %d", iCount );
9130 return;
9131 }
9132
9133 q.m_ePassageSPZ = sphGetPassageBoundary ( q.m_sRawPassageBoundary );
9134
9135 CSphString sError;
9136
9137 if ( !sphCheckOptionsSPZ ( q, q.m_sRawPassageBoundary, sError ) )
9138 {
9139 tReq.SendErrorReply ( "%s", sError.cstr() );
9140 return;
9141 }
9142
9143 CSphVector<ExcerptQuery_t> dQueries ( iCount );
9144
9145 ARRAY_FOREACH ( i, dQueries )
9146 {
9147 dQueries[i] = q; // copy settings
9148 dQueries[i].m_sSource = tReq.GetString (); // fetch data
9149 if ( tReq.GetError() )
9150 {
9151 tReq.SendErrorReply ( "invalid or truncated request" );
9152 return;
9153 }
9154 }
9155
9156 if ( !MakeSnippets ( sIndex, dQueries, sError ) )
9157 {
9158 tReq.SendErrorReply ( "%s", sError.cstr() );
9159 return;
9160 }
9161
9162 ////////////////
9163 // serve result
9164 ////////////////
9165
9166 int iRespLen = 0;
9167 ARRAY_FOREACH ( i, dQueries )
9168 {
9169 // handle errors
9170 if ( !dQueries[i].m_sRes )
9171 {
9172 if ( !bScattered )
9173 {
9174 tReq.SendErrorReply ( "highlighting failed: %s", dQueries[i].m_sError.cstr() );
9175 ARRAY_FOREACH ( j, dQueries )
9176 SafeDeleteArray ( dQueries[j].m_sRes );
9177 return;
9178 }
9179 iRespLen += 4;
9180 } else
9181 iRespLen += 4 + strlen ( dQueries[i].m_sRes );
9182 }
9183
9184 NetOutputBuffer_c tOut ( iSock );
9185 tOut.SendWord ( SEARCHD_OK );
9186 tOut.SendWord ( VER_COMMAND_EXCERPT );
9187 tOut.SendInt ( iRespLen );
9188 ARRAY_FOREACH ( i, dQueries )
9189 {
9190 if ( dQueries[i].m_sRes && strcmp ( dQueries[i].m_sRes, "" ) )
9191 {
9192 tOut.SendString ( dQueries[i].m_sRes );
9193 SafeDeleteArray ( dQueries[i].m_sRes );
9194 } else
9195 tOut.SendString ( "" );
9196 }
9197
9198 tOut.Flush ();
9199 assert ( tOut.GetError()==true || tOut.GetSentCount()==iRespLen+8 );
9200 }
9201
9202 /////////////////////////////////////////////////////////////////////////////
9203 // KEYWORDS HANDLER
9204 /////////////////////////////////////////////////////////////////////////////
9205
HandleCommandKeywords(int iSock,int iVer,InputBuffer_c & tReq)9206 void HandleCommandKeywords ( int iSock, int iVer, InputBuffer_c & tReq )
9207 {
9208 if ( !CheckCommandVersion ( iVer, VER_COMMAND_KEYWORDS, tReq ) )
9209 return;
9210
9211 CSphString sQuery = tReq.GetString ();
9212 CSphString sIndex = tReq.GetString ();
9213 bool bGetStats = !!tReq.GetInt ();
9214
9215 const ServedIndex_t * pIndex = g_pIndexes->GetRlockedEntry ( sIndex );
9216 if ( !pIndex )
9217 {
9218 tReq.SendErrorReply ( "unknown local index '%s' in search request", sIndex.cstr() );
9219 return;
9220 }
9221
9222 CSphString sError;
9223 CSphVector < CSphKeywordInfo > dKeywords;
9224 if ( !pIndex->m_pIndex->GetKeywords ( dKeywords, sQuery.cstr (), bGetStats, sError ) )
9225 {
9226 tReq.SendErrorReply ( "error generating keywords: %s", sError.cstr () );
9227 pIndex->Unlock();
9228 return;
9229 }
9230
9231 pIndex->Unlock();
9232
9233 int iRespLen = 4;
9234 ARRAY_FOREACH ( i, dKeywords )
9235 {
9236 iRespLen += 4 + strlen ( dKeywords[i].m_sTokenized.cstr () );
9237 iRespLen += 4 + strlen ( dKeywords[i].m_sNormalized.cstr () );
9238 if ( bGetStats )
9239 iRespLen += 8;
9240 }
9241
9242 NetOutputBuffer_c tOut ( iSock );
9243 tOut.SendWord ( SEARCHD_OK );
9244 tOut.SendWord ( VER_COMMAND_KEYWORDS );
9245 tOut.SendInt ( iRespLen );
9246 tOut.SendInt ( dKeywords.GetLength () );
9247 ARRAY_FOREACH ( i, dKeywords )
9248 {
9249 tOut.SendString ( dKeywords[i].m_sTokenized.cstr () );
9250 tOut.SendString ( dKeywords[i].m_sNormalized.cstr () );
9251 if ( bGetStats )
9252 {
9253 tOut.SendInt ( dKeywords[i].m_iDocs );
9254 tOut.SendInt ( dKeywords[i].m_iHits );
9255 }
9256 }
9257
9258 tOut.Flush ();
9259 assert ( tOut.GetError()==true || tOut.GetSentCount()==iRespLen+8 );
9260 }
9261
9262 /////////////////////////////////////////////////////////////////////////////
9263 // UPDATES HANDLER
9264 /////////////////////////////////////////////////////////////////////////////
9265
9266 struct UpdateRequestBuilder_t : public IRequestBuilder_t
9267 {
UpdateRequestBuilder_tUpdateRequestBuilder_t9268 explicit UpdateRequestBuilder_t ( const CSphAttrUpdate & pUpd ) : m_tUpd ( pUpd ) {}
9269 virtual void BuildRequest ( const char * sIndexes, NetOutputBuffer_c & tOut, int ) const;
9270
9271 protected:
9272 const CSphAttrUpdate & m_tUpd;
9273 };
9274
9275
9276 struct UpdateReplyParser_t : public IReplyParser_t
9277 {
UpdateReplyParser_tUpdateReplyParser_t9278 explicit UpdateReplyParser_t ( int * pUpd )
9279 : m_pUpdated ( pUpd )
9280 {}
9281
ParseReplyUpdateReplyParser_t9282 virtual bool ParseReply ( MemInputBuffer_c & tReq, AgentConn_t &, int ) const
9283 {
9284 *m_pUpdated += tReq.GetDword ();
9285 return true;
9286 }
9287
9288 protected:
9289 int * m_pUpdated;
9290 };
9291
9292
BuildRequest(const char * sIndexes,NetOutputBuffer_c & tOut,int) const9293 void UpdateRequestBuilder_t::BuildRequest ( const char * sIndexes, NetOutputBuffer_c & tOut, int ) const
9294 {
9295 int iReqSize = 4+strlen(sIndexes); // indexes string
9296 iReqSize += 4; // attrs array len, data
9297 ARRAY_FOREACH ( i, m_tUpd.m_dAttrs )
9298 iReqSize += 8 + strlen ( m_tUpd.m_dAttrs[i].m_sName.cstr() );
9299 iReqSize += 4; // number of updates
9300 iReqSize += 8*m_tUpd.m_dDocids.GetLength() + 4*m_tUpd.m_dPool.GetLength(); // 64bit ids, 32bit values
9301
9302 // header
9303 tOut.SendDword ( SPHINX_SEARCHD_PROTO );
9304 tOut.SendWord ( SEARCHD_COMMAND_UPDATE );
9305 tOut.SendWord ( VER_COMMAND_UPDATE );
9306 tOut.SendInt ( iReqSize );
9307
9308 tOut.SendString ( sIndexes );
9309 tOut.SendInt ( m_tUpd.m_dAttrs.GetLength() );
9310 ARRAY_FOREACH ( i, m_tUpd.m_dAttrs )
9311 {
9312 tOut.SendString ( m_tUpd.m_dAttrs[i].m_sName.cstr() );
9313 tOut.SendInt ( ( m_tUpd.m_dAttrs[i].m_eAttrType==SPH_ATTR_UINT32SET || m_tUpd.m_dAttrs[i].m_eAttrType==SPH_ATTR_INT64SET ) ? 1 : 0 );
9314 }
9315 tOut.SendInt ( m_tUpd.m_dDocids.GetLength() );
9316
9317 ARRAY_FOREACH ( i, m_tUpd.m_dDocids )
9318 {
9319 int iHead = m_tUpd.m_dRowOffset[i];
9320 int iTail = ( (i+1)<m_tUpd.m_dDocids.GetLength() ) ? m_tUpd.m_dRowOffset[i+1] : m_tUpd.m_dPool.GetLength ();
9321
9322 tOut.SendUint64 ( m_tUpd.m_dDocids[i] );
9323 for ( int j=iHead; j<iTail; j++ )
9324 tOut.SendDword ( m_tUpd.m_dPool[j] );
9325 }
9326 }
9327
DoCommandUpdate(const char * sIndex,const CSphAttrUpdate & tUpd,int & iSuccesses,int & iUpdated,SearchFailuresLog_c & dFails,const ServedIndex_t * pServed)9328 static void DoCommandUpdate ( const char * sIndex, const CSphAttrUpdate & tUpd,
9329 int & iSuccesses, int & iUpdated,
9330 SearchFailuresLog_c & dFails, const ServedIndex_t * pServed )
9331 {
9332 if ( !pServed || !pServed->m_pIndex || !pServed->m_bEnabled )
9333 {
9334 dFails.Submit ( sIndex, "index not available" );
9335 return;
9336 }
9337
9338 CSphString sError;
9339 int iUpd = pServed->m_pIndex->UpdateAttributes ( tUpd, -1, sError );
9340
9341 if ( iUpd<0 )
9342 {
9343 dFails.Submit ( sIndex, sError.cstr() );
9344
9345 } else
9346 {
9347 iUpdated += iUpd;
9348 iSuccesses++;
9349 }
9350 }
9351
UpdateGetLockedIndex(const CSphString & sName,bool bMvaUpdate)9352 static const ServedIndex_t * UpdateGetLockedIndex ( const CSphString & sName, bool bMvaUpdate )
9353 {
9354 const ServedIndex_t * pLocked = g_pIndexes->GetRlockedEntry ( sName );
9355 if ( !pLocked )
9356 return NULL;
9357
9358 // MVA updates have to be done sequentially
9359 if ( !( bMvaUpdate ) )
9360 return pLocked;
9361
9362 pLocked->Unlock();
9363 return g_pIndexes->GetWlockedEntry ( sName );
9364 }
9365
9366
HandleCommandUpdate(int iSock,int iVer,InputBuffer_c & tReq)9367 void HandleCommandUpdate ( int iSock, int iVer, InputBuffer_c & tReq )
9368 {
9369 if ( !CheckCommandVersion ( iVer, VER_COMMAND_UPDATE, tReq ) )
9370 return;
9371
9372 // parse request
9373 CSphString sIndexes = tReq.GetString ();
9374 CSphAttrUpdate tUpd;
9375 CSphVector<DWORD> dMva;
9376
9377 bool bMvaUpdate = false;
9378
9379 tUpd.m_dAttrs.Resize ( tReq.GetDword() ); // FIXME! check this
9380 ARRAY_FOREACH ( i, tUpd.m_dAttrs )
9381 {
9382 tUpd.m_dAttrs[i].m_sName = tReq.GetString ();
9383 tUpd.m_dAttrs[i].m_sName.ToLower ();
9384
9385 tUpd.m_dAttrs[i].m_eAttrType = SPH_ATTR_INTEGER;
9386 if ( iVer>=0x102 )
9387 {
9388 if ( tReq.GetDword() )
9389 {
9390 tUpd.m_dAttrs[i].m_eAttrType = SPH_ATTR_UINT32SET;
9391 bMvaUpdate = true;
9392 }
9393 }
9394 }
9395
9396 int iNumUpdates = tReq.GetInt (); // FIXME! check this
9397 tUpd.m_dDocids.Reserve ( iNumUpdates );
9398 tUpd.m_dRowOffset.Reserve ( iNumUpdates );
9399
9400 for ( int i=0; i<iNumUpdates; i++ )
9401 {
9402 // v.1.0 always sends 32-bit ids; v.1.1+ always send 64-bit ones
9403 uint64_t uDocid = ( iVer>=0x101 ) ? tReq.GetUint64 () : tReq.GetDword ();
9404
9405 tUpd.m_dDocids.Add ( (SphDocID_t)uDocid ); // FIXME! check this
9406 tUpd.m_dRowOffset.Add ( tUpd.m_dPool.GetLength() );
9407
9408 ARRAY_FOREACH ( iAttr, tUpd.m_dAttrs )
9409 {
9410 if ( tUpd.m_dAttrs[iAttr].m_eAttrType==SPH_ATTR_UINT32SET )
9411 {
9412 DWORD uCount = tReq.GetDword ();
9413 if ( !uCount )
9414 {
9415 tUpd.m_dPool.Add ( 0 );
9416 continue;
9417 }
9418
9419 dMva.Resize ( uCount );
9420 for ( DWORD j=0; j<uCount; j++ )
9421 {
9422 dMva[j] = tReq.GetDword();
9423 }
9424 dMva.Uniq(); // don't need dupes within MVA
9425
9426 tUpd.m_dPool.Add ( dMva.GetLength()*2 );
9427 ARRAY_FOREACH ( j, dMva )
9428 {
9429 tUpd.m_dPool.Add ( dMva[j] );
9430 tUpd.m_dPool.Add ( 0 ); // dummy expander mva32 -> mva64
9431 }
9432 } else
9433 {
9434 tUpd.m_dPool.Add ( tReq.GetDword() );
9435 }
9436 }
9437 }
9438
9439 if ( tReq.GetError() )
9440 {
9441 tReq.SendErrorReply ( "invalid or truncated request" );
9442 return;
9443 }
9444
9445 // check index names
9446 CSphVector<CSphString> dIndexNames;
9447 ParseIndexList ( sIndexes, dIndexNames );
9448
9449 if ( !dIndexNames.GetLength() )
9450 {
9451 tReq.SendErrorReply ( "no valid indexes in update request" );
9452 return;
9453 }
9454
9455 CSphVector<DistributedIndex_t> dDistributed ( dIndexNames.GetLength() ); // lock safe storage for distributed indexes
9456 ARRAY_FOREACH ( i, dIndexNames )
9457 {
9458 if ( !g_pIndexes->Exists ( dIndexNames[i] ) )
9459 {
9460 // search amongst distributed and copy for further processing
9461 g_tDistLock.Lock();
9462 const DistributedIndex_t * pDistIndex = g_hDistIndexes ( dIndexNames[i] );
9463
9464 if ( pDistIndex )
9465 {
9466 dDistributed[i] = *pDistIndex;
9467 }
9468
9469 g_tDistLock.Unlock();
9470
9471 if ( pDistIndex )
9472 continue;
9473 else
9474 {
9475 tReq.SendErrorReply ( "unknown index '%s' in update request", dIndexNames[i].cstr() );
9476 return;
9477 }
9478 }
9479 }
9480
9481 // do update
9482 SearchFailuresLog_c dFails;
9483 int iSuccesses = 0;
9484 int iUpdated = 0;
9485
9486 ARRAY_FOREACH ( iIdx, dIndexNames )
9487 {
9488 const char * sReqIndex = dIndexNames[iIdx].cstr();
9489 const ServedIndex_t * pLocked = UpdateGetLockedIndex ( sReqIndex, bMvaUpdate );
9490 if ( pLocked )
9491 {
9492 DoCommandUpdate ( sReqIndex, tUpd, iSuccesses, iUpdated, dFails, pLocked );
9493 pLocked->Unlock();
9494 } else
9495 {
9496 assert ( dDistributed[iIdx].m_dLocal.GetLength() || dDistributed[iIdx].m_dAgents.GetLength() );
9497 CSphVector<CSphString>& dLocal = dDistributed[iIdx].m_dLocal;
9498
9499 ARRAY_FOREACH ( i, dLocal )
9500 {
9501 const char * sLocal = dLocal[i].cstr();
9502 const ServedIndex_t * pServed = UpdateGetLockedIndex ( sLocal, bMvaUpdate );
9503 DoCommandUpdate ( sLocal, tUpd, iSuccesses, iUpdated, dFails, pServed );
9504 if ( pServed )
9505 pServed->Unlock();
9506 }
9507 }
9508
9509 // update remote agents
9510 if ( dDistributed[iIdx].m_dAgents.GetLength() )
9511 {
9512 DistributedIndex_t & tDist = dDistributed[iIdx];
9513
9514 CSphVector<AgentConn_t> dAgents ( tDist.m_dAgents.GetLength() );
9515 ARRAY_FOREACH ( i, dAgents )
9516 dAgents[i] = tDist.m_dAgents[i];
9517
9518 // connect to remote agents and query them
9519 ConnectToRemoteAgents ( dAgents, false );
9520
9521 UpdateRequestBuilder_t tReqBuilder ( tUpd );
9522 int iRemote = QueryRemoteAgents ( dAgents, tDist.m_iAgentConnectTimeout, tReqBuilder, NULL ); // FIXME? profile update time too?
9523
9524 if ( iRemote )
9525 {
9526 UpdateReplyParser_t tParser ( &iUpdated );
9527 iSuccesses += WaitForRemoteAgents ( dAgents, tDist.m_iAgentQueryTimeout, tParser, NULL ); // FIXME? profile update time too?
9528 }
9529 }
9530 }
9531
9532 // serve reply to client
9533 CSphStringBuilder sReport;
9534 dFails.BuildReport ( sReport );
9535
9536 if ( !iSuccesses )
9537 {
9538 tReq.SendErrorReply ( "%s", sReport.cstr() );
9539 return;
9540 }
9541
9542 NetOutputBuffer_c tOut ( iSock );
9543 if ( dFails.IsEmpty() )
9544 {
9545 tOut.SendWord ( SEARCHD_OK );
9546 tOut.SendWord ( VER_COMMAND_UPDATE );
9547 tOut.SendInt ( 4 );
9548 } else
9549 {
9550 tOut.SendWord ( SEARCHD_WARNING );
9551 tOut.SendWord ( VER_COMMAND_UPDATE );
9552 tOut.SendInt ( 8 + strlen ( sReport.cstr() ) );
9553 tOut.SendString ( sReport.cstr() );
9554 }
9555 tOut.SendInt ( iUpdated );
9556 tOut.Flush ();
9557 }
9558
9559 //////////////////////////////////////////////////////////////////////////
9560 // STATUS HANDLER
9561 //////////////////////////////////////////////////////////////////////////
9562
FormatMsec(CSphString & sOut,int64_t tmTime)9563 static inline void FormatMsec ( CSphString & sOut, int64_t tmTime )
9564 {
9565 sOut.SetSprintf ( "%d.%03d", (int)( tmTime/1000000 ), (int)( (tmTime%1000000)/1000 ) );
9566 }
9567
9568
BuildStatus(CSphVector<CSphString> & dStatus)9569 void BuildStatus ( CSphVector<CSphString> & dStatus )
9570 {
9571 assert ( g_pStats );
9572 const char * FMT64 = INT64_FMT;
9573 const char * OFF = "OFF";
9574
9575 const int64_t iQueriesDiv = Max ( g_pStats->m_iQueries, 1 );
9576 const int64_t iDistQueriesDiv = Max ( g_pStats->m_iDistQueries, 1 );
9577
9578 // FIXME? non-transactional!!!
9579 dStatus.Add ( "uptime" ); dStatus.Add().SetSprintf ( "%u", (DWORD)time(NULL)-g_pStats->m_uStarted );
9580 dStatus.Add ( "connections" ); dStatus.Add().SetSprintf ( FMT64, g_pStats->m_iConnections );
9581 dStatus.Add ( "maxed_out" ); dStatus.Add().SetSprintf ( FMT64, g_pStats->m_iMaxedOut );
9582 dStatus.Add ( "command_search" ); dStatus.Add().SetSprintf ( FMT64, g_pStats->m_iCommandCount[SEARCHD_COMMAND_SEARCH] );
9583 dStatus.Add ( "command_excerpt" ); dStatus.Add().SetSprintf ( FMT64, g_pStats->m_iCommandCount[SEARCHD_COMMAND_EXCERPT] );
9584 dStatus.Add ( "command_update" ); dStatus.Add().SetSprintf ( FMT64, g_pStats->m_iCommandCount[SEARCHD_COMMAND_UPDATE] );
9585 dStatus.Add ( "command_keywords" ); dStatus.Add().SetSprintf ( FMT64, g_pStats->m_iCommandCount[SEARCHD_COMMAND_KEYWORDS] );
9586 dStatus.Add ( "command_persist" ); dStatus.Add().SetSprintf ( FMT64, g_pStats->m_iCommandCount[SEARCHD_COMMAND_PERSIST] );
9587 dStatus.Add ( "command_status" ); dStatus.Add().SetSprintf ( FMT64, g_pStats->m_iCommandCount[SEARCHD_COMMAND_STATUS] );
9588 dStatus.Add ( "command_flushattrs" ); dStatus.Add().SetSprintf ( FMT64, g_pStats->m_iCommandCount[SEARCHD_COMMAND_FLUSHATTRS] );
9589 dStatus.Add ( "agent_connect" ); dStatus.Add().SetSprintf ( FMT64, g_pStats->m_iAgentConnect );
9590 dStatus.Add ( "agent_retry" ); dStatus.Add().SetSprintf ( FMT64, g_pStats->m_iAgentRetry );
9591 dStatus.Add ( "queries" ); dStatus.Add().SetSprintf ( FMT64, g_pStats->m_iQueries );
9592 dStatus.Add ( "dist_queries" ); dStatus.Add().SetSprintf ( FMT64, g_pStats->m_iDistQueries );
9593
9594 g_tDistLock.Lock();
9595 g_hDistIndexes.IterateStart();
9596 while ( g_hDistIndexes.IterateNext() )
9597 {
9598 const char * sIdx = g_hDistIndexes.IterateGetKey().cstr();
9599 CSphVector<AgentDesc_t> & dAgents = g_hDistIndexes.IterateGet().m_dAgents;
9600 ARRAY_FOREACH ( i, dAgents )
9601 {
9602 int iIndex = dAgents[i].m_iStatsIndex;
9603 if ( iIndex<0 || iIndex>=STATS_MAX_AGENTS )
9604 continue;
9605
9606 AgentStats_t & tStats = g_pStats->m_dAgentStats[iIndex];
9607 dStatus.Add().SetSprintf ( "ag_%s_%d_query_timeouts", sIdx, i ); dStatus.Add().SetSprintf ( FMT64, tStats.m_iTimeoutsQuery );
9608 dStatus.Add().SetSprintf ( "ag_%s_%d_connect_timeouts", sIdx, i ); dStatus.Add().SetSprintf ( FMT64, tStats.m_iTimeoutsConnect );
9609 dStatus.Add().SetSprintf ( "ag_%s_%d_connect_failures", sIdx, i ); dStatus.Add().SetSprintf ( FMT64, tStats.m_iConnectFailures );
9610 dStatus.Add().SetSprintf ( "ag_%s_%d_network_errors", sIdx, i ); dStatus.Add().SetSprintf ( FMT64, tStats.m_iNetworkErrors );
9611 dStatus.Add().SetSprintf ( "ag_%s_%d_wrong_replies", sIdx, i ); dStatus.Add().SetSprintf ( FMT64, tStats.m_iWrongReplies );
9612 dStatus.Add().SetSprintf ( "ag_%s_%d_unexpected_closings", sIdx, i ); dStatus.Add().SetSprintf ( FMT64, tStats.m_iUnexpectedClose );
9613 }
9614 }
9615 g_tDistLock.Unlock();
9616
9617 dStatus.Add ( "query_wall" ); FormatMsec ( dStatus.Add(), g_pStats->m_iQueryTime );
9618
9619 dStatus.Add ( "query_cpu" );
9620 if ( g_bCpuStats )
9621 FormatMsec ( dStatus.Add(), g_pStats->m_iQueryCpuTime );
9622 else
9623 dStatus.Add() = OFF;
9624
9625 dStatus.Add ( "dist_wall" ); FormatMsec ( dStatus.Add(), g_pStats->m_iDistWallTime );
9626 dStatus.Add ( "dist_local" ); FormatMsec ( dStatus.Add(), g_pStats->m_iDistLocalTime );
9627 dStatus.Add ( "dist_wait" ); FormatMsec ( dStatus.Add(), g_pStats->m_iDistWaitTime );
9628
9629 if ( g_bIOStats )
9630 {
9631 dStatus.Add ( "query_reads" ); dStatus.Add().SetSprintf ( FMT64, g_pStats->m_iDiskReads );
9632 dStatus.Add ( "query_readkb" ); dStatus.Add().SetSprintf ( FMT64, g_pStats->m_iDiskReadBytes/1024 );
9633 dStatus.Add ( "query_readtime" ); FormatMsec ( dStatus.Add(), g_pStats->m_iDiskReadTime );
9634 } else
9635 {
9636 dStatus.Add ( "query_reads" ); dStatus.Add() = OFF;
9637 dStatus.Add ( "query_readkb" ); dStatus.Add() = OFF;
9638 dStatus.Add ( "query_readtime" ); dStatus.Add() = OFF;
9639 }
9640
9641 dStatus.Add ( "avg_query_wall" ); FormatMsec ( dStatus.Add(), g_pStats->m_iQueryTime / iQueriesDiv );
9642 dStatus.Add ( "avg_query_cpu" );
9643 if ( g_bCpuStats )
9644 FormatMsec ( dStatus.Add(), g_pStats->m_iQueryCpuTime / iQueriesDiv );
9645 else
9646 dStatus.Add ( OFF );
9647 dStatus.Add ( "avg_dist_wall" ); FormatMsec ( dStatus.Add(), g_pStats->m_iDistWallTime / iDistQueriesDiv );
9648 dStatus.Add ( "avg_dist_local" ); FormatMsec ( dStatus.Add(), g_pStats->m_iDistLocalTime / iDistQueriesDiv );
9649 dStatus.Add ( "avg_dist_wait" ); FormatMsec ( dStatus.Add(), g_pStats->m_iDistWaitTime / iDistQueriesDiv );
9650 if ( g_bIOStats )
9651 {
9652 dStatus.Add ( "avg_query_reads" ); dStatus.Add().SetSprintf ( "%.1f", (float)( g_pStats->m_iDiskReads*10/iQueriesDiv )/10.0f );
9653 dStatus.Add ( "avg_query_readkb" ); dStatus.Add().SetSprintf ( "%.1f", (float)( g_pStats->m_iDiskReadBytes/iQueriesDiv )/1024.0f );
9654 dStatus.Add ( "avg_query_readtime" ); FormatMsec ( dStatus.Add(), g_pStats->m_iDiskReadTime/iQueriesDiv );
9655 } else
9656 {
9657 dStatus.Add ( "avg_query_reads" ); dStatus.Add() = OFF;
9658 dStatus.Add ( "avg_query_readkb" ); dStatus.Add() = OFF;
9659 dStatus.Add ( "avg_query_readtime" ); dStatus.Add() = OFF;
9660 }
9661 }
9662
9663
BuildMeta(CSphVector<CSphString> & dStatus,const CSphQueryResultMeta & tMeta)9664 void BuildMeta ( CSphVector<CSphString> & dStatus, const CSphQueryResultMeta & tMeta )
9665 {
9666 if ( !tMeta.m_sError.IsEmpty() )
9667 {
9668 dStatus.Add ( "error" );
9669 dStatus.Add ( tMeta.m_sError );
9670 }
9671
9672 if ( !tMeta.m_sWarning.IsEmpty() )
9673 {
9674 dStatus.Add ( "warning" );
9675 dStatus.Add ( tMeta.m_sWarning );
9676 }
9677
9678 dStatus.Add ( "total" );
9679 dStatus.Add().SetSprintf ( "%d", tMeta.m_iMatches );
9680
9681 dStatus.Add ( "total_found" );
9682 dStatus.Add().SetSprintf ( INT64_FMT, tMeta.m_iTotalMatches );
9683
9684 dStatus.Add ( "time" );
9685 dStatus.Add().SetSprintf ( "%d.%03d", tMeta.m_iQueryTime/1000, tMeta.m_iQueryTime%1000 );
9686
9687 int iWord = 0;
9688 tMeta.m_hWordStats.IterateStart();
9689 while ( tMeta.m_hWordStats.IterateNext() )
9690 {
9691 const CSphQueryResultMeta::WordStat_t & tStat = tMeta.m_hWordStats.IterateGet();
9692
9693 dStatus.Add().SetSprintf ( "keyword[%d]", iWord );
9694 dStatus.Add ( tMeta.m_hWordStats.IterateGetKey() );
9695
9696 dStatus.Add().SetSprintf ( "docs[%d]", iWord );
9697 dStatus.Add().SetSprintf ( INT64_FMT, tStat.m_iDocs );
9698
9699 dStatus.Add().SetSprintf ( "hits[%d]", iWord );
9700 dStatus.Add().SetSprintf ( INT64_FMT, tStat.m_iHits );
9701
9702 iWord++;
9703 }
9704 }
9705
9706
HandleCommandStatus(int iSock,int iVer,InputBuffer_c & tReq)9707 void HandleCommandStatus ( int iSock, int iVer, InputBuffer_c & tReq )
9708 {
9709 if ( !CheckCommandVersion ( iVer, VER_COMMAND_STATUS, tReq ) )
9710 return;
9711
9712 if ( !g_pStats )
9713 {
9714 tReq.SendErrorReply ( "performance counters disabled" );
9715 return;
9716 }
9717
9718 CSphVector<CSphString> dStatus;
9719 BuildStatus ( dStatus );
9720
9721 int iRespLen = 8; // int rows, int cols
9722 ARRAY_FOREACH ( i, dStatus )
9723 iRespLen += 4 + strlen ( dStatus[i].cstr() );
9724
9725 NetOutputBuffer_c tOut ( iSock );
9726 tOut.SendWord ( SEARCHD_OK );
9727 tOut.SendWord ( VER_COMMAND_STATUS );
9728 tOut.SendInt ( iRespLen );
9729
9730 tOut.SendInt ( dStatus.GetLength()/2 ); // rows
9731 tOut.SendInt ( 2 ); // cols
9732 ARRAY_FOREACH ( i, dStatus )
9733 tOut.SendString ( dStatus[i].cstr() );
9734
9735 tOut.Flush ();
9736 assert ( tOut.GetError()==true || tOut.GetSentCount()==8+iRespLen );
9737 }
9738
9739 //////////////////////////////////////////////////////////////////////////
9740 // FLUSH HANDLER
9741 //////////////////////////////////////////////////////////////////////////
9742
HandleCommandFlush(int iSock,int iVer,InputBuffer_c & tReq)9743 void HandleCommandFlush ( int iSock, int iVer, InputBuffer_c & tReq )
9744 {
9745 if ( !CheckCommandVersion ( iVer, VER_COMMAND_FLUSHATTRS, tReq ) )
9746 return;
9747
9748 if ( g_eWorkers==MPM_NONE )
9749 {
9750 // --console mode, no async thread/process to handle the check
9751 sphLogDebug ( "attrflush: --console mode, command ignored" );
9752
9753 } else
9754 {
9755 // force a check in head process, and wait it until completes
9756 // FIXME! semi active wait..
9757 sphLogDebug ( "attrflush: forcing check, tag=%d", g_pFlush->m_iFlushTag );
9758 g_pFlush->m_bForceCheck = true;
9759 while ( g_pFlush->m_bForceCheck )
9760 sphSleepMsec ( 1 );
9761
9762 // if we are flushing now, wait until flush completes
9763 while ( g_pFlush->m_bFlushing )
9764 sphSleepMsec ( 10 );
9765 sphLogDebug ( "attrflush: check finished, tag=%d", g_pFlush->m_iFlushTag );
9766 }
9767
9768 // return last flush tag, just for the fun of it
9769 NetOutputBuffer_c tOut ( iSock );
9770 tOut.SendWord ( SEARCHD_OK );
9771 tOut.SendWord ( VER_COMMAND_FLUSHATTRS );
9772 tOut.SendInt ( 4 ); // resplen, 1 dword
9773 tOut.SendInt ( g_pFlush->m_iFlushTag );
9774 tOut.Flush ();
9775 assert ( tOut.GetError()==true || tOut.GetSentCount()==12 ); // 8+resplen
9776 }
9777
9778 /////////////////////////////////////////////////////////////////////////////
9779 // GENERAL HANDLER
9780 /////////////////////////////////////////////////////////////////////////////
9781
9782 #define THD_STATE(_state) { if ( pThd ) pThd->m_eThdState = _state; }
9783 void HandleCommandSphinxql ( int iSock, int iVer, InputBuffer_c & tReq ); // definition is below
9784 void StatCountCommand ( int iCmd, int iCount=1 );
9785
HandleClientSphinx(int iSock,const char * sClientIP,ThdDesc_t * pThd)9786 void HandleClientSphinx ( int iSock, const char * sClientIP, ThdDesc_t * pThd )
9787 {
9788 MEMORY ( SPH_MEM_HANDLE_NONSQL );
9789 THD_STATE ( THD_HANDSHAKE );
9790
9791 bool bPersist = false;
9792 int iTimeout = g_iReadTimeout; // wait 5 sec until first command
9793 NetInputBuffer_c tBuf ( iSock );
9794 int64_t iCID = ( pThd ? pThd->m_iConnID : g_iConnID );
9795
9796 // send my version
9797 DWORD uServer = htonl ( SPHINX_SEARCHD_PROTO );
9798 if ( sphSockSend ( iSock, (char*)&uServer, sizeof(DWORD) )!=sizeof(DWORD) )
9799 {
9800 sphWarning ( "failed to send server version (client=%s("INT64_FMT"))", sClientIP, iCID );
9801 return;
9802 }
9803
9804 // get client version and request
9805 tBuf.ReadFrom ( 4 ); // FIXME! magic
9806 int iMagic = tBuf.GetInt (); // client version is for now unused
9807
9808 sphLogDebugv ( "conn %s("INT64_FMT"): got handshake, major v.%d, err %d", sClientIP, iCID, iMagic, (int)tBuf.GetError() );
9809 if ( tBuf.GetError() )
9810 {
9811 sphLogDebugv ( "conn %s("INT64_FMT"): exiting on handshake error", sClientIP, iCID );
9812 return;
9813 }
9814
9815 int iPconnIdle = 0;
9816 do
9817 {
9818 // in "persistent connection" mode, we want interruptible waits
9819 // so that the worker child could be forcibly restarted
9820 //
9821 // currently, the only signal allowed to interrupt this read is SIGTERM
9822 // letting SIGHUP interrupt causes trouble under query/rotation pressure
9823 // see sphSockRead() and ReadFrom() for details
9824 THD_STATE ( THD_NET_READ );
9825 bool bCommand = tBuf.ReadFrom ( 8, iTimeout, bPersist );
9826
9827 // on SIGTERM, bail unconditionally and immediately, at all times
9828 if ( !bCommand && g_bGotSigterm )
9829 {
9830 sphLogDebugv ( "conn %s("INT64_FMT"): bailing on SIGTERM", sClientIP, iCID );
9831 break;
9832 }
9833
9834 // on SIGHUP vs pconn, bail if a pconn was idle for 1 sec
9835 if ( bPersist && !bCommand && g_bGotSighup && sphSockPeekErrno()==ETIMEDOUT )
9836 {
9837 sphLogDebugv ( "conn %s("INT64_FMT"): bailing idle pconn on SIGHUP", sClientIP, iCID );
9838 break;
9839 }
9840
9841 // on pconn that was idle for 300 sec (client_timeout), bail
9842 if ( bPersist && !bCommand && sphSockPeekErrno()==ETIMEDOUT )
9843 {
9844 iPconnIdle += iTimeout;
9845 if ( iPconnIdle>=g_iClientTimeout )
9846 {
9847 sphLogDebugv ( "conn %s("INT64_FMT"): bailing idle pconn on client_timeout", sClientIP, iCID );
9848 break;
9849 }
9850 continue;
9851 } else
9852 iPconnIdle = 0;
9853
9854 // on any other signals vs pconn, ignore and keep looping
9855 // (redundant for now, as the only allowed interruption is SIGTERM, but.. let's keep it)
9856 if ( bPersist && !bCommand && tBuf.IsIntr() )
9857 continue;
9858
9859 // okay, signal related mess should be over, try to parse the command
9860 // (but some other socket error still might had happened, so beware)
9861 int iCommand = tBuf.GetWord ();
9862 int iCommandVer = tBuf.GetWord ();
9863 int iLength = tBuf.GetInt ();
9864 if ( tBuf.GetError() )
9865 {
9866 // under high load, there can be pretty frequent accept() vs connect() timeouts
9867 // lets avoid agent log flood
9868 //
9869 // sphWarning ( "failed to receive client version and request (client=%s, error=%s)", sClientIP, sphSockError() );
9870 sphLogDebugv ( "conn %s("INT64_FMT"): bailing on failed request header (sockerr=%s)", sClientIP, iCID, sphSockError() );
9871 return;
9872 }
9873
9874 // check request
9875 if ( iCommand<0 || iCommand>=SEARCHD_COMMAND_TOTAL
9876 || iLength<0 || iLength>g_iMaxPacketSize )
9877 {
9878 // unknown command, default response header
9879 tBuf.SendErrorReply ( "invalid command (code=%d, len=%d)", iCommand, iLength );
9880
9881 // if request length is insane, low level comm is broken, so we bail out
9882 if ( iLength<0 || iLength>g_iMaxPacketSize )
9883 sphWarning ( "ill-formed client request (length=%d out of bounds)", iLength );
9884
9885 // if command is insane, low level comm is broken, so we bail out
9886 if ( iCommand<0 || iCommand>=SEARCHD_COMMAND_TOTAL )
9887 sphWarning ( "ill-formed client request (command=%d, SEARCHD_COMMAND_TOTAL=%d)", iCommand, SEARCHD_COMMAND_TOTAL );
9888
9889 return;
9890 }
9891
9892 // count commands
9893 StatCountCommand ( iCommand );
9894
9895 // get request body
9896 assert ( iLength>=0 && iLength<=g_iMaxPacketSize );
9897 if ( iLength && !tBuf.ReadFrom ( iLength ) )
9898 {
9899 sphWarning ( "failed to receive client request body (client=%s("INT64_FMT"), exp=%d, error='%s')", sClientIP, iCID, iLength, sphSockError() );
9900 return;
9901 }
9902
9903 // set on query guard
9904 CrashQuery_t tCrashQuery;
9905 tCrashQuery.m_pQuery = tBuf.GetBufferPtr();
9906 tCrashQuery.m_iSize = iLength;
9907 tCrashQuery.m_bMySQL = false;
9908 tCrashQuery.m_uCMD = (WORD)iCommand;
9909 tCrashQuery.m_uVer = (WORD)iCommandVer;
9910 SphCrashLogger_c::SetLastQuery ( tCrashQuery );
9911
9912 // handle known commands
9913 assert ( iCommand>=0 && iCommand<SEARCHD_COMMAND_TOTAL );
9914
9915 if ( pThd )
9916 pThd->m_sCommand = g_dApiCommands[iCommand];
9917 THD_STATE ( THD_QUERY );
9918
9919 sphLogDebugv ( "conn %s("INT64_FMT"): got command %d, handling", sClientIP, iCID, iCommand );
9920 switch ( iCommand )
9921 {
9922 case SEARCHD_COMMAND_SEARCH: HandleCommandSearch ( iSock, iCommandVer, tBuf ); break;
9923 case SEARCHD_COMMAND_EXCERPT: HandleCommandExcerpt ( iSock, iCommandVer, tBuf ); break;
9924 case SEARCHD_COMMAND_KEYWORDS: HandleCommandKeywords ( iSock, iCommandVer, tBuf ); break;
9925 case SEARCHD_COMMAND_UPDATE: HandleCommandUpdate ( iSock, iCommandVer, tBuf ); break;
9926 case SEARCHD_COMMAND_PERSIST:
9927 bPersist = ( tBuf.GetInt()!=0 );
9928 iTimeout = 1;
9929 sphLogDebugv ( "conn %s("INT64_FMT"): pconn is now %s", sClientIP, iCID, bPersist ? "on" : "off" );
9930 break;
9931 case SEARCHD_COMMAND_STATUS: HandleCommandStatus ( iSock, iCommandVer, tBuf ); break;
9932 case SEARCHD_COMMAND_FLUSHATTRS:HandleCommandFlush ( iSock, iCommandVer, tBuf ); break;
9933 case SEARCHD_COMMAND_SPHINXQL: HandleCommandSphinxql ( iSock, iCommandVer, tBuf ); break;
9934 default: assert ( 0 && "INTERNAL ERROR: unhandled command" ); break;
9935 }
9936
9937 // set off query guard
9938 SphCrashLogger_c::SetLastQuery ( CrashQuery_t() );
9939 } while ( bPersist );
9940
9941 sphLogDebugv ( "conn %s("INT64_FMT"): exiting", sClientIP, iCID );
9942 }
9943
9944 //////////////////////////////////////////////////////////////////////////
9945 // MYSQLD PRETENDER
9946 //////////////////////////////////////////////////////////////////////////
9947
9948 // our copy of enum_field_types
9949 // we can't rely on mysql_com.h because it might be unavailable
9950 //
9951 // MYSQL_TYPE_DECIMAL = 0
9952 // MYSQL_TYPE_TINY = 1
9953 // MYSQL_TYPE_SHORT = 2
9954 // MYSQL_TYPE_LONG = 3
9955 // MYSQL_TYPE_FLOAT = 4
9956 // MYSQL_TYPE_DOUBLE = 5
9957 // MYSQL_TYPE_NULL = 6
9958 // MYSQL_TYPE_TIMESTAMP = 7
9959 // MYSQL_TYPE_LONGLONG = 8
9960 // MYSQL_TYPE_INT24 = 9
9961 // MYSQL_TYPE_DATE = 10
9962 // MYSQL_TYPE_TIME = 11
9963 // MYSQL_TYPE_DATETIME = 12
9964 // MYSQL_TYPE_YEAR = 13
9965 // MYSQL_TYPE_NEWDATE = 14
9966 // MYSQL_TYPE_VARCHAR = 15
9967 // MYSQL_TYPE_BIT = 16
9968 // MYSQL_TYPE_NEWDECIMAL = 246
9969 // MYSQL_TYPE_ENUM = 247
9970 // MYSQL_TYPE_SET = 248
9971 // MYSQL_TYPE_TINY_BLOB = 249
9972 // MYSQL_TYPE_MEDIUM_BLOB = 250
9973 // MYSQL_TYPE_LONG_BLOB = 251
9974 // MYSQL_TYPE_BLOB = 252
9975 // MYSQL_TYPE_VAR_STRING = 253
9976 // MYSQL_TYPE_STRING = 254
9977 // MYSQL_TYPE_GEOMETRY = 255
9978
9979 enum MysqlColumnType_e
9980 {
9981 MYSQL_COL_DECIMAL = 0,
9982 MYSQL_COL_LONG = 3,
9983 MYSQL_COL_FLOAT = 4,
9984 MYSQL_COL_LONGLONG = 8,
9985 MYSQL_COL_STRING = 254
9986 };
9987
9988
9989
SendMysqlFieldPacket(NetOutputBuffer_c & tOut,BYTE uPacketID,const char * sCol,MysqlColumnType_e eType)9990 void SendMysqlFieldPacket ( NetOutputBuffer_c & tOut, BYTE uPacketID, const char * sCol, MysqlColumnType_e eType )
9991 {
9992 const char * sDB = "";
9993 const char * sTable = "";
9994
9995 int iLen = 17 + MysqlPackedLen(sDB) + 2*( MysqlPackedLen(sTable) + MysqlPackedLen(sCol) );
9996
9997 int iColLen = 0;
9998 switch ( eType )
9999 {
10000 case MYSQL_COL_DECIMAL: iColLen = 20; break;
10001 case MYSQL_COL_LONG: iColLen = 11; break;
10002 case MYSQL_COL_FLOAT: iColLen = 20; break;
10003 case MYSQL_COL_LONGLONG: iColLen = 20; break;
10004 case MYSQL_COL_STRING: iColLen = 255; break;
10005 }
10006
10007 tOut.SendLSBDword ( (uPacketID<<24) + iLen );
10008 tOut.SendMysqlString ( "def" ); // catalog
10009 tOut.SendMysqlString ( sDB ); // db
10010 tOut.SendMysqlString ( sTable ); // table
10011 tOut.SendMysqlString ( sTable ); // org_table
10012 tOut.SendMysqlString ( sCol ); // name
10013 tOut.SendMysqlString ( sCol ); // org_name
10014
10015 tOut.SendByte ( 12 ); // filler, must be 12 (following pseudo-string length)
10016 tOut.SendByte ( 8 ); // charset_nr, 8 is latin1
10017 tOut.SendByte ( 0 ); // charset_nr
10018 tOut.SendLSBDword ( iColLen ); // length
10019 tOut.SendByte ( BYTE(eType) ); // type (0=decimal)
10020 tOut.SendWord ( 0 ); // flags
10021 tOut.SendByte ( 0 ); // decimals
10022 tOut.SendWord ( 0 ); // filler
10023 }
10024
10025
10026 // from mysqld_error.h
10027 enum MysqlErrors_e
10028 {
10029 MYSQL_ERR_UNKNOWN_COM_ERROR = 1047,
10030 MYSQL_ERR_SERVER_SHUTDOWN = 1053,
10031 MYSQL_ERR_PARSE_ERROR = 1064,
10032 MYSQL_ERR_FIELD_SPECIFIED_TWICE = 1110,
10033 MYSQL_ERR_NO_SUCH_TABLE = 1146
10034 };
10035
10036
SendMysqlErrorPacket(NetOutputBuffer_c & tOut,BYTE uPacketID,const char * sStmt,const char * sError,MysqlErrors_e iErr=MYSQL_ERR_PARSE_ERROR)10037 void SendMysqlErrorPacket ( NetOutputBuffer_c & tOut, BYTE uPacketID, const char * sStmt, const char * sError, MysqlErrors_e iErr=MYSQL_ERR_PARSE_ERROR )
10038 {
10039 if ( sError==NULL )
10040 sError = "(null)";
10041
10042 LogSphinxqlError ( sStmt, sError );
10043
10044 int iErrorLen = strlen(sError)+1; // including the trailing zero
10045 int iLen = 9 + iErrorLen;
10046 int iError = iErr; // pretend to be mysql syntax error for now
10047
10048 // send packet header
10049 tOut.SendLSBDword ( (uPacketID<<24) + iLen );
10050 tOut.SendByte ( 0xff ); // field count, always 0xff for error packet
10051 tOut.SendByte ( (BYTE)( iError & 0xff ) );
10052 tOut.SendByte ( (BYTE)( iError>>8 ) );
10053
10054 // send sqlstate (1 byte marker, 5 byte state)
10055 switch ( iErr )
10056 {
10057 case MYSQL_ERR_SERVER_SHUTDOWN:
10058 case MYSQL_ERR_UNKNOWN_COM_ERROR:
10059 tOut.SendBytes ( "#08S01", 6 );
10060 break;
10061 case MYSQL_ERR_NO_SUCH_TABLE:
10062 tOut.SendBytes ( "#42S02", 6 );
10063 break;
10064 default:
10065 tOut.SendBytes ( "#42000", 6 );
10066 break;
10067 }
10068
10069 // send error message
10070 tOut.SendBytes ( sError, iErrorLen );
10071 }
10072
10073
SendMysqlErrorPacketEx(NetOutputBuffer_c & tOut,BYTE uPacketID,MysqlErrors_e iErr,const char * sTemplate,...)10074 void SendMysqlErrorPacketEx ( NetOutputBuffer_c & tOut, BYTE uPacketID, MysqlErrors_e iErr, const char * sTemplate, ... )
10075 {
10076 char sBuf[1024];
10077 va_list ap;
10078
10079 va_start ( ap, sTemplate );
10080 vsnprintf ( sBuf, sizeof(sBuf), sTemplate, ap );
10081 va_end ( ap );
10082
10083 SendMysqlErrorPacket ( tOut, uPacketID, NULL, sBuf, iErr );
10084 }
10085
10086
SendMysqlEofPacket(NetOutputBuffer_c & tOut,BYTE uPacketID,int iWarns,bool bMoreResults=false)10087 void SendMysqlEofPacket ( NetOutputBuffer_c & tOut, BYTE uPacketID, int iWarns, bool bMoreResults=false )
10088 {
10089 if ( iWarns<0 ) iWarns = 0;
10090 if ( iWarns>65535 ) iWarns = 65535;
10091 if ( bMoreResults )
10092 #if USE_MYSQL
10093 iWarns |= ( SERVER_MORE_RESULTS_EXISTS<<16 );
10094 #else
10095 iWarns = iWarns;
10096 #endif
10097
10098 tOut.SendLSBDword ( (uPacketID<<24) + 5 );
10099 tOut.SendByte ( 0xfe );
10100 tOut.SendLSBDword ( iWarns ); // N warnings, 0 status
10101 }
10102
10103
SendMysqlOkPacket(NetOutputBuffer_c & tOut,BYTE uPacketID,int iAffectedRows=0,int iWarns=0,const char * sMessage=NULL)10104 void SendMysqlOkPacket ( NetOutputBuffer_c & tOut, BYTE uPacketID, int iAffectedRows=0, int iWarns=0, const char * sMessage=NULL )
10105 {
10106 DWORD iInsert_id = 0;
10107 char sVarLen[20] = {0}; // max 18 for packed number, +1 more just for fun
10108 void * pBuf = sVarLen;
10109 pBuf = MysqlPack ( pBuf, iAffectedRows );
10110 pBuf = MysqlPack ( pBuf, iInsert_id );
10111 int iLen = (char *) pBuf - sVarLen;
10112
10113 int iMsgLen = 0;
10114 if ( sMessage )
10115 iMsgLen = strlen(sMessage) + 1; // FIXME! does or doesn't the trailing zero necessary in Ok packet?
10116
10117 tOut.SendLSBDword ( (uPacketID<<24) + iLen + iMsgLen + 5);
10118 tOut.SendByte ( 0 ); // ok packet
10119 tOut.SendBytes ( sVarLen, iLen ); // packed affected rows & insert_id
10120 if ( iWarns<0 ) iWarns = 0;
10121 if ( iWarns>65535 ) iWarns = 65535;
10122 DWORD uWarnStatus = iWarns<<16;
10123 tOut.SendLSBDword ( uWarnStatus ); // N warnings, 0 status
10124 if ( iMsgLen > 0 )
10125 tOut.SendBytes ( sMessage, iMsgLen );
10126 }
10127
10128
10129 struct CmpColumns_fn
10130 {
IsLessCmpColumns_fn10131 inline bool IsLess ( const CSphString & a, const CSphString & b ) const
10132 {
10133 return CmpString ( a, b )<0;
10134 }
10135 };
10136
10137
HandleMysqlInsert(const SqlStmt_t & tStmt,NetOutputBuffer_c & tOut,BYTE uPacketID,bool bReplace,bool bCommit)10138 void HandleMysqlInsert ( const SqlStmt_t & tStmt, NetOutputBuffer_c & tOut, BYTE uPacketID, bool bReplace, bool bCommit )
10139 {
10140 MEMORY ( SPH_MEM_INSERT_SQL );
10141
10142 CSphString sError;
10143
10144 // get that index
10145 const ServedIndex_t * pServed = g_pIndexes->GetRlockedEntry ( tStmt.m_sIndex );
10146 if ( !pServed )
10147 {
10148 sError.SetSprintf ( "no such index '%s'", tStmt.m_sIndex.cstr() );
10149 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, sError.cstr() );
10150 return;
10151 }
10152
10153 if ( !pServed->m_bRT || !pServed->m_bEnabled )
10154 {
10155 pServed->Unlock();
10156 sError.SetSprintf ( "index '%s' does not support INSERT (enabled=%d)", tStmt.m_sIndex.cstr(), pServed->m_bEnabled );
10157 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, sError.cstr() );
10158 return;
10159 }
10160
10161 ISphRtIndex * pIndex = dynamic_cast<ISphRtIndex*> ( pServed->m_pIndex ); // FIXME? remove dynamic_cast?
10162 assert ( pIndex );
10163
10164 // get schema, check values count
10165 const CSphSchema & tSchema = pIndex->GetInternalSchema();
10166 int iSchemaSz = tSchema.GetAttrsCount() + tSchema.m_dFields.GetLength() + 1;
10167 int iExp = tStmt.m_iSchemaSz;
10168 int iGot = tStmt.m_dInsertValues.GetLength();
10169 if ( !tStmt.m_dInsertSchema.GetLength() && ( iSchemaSz!=tStmt.m_iSchemaSz ) )
10170 {
10171 pServed->Unlock();
10172 sError.SetSprintf ( "column count does not match schema (expected %d, got %d)", iSchemaSz, iGot );
10173 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, sError.cstr() );
10174 return;
10175 }
10176
10177 if ( ( iGot % iExp )!=0 )
10178 {
10179 pServed->Unlock();
10180 sError.SetSprintf ( "column count does not match value count (expected %d, got %d)", iExp, iGot );
10181 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, sError.cstr() );
10182 return;
10183 }
10184
10185 CSphVector<int> dAttrSchema ( tSchema.GetAttrsCount() );
10186 CSphVector<int> dFieldSchema ( tSchema.m_dFields.GetLength() );
10187 int iIdIndex = 0;
10188 if ( !tStmt.m_dInsertSchema.GetLength() )
10189 {
10190 // no columns list, use index schema
10191 ARRAY_FOREACH ( i, dFieldSchema )
10192 dFieldSchema[i] = i+1;
10193 int iFields = dFieldSchema.GetLength();
10194 ARRAY_FOREACH ( j, dAttrSchema )
10195 dAttrSchema[j] = j+iFields+1;
10196 } else
10197 {
10198 // got a list of columns, check for 1) existance, 2) dupes
10199 CSphVector<CSphString> dCheck = tStmt.m_dInsertSchema;
10200 ARRAY_FOREACH ( i, dCheck )
10201 // OPTIMIZE! GetAttrIndex and GetFieldIndex use the linear searching. M.b. hash instead?
10202 if ( dCheck[i]!="id" && tSchema.GetAttrIndex ( dCheck[i].cstr() )==-1 && tSchema.GetFieldIndex ( dCheck[i].cstr() )==-1 )
10203 {
10204 pServed->Unlock();
10205 sError.SetSprintf ( "unknown column: '%s'", dCheck[i].cstr() );
10206 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, sError.cstr(), MYSQL_ERR_PARSE_ERROR );
10207 return;
10208 }
10209
10210 dCheck.Sort ( CmpColumns_fn() );
10211
10212 ARRAY_FOREACH ( i, dCheck )
10213 if ( i>0 && dCheck[i-1]==dCheck[i] )
10214 {
10215 pServed->Unlock();
10216 sError.SetSprintf ( "column '%s' specified twice", dCheck[i].cstr() );
10217 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, sError.cstr(), MYSQL_ERR_FIELD_SPECIFIED_TWICE );
10218 return;
10219 }
10220
10221 // hash column list
10222 // OPTIMIZE! hash index columns once (!) instead
10223 SmallStringHash_T<int> dInsertSchema;
10224 ARRAY_FOREACH ( i, tStmt.m_dInsertSchema )
10225 dInsertSchema.Add ( i, tStmt.m_dInsertSchema[i] );
10226
10227 // get id index
10228 if ( !dInsertSchema.Exists("id") )
10229 {
10230 pServed->Unlock();
10231 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, "column list must contain an 'id' column" );
10232 return;
10233 }
10234 iIdIndex = dInsertSchema["id"];
10235
10236 // map fields
10237 bool bIdDupe = false;
10238 ARRAY_FOREACH ( i, dFieldSchema )
10239 {
10240 if ( dInsertSchema.Exists ( tSchema.m_dFields[i].m_sName ) )
10241 {
10242 int iField = dInsertSchema[tSchema.m_dFields[i].m_sName];
10243 if ( iField==iIdIndex )
10244 {
10245 bIdDupe = true;
10246 break;
10247 }
10248 dFieldSchema[i] = iField;
10249 } else
10250 dFieldSchema[i] = -1;
10251 }
10252 if ( bIdDupe )
10253 {
10254 pServed->Unlock();
10255 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, "fields must never be named 'id' (fix your config)" );
10256 return;
10257 }
10258
10259 // map attrs
10260 ARRAY_FOREACH ( j, dAttrSchema )
10261 {
10262 if ( dInsertSchema.Exists ( tSchema.GetAttr(j).m_sName ) )
10263 {
10264 int iField = dInsertSchema[tSchema.GetAttr(j).m_sName];
10265 if ( iField==iIdIndex )
10266 {
10267 bIdDupe = true;
10268 break;
10269 }
10270 dAttrSchema[j] = iField;
10271 } else
10272 dAttrSchema[j] = -1;
10273 }
10274 if ( bIdDupe )
10275 {
10276 pServed->Unlock();
10277 sError.SetSprintf ( "attributes must never be named 'id' (fix your config)" );
10278 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, sError.cstr() );
10279 return;
10280 }
10281 }
10282
10283 CSphVector<const char *> dStrings;
10284 CSphVector<DWORD> dMvas;
10285
10286 // convert attrs
10287 for ( int c=0; c<tStmt.m_iRowsAffected; c++ )
10288 {
10289 assert ( sError.IsEmpty() );
10290
10291 CSphMatchVariant tDoc;
10292 tDoc.Reset ( tSchema.GetRowSize() );
10293 tDoc.m_iDocID = (SphDocID_t)CSphMatchVariant::ToDocid ( tStmt.m_dInsertValues[iIdIndex + c * iExp] );
10294 dStrings.Resize ( 0 );
10295 dMvas.Resize ( 0 );
10296
10297 for ( int i=0; i<tSchema.GetAttrsCount(); i++ )
10298 {
10299 // shortcuts!
10300 const CSphColumnInfo & tCol = tSchema.GetAttr(i);
10301 CSphAttrLocator tLoc = tCol.m_tLocator;
10302 tLoc.m_bDynamic = true;
10303
10304 int iQuerySchemaIdx = dAttrSchema[i];
10305 bool bResult;
10306 if ( iQuerySchemaIdx < 0 )
10307 {
10308 bResult = tDoc.SetDefaultAttr ( tLoc, tCol.m_eAttrType );
10309 if ( tCol.m_eAttrType==SPH_ATTR_STRING )
10310 dStrings.Add ( NULL );
10311 if ( tCol.m_eAttrType==SPH_ATTR_UINT32SET || tCol.m_eAttrType==SPH_ATTR_INT64SET )
10312 dMvas.Add ( 0 );
10313 } else
10314 {
10315 const SqlInsert_t & tVal = tStmt.m_dInsertValues[iQuerySchemaIdx + c * iExp];
10316
10317 // sanity checks
10318 if ( tVal.m_iType!=TOK_QUOTED_STRING && tVal.m_iType!=TOK_CONST_INT && tVal.m_iType!=TOK_CONST_FLOAT && tVal.m_iType!=TOK_CONST_MVA )
10319 {
10320 sError.SetSprintf ( "raw %d, column %d: internal error: unknown insval type %d", 1+c, 1+iQuerySchemaIdx, tVal.m_iType ); // 1 for human base
10321 break;
10322 }
10323 if ( tVal.m_iType==TOK_CONST_MVA && !( tCol.m_eAttrType==SPH_ATTR_UINT32SET || tCol.m_eAttrType==SPH_ATTR_INT64SET ) )
10324 {
10325 sError.SetSprintf ( "raw %d, column %d: MVA value specified for a non-MVA column", 1+c, 1+iQuerySchemaIdx ); // 1 for human base
10326 break;
10327 }
10328 if ( ( tCol.m_eAttrType==SPH_ATTR_UINT32SET || tCol.m_eAttrType==SPH_ATTR_INT64SET ) && tVal.m_iType!=TOK_CONST_MVA )
10329 {
10330 sError.SetSprintf ( "raw %d, column %d: non-MVA value specified for a MVA column", 1+c, 1+iQuerySchemaIdx ); // 1 for human base
10331 break;
10332 }
10333
10334 if ( tCol.m_eAttrType==SPH_ATTR_UINT32SET || tCol.m_eAttrType==SPH_ATTR_INT64SET )
10335 {
10336 // collect data from scattered insvals
10337 // FIXME! maybe remove this mess, and just have a single m_dMvas pool in parser instead?
10338 int iLen = 0;
10339 if ( tVal.m_pVals.Ptr() )
10340 {
10341 tVal.m_pVals->Uniq();
10342 iLen = tVal.m_pVals->GetLength();
10343 }
10344 if ( tCol.m_eAttrType==SPH_ATTR_INT64SET )
10345 {
10346 dMvas.Add ( iLen*2 );
10347 for ( int j=0; j<iLen; j++ )
10348 {
10349 uint64_t uVal = ( *tVal.m_pVals.Ptr() )[j];
10350 DWORD uLow = (DWORD)uVal;
10351 DWORD uHi = (DWORD)( uVal>>32 );
10352 dMvas.Add ( uLow );
10353 dMvas.Add ( uHi );
10354 }
10355 } else
10356 {
10357 dMvas.Add ( iLen );
10358 for ( int j=0; j<iLen; j++ )
10359 dMvas.Add ( (DWORD)( *tVal.m_pVals.Ptr() )[j] );
10360 }
10361 }
10362
10363 // FIXME? index schema is lawfully static, but our temp match obviously needs to be dynamic
10364 bResult = tDoc.SetAttr ( tLoc, tVal, tCol.m_eAttrType );
10365 if ( tCol.m_eAttrType==SPH_ATTR_STRING )
10366 dStrings.Add ( tVal.m_sVal.cstr() );
10367 }
10368
10369 if ( !bResult )
10370 {
10371 sError.SetSprintf ( "internal error: unknown attribute type in INSERT (typeid=%d)", tCol.m_eAttrType );
10372 break;
10373 }
10374 }
10375 if ( !sError.IsEmpty() )
10376 break;
10377
10378 // convert fields
10379 CSphVector<const char*> dFields;
10380 ARRAY_FOREACH ( i, tSchema.m_dFields )
10381 {
10382 int iQuerySchemaIdx = dFieldSchema[i];
10383 if ( iQuerySchemaIdx < 0 )
10384 dFields.Add ( "" ); // default value
10385 else
10386 {
10387 if ( tStmt.m_dInsertValues [ iQuerySchemaIdx + c * iExp ].m_iType!=TOK_QUOTED_STRING )
10388 {
10389 sError.SetSprintf ( "row %d, column %d: string expected", 1+c, 1+iQuerySchemaIdx ); // 1 for human base
10390 break;
10391 }
10392 dFields.Add ( tStmt.m_dInsertValues[ iQuerySchemaIdx + c * iExp ].m_sVal.cstr() );
10393 }
10394 }
10395 if ( !sError.IsEmpty() )
10396 break;
10397
10398 // do add
10399 pIndex->AddDocument ( dFields.GetLength(), dFields.Begin(), tDoc, bReplace, dStrings.Begin(), dMvas, sError );
10400
10401 if ( !sError.IsEmpty() )
10402 break;
10403 }
10404
10405 // fire exit
10406 if ( !sError.IsEmpty() )
10407 {
10408 pServed->Unlock();
10409 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, sError.cstr() );
10410 return;
10411 }
10412
10413 // no errors so far
10414 if ( bCommit )
10415 pIndex->Commit ();
10416
10417 pServed->Unlock();
10418
10419 // my OK packet
10420 SendMysqlOkPacket ( tOut, uPacketID, tStmt.m_iRowsAffected );
10421 }
10422
10423
10424 // our copy of enum_server_command
10425 // we can't rely on mysql_com.h because it might be unavailable
10426 //
10427 // MYSQL_COM_SLEEP = 0
10428 // MYSQL_COM_QUIT = 1
10429 // MYSQL_COM_INIT_DB = 2
10430 // MYSQL_COM_QUERY = 3
10431 // MYSQL_COM_FIELD_LIST = 4
10432 // MYSQL_COM_CREATE_DB = 5
10433 // MYSQL_COM_DROP_DB = 6
10434 // MYSQL_COM_REFRESH = 7
10435 // MYSQL_COM_SHUTDOWN = 8
10436 // MYSQL_COM_STATISTICS = 9
10437 // MYSQL_COM_PROCESS_INFO = 10
10438 // MYSQL_COM_CONNECT = 11
10439 // MYSQL_COM_PROCESS_KILL = 12
10440 // MYSQL_COM_DEBUG = 13
10441 // MYSQL_COM_PING = 14
10442 // MYSQL_COM_TIME = 15
10443 // MYSQL_COM_DELAYED_INSERT = 16
10444 // MYSQL_COM_CHANGE_USER = 17
10445 // MYSQL_COM_BINLOG_DUMP = 18
10446 // MYSQL_COM_TABLE_DUMP = 19
10447 // MYSQL_COM_CONNECT_OUT = 20
10448 // MYSQL_COM_REGISTER_SLAVE = 21
10449 // MYSQL_COM_STMT_PREPARE = 22
10450 // MYSQL_COM_STMT_EXECUTE = 23
10451 // MYSQL_COM_STMT_SEND_LONG_DATA = 24
10452 // MYSQL_COM_STMT_CLOSE = 25
10453 // MYSQL_COM_STMT_RESET = 26
10454 // MYSQL_COM_SET_OPTION = 27
10455 // MYSQL_COM_STMT_FETCH = 28
10456
10457 enum
10458 {
10459 MYSQL_COM_QUIT = 1,
10460 MYSQL_COM_INIT_DB = 2,
10461 MYSQL_COM_QUERY = 3,
10462 MYSQL_COM_PING = 14,
10463 MYSQL_COM_SET_OPTION = 27
10464 };
10465
10466
HandleMysqlCallSnippets(NetOutputBuffer_c & tOut,BYTE uPacketID,SqlStmt_t & tStmt)10467 void HandleMysqlCallSnippets ( NetOutputBuffer_c & tOut, BYTE uPacketID, SqlStmt_t & tStmt )
10468 {
10469 CSphString sError;
10470
10471 // check arguments
10472 // string data, string index, string query, [named opts]
10473 if ( tStmt.m_dInsertValues.GetLength()!=3 )
10474 {
10475 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, "SNIPPETS() expectes exactly 3 arguments (data, index, query)" );
10476 return;
10477 }
10478 if ( tStmt.m_dInsertValues[0].m_iType!=TOK_QUOTED_STRING && tStmt.m_dInsertValues[0].m_iType!=TOK_CONST_STRINGS )
10479 {
10480 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, "SNIPPETS() argument 1 must be a string or a string list" );
10481 return;
10482 }
10483 if ( tStmt.m_dInsertValues[1].m_iType!=TOK_QUOTED_STRING )
10484 {
10485 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, "SNIPPETS() argument 2 must be a string" );
10486 return;
10487 }
10488 if ( tStmt.m_dInsertValues[2].m_iType!=TOK_QUOTED_STRING )
10489 {
10490 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, "SNIPPETS() argument 3 must be a string" );
10491 return;
10492 }
10493
10494 // do magics
10495 CSphString sIndex = tStmt.m_dInsertValues[1].m_sVal;
10496
10497 ExcerptQuery_t q;
10498 q.m_sWords = tStmt.m_dInsertValues[2].m_sVal;
10499
10500 ARRAY_FOREACH ( i, tStmt.m_dCallOptNames )
10501 {
10502 CSphString & sOpt = tStmt.m_dCallOptNames[i];
10503 const SqlInsert_t & v = tStmt.m_dCallOptValues[i];
10504
10505 sOpt.ToLower();
10506 int iExpType = -1;
10507
10508 if ( sOpt=="before_match" ) { q.m_sBeforeMatch = v.m_sVal; iExpType = TOK_QUOTED_STRING; }
10509 else if ( sOpt=="after_match" ) { q.m_sAfterMatch = v.m_sVal; iExpType = TOK_QUOTED_STRING; }
10510 else if ( sOpt=="chunk_separator" ) { q.m_sChunkSeparator = v.m_sVal; iExpType = TOK_QUOTED_STRING; }
10511 else if ( sOpt=="html_strip_mode" ) { q.m_sStripMode = v.m_sVal; iExpType = TOK_QUOTED_STRING; }
10512 else if ( sOpt=="passage_boundary" ) { q.m_sRawPassageBoundary = v.m_sVal; iExpType = TOK_QUOTED_STRING; }
10513
10514 else if ( sOpt=="limit" ) { q.m_iLimit = (int)v.m_iVal; iExpType = TOK_CONST_INT; }
10515 else if ( sOpt=="limit_words" ) { q.m_iLimitWords = (int)v.m_iVal; iExpType = TOK_CONST_INT; }
10516 else if ( sOpt=="limit_passages" ) { q.m_iLimitPassages = (int)v.m_iVal; iExpType = TOK_CONST_INT; }
10517 else if ( sOpt=="around" ) { q.m_iAround = (int)v.m_iVal; iExpType = TOK_CONST_INT; }
10518 else if ( sOpt=="start_passage_id" ) { q.m_iPassageId = (int)v.m_iVal; iExpType = TOK_CONST_INT; }
10519
10520 else if ( sOpt=="exact_phrase" ) { q.m_bExactPhrase = ( v.m_iVal!=0 ); iExpType = TOK_CONST_INT; }
10521 else if ( sOpt=="use_boundaries" ) { q.m_bUseBoundaries = ( v.m_iVal!=0 ); iExpType = TOK_CONST_INT; }
10522 else if ( sOpt=="weight_order" ) { q.m_bWeightOrder = ( v.m_iVal!=0 ); iExpType = TOK_CONST_INT; }
10523 else if ( sOpt=="query_mode" ) { q.m_bHighlightQuery = ( v.m_iVal!=0 ); iExpType = TOK_CONST_INT; }
10524 else if ( sOpt=="force_all_words" ) { q.m_bForceAllWords = ( v.m_iVal!=0 ); iExpType = TOK_CONST_INT; }
10525 else if ( sOpt=="load_files" ) { q.m_iLoadFiles = ( v.m_iVal!=0 ); iExpType = TOK_CONST_INT; }
10526 else if ( sOpt=="load_files_scattered" ) { q.m_iLoadFiles |= ( v.m_iVal!=0 )?2:0; iExpType = TOK_CONST_INT; }
10527 else if ( sOpt=="allow_empty" ) { q.m_bAllowEmpty = ( v.m_iVal!=0 ); iExpType = TOK_CONST_INT; }
10528 else if ( sOpt=="emit_zones" ) { q.m_bEmitZones = ( v.m_iVal!=0 ); iExpType = TOK_CONST_INT; }
10529
10530 else
10531 {
10532 sError.SetSprintf ( "unknown option %s", sOpt.cstr() );
10533 break;
10534 }
10535
10536 // post-conf type check
10537 if ( iExpType!=v.m_iType )
10538 {
10539 sError.SetSprintf ( "unexpected option %s type", sOpt.cstr() );
10540 break;
10541 }
10542 }
10543 if ( !sError.IsEmpty() )
10544 {
10545 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, sError.cstr() );
10546 return;
10547 }
10548
10549 q.m_ePassageSPZ = sphGetPassageBoundary ( q.m_sRawPassageBoundary );
10550
10551 if ( !sphCheckOptionsSPZ ( q, q.m_sRawPassageBoundary, sError ) )
10552 {
10553 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, sError.cstr() );
10554 return;
10555 }
10556
10557 q.m_bHasBeforePassageMacro = SnippetTransformPassageMacros ( q.m_sBeforeMatch, q.m_sBeforeMatchPassage );
10558 q.m_bHasAfterPassageMacro = SnippetTransformPassageMacros ( q.m_sAfterMatch, q.m_sAfterMatchPassage );
10559 q.m_iRawFlags = GetRawSnippetFlags ( q );
10560
10561 CSphVector<ExcerptQuery_t> dQueries;
10562 if ( tStmt.m_dInsertValues[0].m_iType==TOK_QUOTED_STRING )
10563 {
10564 q.m_sSource = tStmt.m_dInsertValues[0].m_sVal; // OPTIMIZE?
10565 dQueries.Add ( q );
10566 } else
10567 {
10568 dQueries.Resize ( tStmt.m_dCallStrings.GetLength() );
10569 ARRAY_FOREACH ( i, tStmt.m_dCallStrings )
10570 {
10571 dQueries[i] = q; // copy the settings
10572 dQueries[i].m_sSource = tStmt.m_dCallStrings[i]; // OPTIMIZE?
10573 }
10574 }
10575
10576 if ( !MakeSnippets ( sIndex, dQueries, sError ) )
10577 {
10578 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, sError.cstr() );
10579 return;
10580 }
10581
10582 CSphVector<char*> dResults ( dQueries.GetLength() );
10583 ARRAY_FOREACH ( i, dResults )
10584 dResults[i] = dQueries[i].m_sRes;
10585
10586 bool bGotData = ARRAY_ANY ( bGotData, dResults, dResults[_any]!=NULL );
10587 if ( !bGotData )
10588 {
10589 // just one last error instead of all errors is hopefully ok
10590 sError.SetSprintf ( "highlighting failed: %s", sError.cstr() );
10591 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, sError.cstr() );
10592 return;
10593 }
10594
10595 // result set header packet
10596 tOut.SendLSBDword ( ((uPacketID++)<<24) + 2 );
10597 tOut.SendByte ( 1 ); // field count (snippet)
10598 tOut.SendByte ( 0 ); // extra
10599
10600 // fields
10601 SendMysqlFieldPacket ( tOut, uPacketID++, "snippet", MYSQL_COL_STRING );
10602 SendMysqlEofPacket ( tOut, uPacketID++, 0 );
10603
10604 // data
10605 ARRAY_FOREACH ( i, dResults )
10606 {
10607 const char * sResult = dResults[i] ? dResults[i] : "";
10608 tOut.SendLSBDword ( ((uPacketID++)<<24) + MysqlPackedLen ( sResult ) );
10609 tOut.SendMysqlString ( sResult );
10610 }
10611
10612 SendMysqlEofPacket ( tOut, uPacketID++, 0 );
10613 ARRAY_FOREACH ( i, dResults )
10614 SafeDeleteArray ( dResults[i] );
10615 }
10616
10617
HandleMysqlCallKeywords(NetOutputBuffer_c & tOut,BYTE uPacketID,SqlStmt_t & tStmt)10618 void HandleMysqlCallKeywords ( NetOutputBuffer_c & tOut, BYTE uPacketID, SqlStmt_t & tStmt )
10619 {
10620 CSphString sError;
10621
10622 // string query, string index, [bool hits]
10623 int iArgs = tStmt.m_dInsertValues.GetLength();
10624 if ( iArgs<2
10625 || iArgs>3
10626 || tStmt.m_dInsertValues[0].m_iType!=TOK_QUOTED_STRING
10627 || tStmt.m_dInsertValues[1].m_iType!=TOK_QUOTED_STRING
10628 || ( iArgs==3 && tStmt.m_dInsertValues[2].m_iType!=TOK_CONST_INT ) )
10629 {
10630 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, "bad argument count or types in KEYWORDS() call" );
10631 return;
10632 }
10633
10634 const ServedIndex_t * pServed = g_pIndexes->GetRlockedEntry ( tStmt.m_dInsertValues[1].m_sVal );
10635 if ( !pServed || !pServed->m_bEnabled || !pServed->m_pIndex )
10636 {
10637 if ( pServed )
10638 pServed->Unlock();
10639 sError.SetSprintf ( "no such index %s", tStmt.m_dInsertValues[1].m_sVal.cstr() );
10640 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, sError.cstr() );
10641 return;
10642 }
10643
10644 CSphVector<CSphKeywordInfo> dKeywords;
10645 bool bStats = ( iArgs==3 && tStmt.m_dInsertValues[2].m_iVal!=0 );
10646 bool bRes = pServed->m_pIndex->GetKeywords ( dKeywords, tStmt.m_dInsertValues[0].m_sVal.cstr(), bStats, sError );
10647 pServed->Unlock ();
10648
10649 if ( !bRes )
10650 {
10651 sError.SetSprintf ( "keyword extraction failed: %s", sError.cstr() );
10652 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, sError.cstr() );
10653 return;
10654 }
10655
10656 // result set header packet
10657 tOut.SendLSBDword ( ((uPacketID++)<<24) + 2 );
10658 tOut.SendByte ( 2 + ( bStats ? 2 : 0 ) ); // field count (tokenized, normalized, docs, hits)
10659 tOut.SendByte ( 0 ); // extra
10660
10661 // fields
10662 SendMysqlFieldPacket ( tOut, uPacketID++, "tokenized", MYSQL_COL_STRING );
10663 SendMysqlFieldPacket ( tOut, uPacketID++, "normalized", MYSQL_COL_STRING );
10664 if ( bStats )
10665 {
10666 SendMysqlFieldPacket ( tOut, uPacketID++, "docs", MYSQL_COL_STRING );
10667 SendMysqlFieldPacket ( tOut, uPacketID++, "hits", MYSQL_COL_STRING );
10668 }
10669 SendMysqlEofPacket ( tOut, uPacketID++, 0 );
10670
10671 // data
10672 ARRAY_FOREACH ( i, dKeywords )
10673 {
10674 char sDocs[16], sHits[16];
10675 snprintf ( sDocs, sizeof(sDocs), "%d", dKeywords[i].m_iDocs );
10676 snprintf ( sHits, sizeof(sHits), "%d", dKeywords[i].m_iHits );
10677
10678 int iPacketLen = MysqlPackedLen ( dKeywords[i].m_sTokenized.cstr() ) + MysqlPackedLen ( dKeywords[i].m_sNormalized.cstr() );
10679 if ( bStats )
10680 iPacketLen += MysqlPackedLen ( sDocs ) + MysqlPackedLen ( sHits );
10681
10682 tOut.SendLSBDword ( ((uPacketID++)<<24) + iPacketLen );
10683 tOut.SendMysqlString ( dKeywords[i].m_sTokenized.cstr() );
10684 tOut.SendMysqlString ( dKeywords[i].m_sNormalized.cstr() );
10685 if ( bStats )
10686 {
10687 tOut.SendMysqlString ( sDocs );
10688 tOut.SendMysqlString ( sHits );
10689 }
10690 }
10691
10692 SendMysqlEofPacket ( tOut, uPacketID++, 0 );
10693 }
10694
10695
HandleMysqlDescribe(NetOutputBuffer_c & tOut,BYTE uPacketID,SqlStmt_t & tStmt)10696 void HandleMysqlDescribe ( NetOutputBuffer_c & tOut, BYTE uPacketID, SqlStmt_t & tStmt )
10697 {
10698 const ServedIndex_t * pServed = g_pIndexes->GetRlockedEntry ( tStmt.m_sIndex );
10699 if ( !pServed || !pServed->m_bEnabled || !pServed->m_pIndex )
10700 {
10701 CSphString sError;
10702 if ( pServed )
10703 pServed->Unlock();
10704 sError.SetSprintf ( "no such index '%s'", tStmt.m_sIndex.cstr() );
10705 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, sError.cstr(), MYSQL_ERR_NO_SUCH_TABLE );
10706 return;
10707 }
10708
10709 // result set header packet
10710 tOut.SendLSBDword ( ((uPacketID++)<<24) + 2 );
10711 tOut.SendByte ( 2 ); // field count (field, type)
10712 tOut.SendByte ( 0 ); // extra
10713
10714 // fields
10715 SendMysqlFieldPacket ( tOut, uPacketID++, "Field", MYSQL_COL_STRING );
10716 SendMysqlFieldPacket ( tOut, uPacketID++, "Type", MYSQL_COL_STRING );
10717 SendMysqlEofPacket ( tOut, uPacketID++, 0 );
10718
10719 // data
10720 const char * sIdType = USE_64BIT ? "bigint" : "integer";
10721 tOut.SendLSBDword ( ((uPacketID++)<<24) + 3 + MysqlPackedLen ( sIdType ) );
10722 tOut.SendMysqlString ( "id" );
10723 tOut.SendMysqlString ( sIdType );
10724
10725 const CSphSchema & tSchema = pServed->m_pIndex->GetMatchSchema();
10726 ARRAY_FOREACH ( i, tSchema.m_dFields )
10727 {
10728 const CSphColumnInfo & tCol = tSchema.m_dFields[i];
10729 tOut.SendLSBDword ( ((uPacketID++)<<24) + MysqlPackedLen ( tCol.m_sName.cstr() ) + 6 );
10730 tOut.SendMysqlString ( tCol.m_sName.cstr() );
10731 tOut.SendMysqlString ( "field" );
10732 }
10733
10734 for ( int i=0; i<tSchema.GetAttrsCount(); i++ )
10735 {
10736 const CSphColumnInfo & tCol = tSchema.GetAttr(i);
10737 const char * sType = sphTypeName ( tCol.m_eAttrType );
10738
10739 tOut.SendLSBDword ( ((uPacketID++)<<24) + MysqlPackedLen ( tCol.m_sName.cstr() ) + MysqlPackedLen ( sType ) );
10740 tOut.SendMysqlString ( tCol.m_sName.cstr() );
10741 tOut.SendMysqlString ( sType );
10742 }
10743
10744 pServed->Unlock();
10745 SendMysqlEofPacket ( tOut, uPacketID++, 0 );
10746 }
10747
10748
10749 struct IndexNameLess_fn
10750 {
IsLessIndexNameLess_fn10751 inline bool IsLess ( const CSphNamedInt & a, const CSphNamedInt & b ) const
10752 {
10753 return strcasecmp ( a.m_sName.cstr(), b.m_sName.cstr() )<0;
10754 }
10755 };
10756
10757
HandleMysqlShowTables(NetOutputBuffer_c & tOut,BYTE uPacketID)10758 void HandleMysqlShowTables ( NetOutputBuffer_c & tOut, BYTE uPacketID )
10759 {
10760 // result set header packet
10761 tOut.SendLSBDword ( ((uPacketID++)<<24) + 2 );
10762 tOut.SendByte ( 2 ); // field count (index, type)
10763 tOut.SendByte ( 0 ); // extra
10764
10765 // fields
10766 SendMysqlFieldPacket ( tOut, uPacketID++, "Index", MYSQL_COL_STRING );
10767 SendMysqlFieldPacket ( tOut, uPacketID++, "Type", MYSQL_COL_STRING );
10768 SendMysqlEofPacket ( tOut, uPacketID++, 0 );
10769
10770 // all the indexes
10771 // 0 local, 1 distributed, 2 rt
10772 CSphVector<CSphNamedInt> dIndexes;
10773
10774 for ( IndexHashIterator_c it ( g_pIndexes ); it.Next(); )
10775 if ( it.Get().m_bEnabled )
10776 {
10777 CSphNamedInt & tIdx = dIndexes.Add();
10778 tIdx.m_sName = it.GetKey();
10779 tIdx.m_iValue = it.Get().m_bRT ? 2 : 0;
10780 }
10781
10782 g_tDistLock.Lock();
10783 g_hDistIndexes.IterateStart();
10784 while ( g_hDistIndexes.IterateNext() )
10785 {
10786 CSphNamedInt & tIdx = dIndexes.Add();
10787 tIdx.m_sName = g_hDistIndexes.IterateGetKey();
10788 tIdx.m_iValue = 1;
10789 }
10790 g_tDistLock.Unlock();
10791
10792 dIndexes.Sort ( IndexNameLess_fn() );
10793 ARRAY_FOREACH ( i, dIndexes )
10794 {
10795 const char * sType = "?";
10796 switch ( dIndexes[i].m_iValue )
10797 {
10798 case 0: sType = "local"; break;
10799 case 1: sType = "distributed"; break;
10800 case 2: sType = "rt"; break;
10801 }
10802
10803 tOut.SendLSBDword ( ((uPacketID++)<<24) + MysqlPackedLen ( dIndexes[i].m_sName.cstr() ) + MysqlPackedLen ( sType ) );
10804 tOut.SendMysqlString ( dIndexes[i].m_sName.cstr() );
10805 tOut.SendMysqlString ( sType );
10806 }
10807
10808 SendMysqlEofPacket ( tOut, uPacketID++, 0 );
10809 }
10810
10811 /////////////////////////////////////////////////////////////////////////////
10812 // SMART UPDATES HANDLER
10813 /////////////////////////////////////////////////////////////////////////////
10814
10815 struct SphinxqlRequestBuilder_t : public IRequestBuilder_t
10816 {
SphinxqlRequestBuilder_tSphinxqlRequestBuilder_t10817 explicit SphinxqlRequestBuilder_t ( const CSphString sQuery, const SqlStmt_t & tStmt )
10818 {
10819 m_sBegin.SetBinary ( sQuery.cstr(), tStmt.m_iListStart );
10820 m_sEnd.SetBinary ( sQuery.cstr() + tStmt.m_iListEnd, sQuery.Length() - tStmt.m_iListEnd );
10821 }
10822 virtual void BuildRequest ( const char * sIndexes, NetOutputBuffer_c & tOut, int ) const;
10823
10824 protected:
10825 CSphString m_sBegin;
10826 CSphString m_sEnd;
10827 };
10828
10829
10830 struct SphinxqlReplyParser_t : public IReplyParser_t
10831 {
SphinxqlReplyParser_tSphinxqlReplyParser_t10832 explicit SphinxqlReplyParser_t ( int * pUpd, int * pWarns )
10833 : m_pUpdated ( pUpd )
10834 , m_pWarns ( pWarns )
10835 {}
10836
ParseReplySphinxqlReplyParser_t10837 virtual bool ParseReply ( MemInputBuffer_c & tReq, AgentConn_t &, int ) const
10838 {
10839 DWORD uSize = ( tReq.GetLSBDword() & 0x00ffffff ) - 1;
10840 BYTE uCommand = tReq.GetByte();
10841
10842 if ( uCommand==0 )
10843 {
10844 *m_pUpdated += MysqlUnpack ( tReq, &uSize );
10845 MysqlUnpack ( tReq, &uSize ); // insert id
10846 *m_pWarns += tReq.GetLSBDword();
10847 uSize -= 4;
10848 if ( uSize )
10849 tReq.GetRawString ( uSize ); // message
10850 return true;
10851 }
10852 if ( uCommand==0xff )
10853 {
10854 tReq.GetByte(); tReq.GetByte(); // error
10855 uSize -= 2;
10856 if ( uSize )
10857 tReq.GetRawString ( uSize ); // ignoring message
10858 }
10859 return false;
10860 }
10861
10862 protected:
10863 int * m_pUpdated;
10864 int * m_pWarns;
10865 };
10866
10867
BuildRequest(const char * sIndexes,NetOutputBuffer_c & tOut,int) const10868 void SphinxqlRequestBuilder_t::BuildRequest ( const char * sIndexes, NetOutputBuffer_c & tOut, int ) const
10869 {
10870 int iReqSize = strlen(sIndexes) + m_sBegin.Length() + m_sEnd.Length(); // indexes string
10871
10872 // header
10873 tOut.SendDword ( SPHINX_SEARCHD_PROTO );
10874 tOut.SendWord ( SEARCHD_COMMAND_SPHINXQL );
10875 tOut.SendWord ( VER_COMMAND_SPHINXQL );
10876 tOut.SendInt ( iReqSize + 4 );
10877
10878 tOut.SendInt ( iReqSize );
10879 tOut.SendBytes ( m_sBegin.cstr(), m_sBegin.Length() );
10880 tOut.SendBytes ( sIndexes, strlen(sIndexes) );
10881 tOut.SendBytes ( m_sEnd.cstr(), m_sEnd.Length() );
10882 }
10883
10884 //////////////////////////////////////////////////////////////////////////
DoExtendedUpdate(const char * sIndex,const SqlStmt_t & tStmt,int & iSuccesses,int & iUpdated,bool bCommit,SearchFailuresLog_c & dFails,const ServedIndex_t * pServed)10885 static void DoExtendedUpdate ( const char * sIndex, const SqlStmt_t & tStmt,
10886 int & iSuccesses, int & iUpdated, bool bCommit,
10887 SearchFailuresLog_c & dFails, const ServedIndex_t * pServed )
10888 {
10889 if ( !pServed || !pServed->m_pIndex || !pServed->m_bEnabled )
10890 {
10891 if ( pServed )
10892 pServed->Unlock();
10893 dFails.Submit ( sIndex, "index not available" );
10894 return;
10895 }
10896
10897 SearchHandler_c tHandler ( 1, true ); // handler unlocks index at destructor - no need to do it manually
10898 CSphAttrUpdateEx tUpdate;
10899 CSphString sError;
10900
10901 tUpdate.m_pUpdate = &tStmt.m_tUpdate;
10902 tUpdate.m_pIndex = pServed->m_pIndex;
10903 tUpdate.m_pError = &sError;
10904
10905 tHandler.RunUpdates ( tStmt.m_tQuery, sIndex, &tUpdate );
10906
10907 if ( sError.Length() )
10908 {
10909 dFails.Submit ( sIndex, sError.cstr() );
10910 return;
10911 }
10912
10913 if ( bCommit && pServed->m_bRT )
10914 {
10915 ISphRtIndex * pIndex = static_cast<ISphRtIndex *> ( pServed->m_pIndex );
10916 pIndex->Commit ();
10917 }
10918
10919 iUpdated += tUpdate.m_iAffected;
10920 iSuccesses++;
10921 }
10922
10923
10924
HandleMysqlUpdate(NetOutputBuffer_c & tOut,BYTE uPacketID,const SqlStmt_t & tStmt,const CSphString & sQuery,bool bCommit)10925 void HandleMysqlUpdate ( NetOutputBuffer_c & tOut, BYTE uPacketID, const SqlStmt_t & tStmt, const CSphString & sQuery, bool bCommit )
10926 {
10927 CSphString sError;
10928
10929 // check index names
10930 CSphVector<CSphString> dIndexNames;
10931 ParseIndexList ( tStmt.m_sIndex, dIndexNames );
10932
10933 if ( !dIndexNames.GetLength() )
10934 {
10935 sError.SetSprintf ( "no such index '%s'", tStmt.m_sIndex.cstr() );
10936 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, sError.cstr() );
10937 return;
10938 }
10939
10940 CSphVector<DistributedIndex_t> dDistributed ( dIndexNames.GetLength() ); // lock safe storage for distributed indexes
10941 ARRAY_FOREACH ( i, dIndexNames )
10942 {
10943 if ( !g_pIndexes->Exists ( dIndexNames[i] ) )
10944 {
10945 // search amongst distributed and copy for further processing
10946 g_tDistLock.Lock();
10947 const DistributedIndex_t * pDistIndex = g_hDistIndexes ( dIndexNames[i] );
10948
10949 if ( pDistIndex )
10950 {
10951 dDistributed[i] = *pDistIndex;
10952 }
10953
10954 g_tDistLock.Unlock();
10955
10956 if ( pDistIndex )
10957 continue;
10958 else
10959 {
10960 sError.SetSprintf ( "unknown index '%s' in update request", dIndexNames[i].cstr() );
10961 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, sError.cstr() );
10962 return;
10963 }
10964 }
10965 }
10966
10967 // do update
10968 SearchFailuresLog_c dFails;
10969 int iSuccesses = 0;
10970 int iUpdated = 0;
10971 int iWarns = 0;
10972
10973 bool bMvaUpdate = false;
10974 ARRAY_FOREACH_COND ( i, tStmt.m_tUpdate.m_dAttrs, !bMvaUpdate )
10975 {
10976 bMvaUpdate = ( tStmt.m_tUpdate.m_dAttrs[i].m_eAttrType==SPH_ATTR_UINT32SET
10977 || tStmt.m_tUpdate.m_dAttrs[i].m_eAttrType==SPH_ATTR_INT64SET );
10978 }
10979
10980 ARRAY_FOREACH ( iIdx, dIndexNames )
10981 {
10982 const char * sReqIndex = dIndexNames[iIdx].cstr();
10983 const ServedIndex_t * pLocked = UpdateGetLockedIndex ( sReqIndex, bMvaUpdate );
10984 if ( pLocked )
10985 {
10986 DoExtendedUpdate ( sReqIndex, tStmt, iSuccesses, iUpdated, bCommit, dFails, pLocked );
10987 } else
10988 {
10989 assert ( dDistributed[iIdx].m_dLocal.GetLength() || dDistributed[iIdx].m_dAgents.GetLength() );
10990 CSphVector<CSphString>& dLocal = dDistributed[iIdx].m_dLocal;
10991
10992 ARRAY_FOREACH ( i, dLocal )
10993 {
10994 const char * sLocal = dLocal[i].cstr();
10995 const ServedIndex_t * pServed = UpdateGetLockedIndex ( sLocal, bMvaUpdate );
10996 DoExtendedUpdate ( sLocal, tStmt, iSuccesses, iUpdated, bCommit, dFails, pServed );
10997 }
10998 }
10999
11000 // update remote agents
11001 if ( dDistributed[iIdx].m_dAgents.GetLength() )
11002 {
11003 DistributedIndex_t & tDist = dDistributed[iIdx];
11004
11005 CSphVector<AgentConn_t> dAgents ( tDist.m_dAgents.GetLength() );
11006 ARRAY_FOREACH ( i, dAgents )
11007 dAgents[i] = tDist.m_dAgents[i];
11008
11009 // connect to remote agents and query them
11010 ConnectToRemoteAgents ( dAgents, false );
11011
11012 SphinxqlRequestBuilder_t tReqBuilder ( sQuery, tStmt );
11013 int iRemote = QueryRemoteAgents ( dAgents, tDist.m_iAgentConnectTimeout, tReqBuilder, NULL ); // FIXME? profile update time too?
11014
11015 if ( iRemote )
11016 {
11017 SphinxqlReplyParser_t tParser ( &iUpdated, &iWarns );
11018 iSuccesses += WaitForRemoteAgents ( dAgents, tDist.m_iAgentQueryTimeout, tParser, NULL ); // FIXME? profile update time too?
11019 }
11020 }
11021 }
11022
11023 CSphStringBuilder sReport;
11024 dFails.BuildReport ( sReport );
11025
11026 if ( !iSuccesses )
11027 {
11028 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, sReport.cstr() );
11029 return;
11030 }
11031
11032 SendMysqlOkPacket ( tOut, uPacketID, iUpdated, iWarns );
11033 }
11034
11035 //////////////////////////////////////////////////////////////////////////
11036
11037 #define SPH_MAX_NUMERIC_STR 64
11038 class SqlRowBuffer_c
11039 {
11040 public:
11041
SqlRowBuffer_c()11042 SqlRowBuffer_c ()
11043 : m_pBuf ( NULL )
11044 , m_iLen ( 0 )
11045 , m_iLimit ( sizeof ( m_dBuf ) )
11046 {
11047 }
11048
~SqlRowBuffer_c()11049 ~SqlRowBuffer_c ()
11050 {
11051 SafeDeleteArray ( m_pBuf );
11052 }
11053
Reserve(int iLen)11054 char * Reserve ( int iLen )
11055 {
11056 int iNewSize = m_iLen+iLen;
11057 if ( iNewSize<=m_iLimit )
11058 return Get();
11059
11060 int iNewLimit = Max ( m_iLimit*2, iNewSize );
11061 char * pBuf = new char [iNewLimit];
11062 memcpy ( pBuf, m_pBuf ? m_pBuf : m_dBuf, m_iLen );
11063 SafeDeleteArray ( m_pBuf );
11064 m_pBuf = pBuf;
11065 m_iLimit = iNewLimit;
11066 return Get();
11067 }
11068
Get()11069 char * Get ()
11070 {
11071 return m_pBuf ? m_pBuf+m_iLen : m_dBuf+m_iLen;
11072 }
11073
Off(int iOff)11074 char * Off ( int iOff )
11075 {
11076 assert ( iOff<m_iLimit );
11077 return m_pBuf ? m_pBuf+iOff : m_dBuf+iOff;
11078 }
11079
Length() const11080 int Length () const
11081 {
11082 return m_iLen;
11083 }
11084
IncPtr(int iLen)11085 void IncPtr ( int iLen )
11086 {
11087 assert ( m_iLen+iLen<=m_iLimit );
11088 m_iLen += iLen;
11089 }
11090
Reset()11091 void Reset ()
11092 {
11093 m_iLen = 0;
11094 }
11095
11096 template < typename T>
PutNumeric(const char * sFormat,T tVal)11097 void PutNumeric ( const char * sFormat, T tVal )
11098 {
11099 Reserve ( SPH_MAX_NUMERIC_STR );
11100 int iLen = snprintf ( Get()+1, SPH_MAX_NUMERIC_STR-1, sFormat, tVal );
11101 *Get() = BYTE(iLen);
11102 IncPtr ( 1+iLen );
11103 }
11104
PutString(const char * sMsg)11105 void PutString ( const char * sMsg )
11106 {
11107 assert ( sMsg );
11108
11109 int iLen = sMsg ? strlen ( sMsg ) : 0;
11110 Reserve ( 1+iLen );
11111 char * pBegin = Get();
11112 char * pStr = (char *)MysqlPack ( pBegin, iLen );
11113 if ( pStr>pBegin )
11114 {
11115 memcpy ( pStr, sMsg, iLen );
11116 IncPtr ( ( pStr-pBegin )+iLen );
11117 }
11118 }
11119
11120
11121 private:
11122
11123 char m_dBuf[4096];
11124 char * m_pBuf;
11125 int m_iLen;
11126 int m_iLimit;
11127 };
11128
11129
HandleMysqlSelect(NetOutputBuffer_c & tOut,BYTE & uPacketID,SearchHandler_c & tHandler)11130 bool HandleMysqlSelect ( NetOutputBuffer_c & tOut, BYTE & uPacketID, SearchHandler_c & tHandler )
11131 {
11132 // lets check all query for errors
11133 CSphString sError;
11134 CSphVector<int64_t> dAgentTimes; // dummy for error reporting
11135 ARRAY_FOREACH ( i, tHandler.m_dQueries )
11136 {
11137 CheckQuery ( tHandler.m_dQueries[i], tHandler.m_dResults[i].m_sError );
11138 if ( !tHandler.m_dResults[i].m_sError.IsEmpty() )
11139 {
11140 LogQuery ( tHandler.m_dQueries[i], tHandler.m_dResults[i], dAgentTimes );
11141 if ( sError.IsEmpty() )
11142 sError.SetSprintf ( "query %d error: %s", i, tHandler.m_dResults[i].m_sError.cstr() );
11143 else
11144 sError.SetSprintf ( "%s; query %d error: %s", sError.cstr(), i, tHandler.m_dResults[i].m_sError.cstr() );
11145 }
11146 }
11147
11148 if ( sError.Length() )
11149 {
11150 // stmt is intentionally NULL, as we did all the reporting just above
11151 SendMysqlErrorPacket ( tOut, uPacketID, NULL, sError.cstr() );
11152 return false;
11153 }
11154
11155 // actual searching
11156 tHandler.RunQueries ();
11157
11158 if ( g_bGotSigterm )
11159 {
11160 sphLogDebug ( "HandleClientMySQL: got SIGTERM, sending the packet MYSQL_ERR_SERVER_SHUTDOWN" );
11161 SendMysqlErrorPacket ( tOut, uPacketID, NULL, "Server shutdown in progress", MYSQL_ERR_SERVER_SHUTDOWN );
11162 return false;
11163 }
11164
11165 return true;
11166 }
11167
11168
SendMysqlSelectResult(NetOutputBuffer_c & tOut,BYTE & uPacketID,SqlRowBuffer_c & dRows,const AggrResult_t & tRes,bool bMoreResultsFollow)11169 void SendMysqlSelectResult ( NetOutputBuffer_c & tOut, BYTE & uPacketID, SqlRowBuffer_c & dRows, const AggrResult_t & tRes, bool bMoreResultsFollow )
11170 {
11171 if ( !tRes.m_iSuccesses )
11172 {
11173 // at this point, SELECT error logging should have been handled, so pass a NULL stmt to logger
11174 SendMysqlErrorPacket ( tOut, uPacketID++, NULL, tRes.m_sError.cstr() );
11175 return;
11176 }
11177
11178 // empty result sets just might carry the full uberschema
11179 // bummer! lets protect ourselves against that
11180 int iSchemaAttrsCount = 0;
11181 int iAttrsCount = 1;
11182 if ( tRes.m_dMatches.GetLength() )
11183 {
11184 iSchemaAttrsCount = SendGetAttrCount ( tRes.m_tSchema );
11185 iAttrsCount = iSchemaAttrsCount;
11186 if ( g_bCompatResults )
11187 iAttrsCount += 2;
11188 }
11189 if ( iAttrsCount>=251 )
11190 {
11191 // this will show up as success in query log, as the query itself was ok
11192 // but we need some kind of a notice anyway, to nail down issues based on logs only
11193 sphWarning ( "selecting more than 250 columns is not supported yet" );
11194 SendMysqlErrorPacket ( tOut, uPacketID++, NULL, "selecting more than 250 columns is not supported yet" );
11195 return;
11196 }
11197
11198 // result set header packet
11199 tOut.SendLSBDword ( ((uPacketID++)<<24) + 2 );
11200 tOut.SendByte ( BYTE(iAttrsCount) );
11201 tOut.SendByte ( 0 ); // extra
11202
11203 // field packets
11204 if ( !tRes.m_dMatches.GetLength() )
11205 {
11206 // in case there are no matches, send a dummy schema
11207 SendMysqlFieldPacket ( tOut, uPacketID++, "id", USE_64BIT ? MYSQL_COL_LONGLONG : MYSQL_COL_LONG );
11208 } else
11209 {
11210 // send result set schema
11211 if ( g_bCompatResults )
11212 {
11213 SendMysqlFieldPacket ( tOut, uPacketID++, "id", USE_64BIT ? MYSQL_COL_LONGLONG : MYSQL_COL_LONG );
11214 SendMysqlFieldPacket ( tOut, uPacketID++, "weight", MYSQL_COL_LONG );
11215 }
11216
11217 for ( int i=0; i<iSchemaAttrsCount; i++ )
11218 {
11219 const CSphColumnInfo & tCol = tRes.m_tSchema.GetAttr(i);
11220 MysqlColumnType_e eType = MYSQL_COL_STRING;
11221 if ( tCol.m_eAttrType==SPH_ATTR_INTEGER || tCol.m_eAttrType==SPH_ATTR_TIMESTAMP || tCol.m_eAttrType==SPH_ATTR_BOOL
11222 || tCol.m_eAttrType==SPH_ATTR_ORDINAL || tCol.m_eAttrType==SPH_ATTR_WORDCOUNT )
11223 eType = MYSQL_COL_LONG;
11224 if ( tCol.m_eAttrType==SPH_ATTR_FLOAT )
11225 eType = MYSQL_COL_FLOAT;
11226 if ( tCol.m_eAttrType==SPH_ATTR_BIGINT )
11227 eType = MYSQL_COL_LONGLONG;
11228 if ( tCol.m_eAttrType==SPH_ATTR_STRING )
11229 eType = MYSQL_COL_STRING;
11230 SendMysqlFieldPacket ( tOut, uPacketID++, tCol.m_sName.cstr(), eType );
11231 }
11232 }
11233
11234 // eof packet
11235 BYTE iWarns = ( !tRes.m_sWarning.IsEmpty() ) ? 1 : 0;
11236 SendMysqlEofPacket ( tOut, uPacketID++, iWarns, bMoreResultsFollow );
11237
11238 // rows
11239 dRows.Reset();
11240
11241 for ( int iMatch = tRes.m_iOffset; iMatch < tRes.m_iOffset + tRes.m_iCount; iMatch++ )
11242 {
11243 const CSphMatch & tMatch = tRes.m_dMatches [ iMatch ];
11244
11245 if ( g_bCompatResults )
11246 {
11247 dRows.PutNumeric<SphDocID_t> ( DOCID_FMT, tMatch.m_iDocID );
11248 dRows.PutNumeric ( "%u", tMatch.m_iWeight );
11249 }
11250
11251 const CSphSchema & tSchema = tRes.m_tSchema;
11252 for ( int i=0; i<iSchemaAttrsCount; i++ )
11253 {
11254 CSphAttrLocator tLoc = tSchema.GetAttr(i).m_tLocator;
11255 ESphAttr eAttrType = tSchema.GetAttr(i).m_eAttrType;
11256
11257 switch ( eAttrType )
11258 {
11259 case SPH_ATTR_INTEGER:
11260 case SPH_ATTR_TIMESTAMP:
11261 case SPH_ATTR_BOOL:
11262 case SPH_ATTR_BIGINT:
11263 case SPH_ATTR_ORDINAL:
11264 case SPH_ATTR_WORDCOUNT:
11265 if ( eAttrType==SPH_ATTR_BIGINT )
11266 dRows.PutNumeric<SphAttr_t> ( INT64_FMT, tMatch.GetAttr(tLoc) );
11267 else
11268 dRows.PutNumeric<DWORD> ( "%u", (DWORD)tMatch.GetAttr(tLoc) );
11269 break;
11270
11271 case SPH_ATTR_FLOAT:
11272 dRows.PutNumeric ( "%f", tMatch.GetAttrFloat(tLoc) );
11273 break;
11274
11275 case SPH_ATTR_INT64SET:
11276 case SPH_ATTR_UINT32SET:
11277 {
11278 int iLenOff = dRows.Length();
11279 dRows.Reserve ( 4 );
11280 dRows.IncPtr ( 4 );
11281
11282 assert ( tMatch.GetAttr ( tLoc )==0 || tRes.m_dTag2Pools [ tMatch.m_iTag ].m_pMva );
11283 const DWORD * pValues = tMatch.GetAttrMVA ( tLoc, tRes.m_dTag2Pools [ tMatch.m_iTag ].m_pMva );
11284 if ( pValues )
11285 {
11286 DWORD nValues = *pValues++;
11287 assert ( eAttrType==SPH_ATTR_UINT32SET || ( nValues%2 )==0 );
11288 if ( eAttrType==SPH_ATTR_UINT32SET )
11289 {
11290 while ( nValues-- )
11291 {
11292 dRows.Reserve ( SPH_MAX_NUMERIC_STR );
11293 int iLen = snprintf ( dRows.Get(), SPH_MAX_NUMERIC_STR, nValues>0 ? "%u," : "%u", *pValues++ );
11294 dRows.IncPtr ( iLen );
11295 }
11296 } else
11297 {
11298 for ( ; nValues; nValues-=2, pValues+=2 )
11299 {
11300 int64_t iVal = MVA_UPSIZE ( pValues );
11301 dRows.Reserve ( SPH_MAX_NUMERIC_STR );
11302 int iLen = snprintf ( dRows.Get(), SPH_MAX_NUMERIC_STR, nValues>2 ? INT64_FMT"," : INT64_FMT, iVal );
11303 dRows.IncPtr ( iLen );
11304 }
11305 }
11306 }
11307
11308 // manually pack length, forcibly into exactly 3 bytes
11309 int iLen = dRows.Length()-iLenOff-4;
11310 char * pLen = dRows.Off ( iLenOff );
11311 pLen[0] = (BYTE)0xfd;
11312 pLen[1] = (BYTE)( iLen & 0xff );
11313 pLen[2] = (BYTE)( ( iLen>>8 ) & 0xff );
11314 pLen[3] = (BYTE)( ( iLen>>16 ) & 0xff );
11315 break;
11316 }
11317
11318 case SPH_ATTR_STRING:
11319 {
11320 const BYTE * pStrings = tRes.m_dTag2Pools [ tMatch.m_iTag ].m_pStrings;
11321
11322 // get that string
11323 const BYTE * pStr = NULL;
11324 int iLen = 0;
11325
11326 DWORD uOffset = (DWORD) tMatch.GetAttr ( tLoc );
11327 if ( uOffset )
11328 {
11329 assert ( pStrings );
11330 iLen = sphUnpackStr ( pStrings+uOffset, &pStr );
11331 }
11332
11333 // send length
11334 dRows.Reserve ( iLen+4 );
11335 char * pOutStr = (char*)MysqlPack ( dRows.Get(), iLen );
11336
11337 // send string data
11338 if ( iLen )
11339 memcpy ( pOutStr, pStr, iLen );
11340
11341 dRows.IncPtr ( pOutStr-dRows.Get()+iLen );
11342 break;
11343 }
11344
11345 default:
11346 char * pDef = dRows.Reserve ( 2 );
11347 pDef[0] = 1;
11348 pDef[1] = '-';
11349 dRows.IncPtr ( 2 );
11350 break;
11351 }
11352 }
11353
11354 tOut.SendLSBDword ( ((uPacketID++)<<24) + ( dRows.Length() ) );
11355 tOut.SendBytes ( dRows.Off ( 0 ), dRows.Length() );
11356 dRows.Reset();
11357 }
11358
11359 // eof packet
11360 SendMysqlEofPacket ( tOut, uPacketID++, iWarns, bMoreResultsFollow );
11361 }
11362
11363
HandleMysqlWarning(NetOutputBuffer_c & tOut,BYTE & uPacketID,const CSphQueryResultMeta & tLastMeta,SqlRowBuffer_c & dRows,bool bMoreResultsFollow)11364 void HandleMysqlWarning ( NetOutputBuffer_c & tOut, BYTE & uPacketID, const CSphQueryResultMeta & tLastMeta, SqlRowBuffer_c & dRows, bool bMoreResultsFollow )
11365 {
11366 // can't send simple ok if there are more results to send
11367 // as it breaks order of multi-result output
11368 if ( tLastMeta.m_sWarning.IsEmpty() && !bMoreResultsFollow )
11369 {
11370 SendMysqlOkPacket ( tOut, uPacketID );
11371 return;
11372 }
11373
11374 // result set header packet
11375 tOut.SendLSBDword ( ((uPacketID++)<<24) + 2 );
11376 tOut.SendByte ( 3 ); // field count (level+code+message)
11377 tOut.SendByte ( 0 ); // extra
11378
11379 // field packets
11380 SendMysqlFieldPacket ( tOut, uPacketID++, "Level", MYSQL_COL_STRING );
11381 SendMysqlFieldPacket ( tOut, uPacketID++, "Code", MYSQL_COL_DECIMAL );
11382 SendMysqlFieldPacket ( tOut, uPacketID++, "Message", MYSQL_COL_STRING );
11383 SendMysqlEofPacket ( tOut, uPacketID++, 0, bMoreResultsFollow );
11384
11385 // row
11386 dRows.Reset();
11387 dRows.PutString ( "warning" );
11388 dRows.PutString ( "1000" );
11389 dRows.PutString ( tLastMeta.m_sWarning.cstr() );
11390
11391 tOut.SendLSBDword ( ((uPacketID++)<<24) + ( dRows.Length() ) );
11392 tOut.SendBytes ( dRows.Off ( 0 ), dRows.Length() );
11393 dRows.Reset();
11394
11395 // cleanup
11396 SendMysqlEofPacket ( tOut, uPacketID++, 0, bMoreResultsFollow );
11397 }
11398
11399
HandleMysqlMeta(NetOutputBuffer_c & tOut,BYTE & uPacketID,const CSphQueryResultMeta & tLastMeta,SqlRowBuffer_c & dRows,bool bStatus,bool bMoreResultsFollow)11400 void HandleMysqlMeta ( NetOutputBuffer_c & tOut, BYTE & uPacketID, const CSphQueryResultMeta & tLastMeta, SqlRowBuffer_c & dRows, bool bStatus, bool bMoreResultsFollow )
11401 {
11402 CSphVector<CSphString> dStatus;
11403
11404 if ( bStatus )
11405 BuildStatus ( dStatus );
11406 else
11407 BuildMeta ( dStatus, tLastMeta );
11408
11409 // result set header packet
11410 tOut.SendLSBDword ( ((uPacketID++)<<24) + 2 );
11411 tOut.SendByte ( 2 ); // field count (level+code+message)
11412 tOut.SendByte ( 0 ); // extra
11413
11414 // field packets
11415 SendMysqlFieldPacket ( tOut, uPacketID++, "Variable_name", MYSQL_COL_STRING );
11416 SendMysqlFieldPacket ( tOut, uPacketID++, "Value", MYSQL_COL_STRING );
11417 SendMysqlEofPacket ( tOut, uPacketID++, 0, bMoreResultsFollow );
11418
11419 // send rows
11420 dRows.Reset();
11421
11422 for ( int iRow=0; iRow<dStatus.GetLength(); iRow+=2 )
11423 {
11424 dRows.PutString ( dStatus[iRow+0].cstr() );
11425 dRows.PutString ( dStatus[iRow+1].cstr() );
11426
11427 tOut.SendLSBDword ( ((uPacketID++)<<24) + ( dRows.Length() ) );
11428 tOut.SendBytes ( dRows.Off ( 0 ), dRows.Length() );
11429 dRows.Reset();
11430 }
11431
11432 // cleanup
11433 SendMysqlEofPacket ( tOut, uPacketID++, 0, bMoreResultsFollow );
11434 }
11435
11436
HandleMysqlDelete(NetOutputBuffer_c & tOut,BYTE & uPacketID,const SqlStmt_t & tStmt,bool bCommit)11437 void HandleMysqlDelete ( NetOutputBuffer_c & tOut, BYTE & uPacketID, const SqlStmt_t & tStmt, bool bCommit )
11438 {
11439 MEMORY ( SPH_MEM_DELETE_SQL );
11440
11441 CSphString sError;
11442
11443 const ServedIndex_t * pServed = g_pIndexes->GetRlockedEntry ( tStmt.m_sIndex );
11444 if ( !pServed )
11445 {
11446 sError.SetSprintf ( "no such index '%s'", tStmt.m_sIndex.cstr() );
11447 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, sError.cstr() );
11448 return;
11449 }
11450
11451 if ( !pServed->m_bRT || !pServed->m_bEnabled )
11452 {
11453 pServed->Unlock();
11454 sError.SetSprintf ( "index '%s' does not support DELETE (enabled=%d)", tStmt.m_sIndex.cstr(), pServed->m_bEnabled );
11455 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, sError.cstr() );
11456 return;
11457 }
11458
11459 ISphRtIndex * pIndex = static_cast<ISphRtIndex *> ( pServed->m_pIndex );
11460
11461 if ( !pIndex->DeleteDocument ( tStmt.m_dDeleteIds.Begin(), tStmt.m_dDeleteIds.GetLength(), sError ) )
11462 {
11463 pServed->Unlock();
11464 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, sError.cstr() );
11465 return;
11466 }
11467
11468 if ( bCommit )
11469 pIndex->Commit ();
11470
11471 pServed->Unlock();
11472
11473 SendMysqlOkPacket ( tOut, uPacketID ); // FIXME? affected rows
11474 }
11475
11476
HandleMysqlMultiStmt(NetOutputBuffer_c & tOut,BYTE uPacketID,const CSphVector<SqlStmt_t> & dStmt,CSphQueryResultMeta & tLastMeta,SqlRowBuffer_c & dRows,ThdDesc_t * pThd,const CSphString & sWarning)11477 void HandleMysqlMultiStmt ( NetOutputBuffer_c & tOut, BYTE uPacketID, const CSphVector<SqlStmt_t> & dStmt, CSphQueryResultMeta & tLastMeta,
11478 SqlRowBuffer_c & dRows, ThdDesc_t * pThd, const CSphString& sWarning )
11479 {
11480 // select count
11481 int iSelect = 0;
11482 ARRAY_FOREACH ( i, dStmt )
11483 if ( dStmt[i].m_eStmt==STMT_SELECT )
11484 iSelect++;
11485
11486 CSphQueryResultMeta tPrevMeta = tLastMeta;
11487
11488 if ( pThd )
11489 pThd->m_sCommand = g_dSqlStmts[STMT_SELECT];
11490
11491 StatCountCommand ( SEARCHD_COMMAND_SEARCH, iSelect );
11492
11493 // setup query for searching
11494 SearchHandler_c tHandler ( iSelect, true );
11495 iSelect = 0;
11496 ARRAY_FOREACH ( i, dStmt )
11497 {
11498 if ( dStmt[i].m_eStmt==STMT_SELECT )
11499 tHandler.m_dQueries[iSelect++] = dStmt[i].m_tQuery;
11500 }
11501
11502 // do search
11503 bool bSearchOK = true;
11504 if ( iSelect )
11505 {
11506 bSearchOK = HandleMysqlSelect ( tOut, uPacketID, tHandler );
11507
11508 // save meta for SHOW *
11509 tLastMeta = tHandler.m_dResults.Last();
11510 }
11511
11512 if ( !bSearchOK )
11513 return;
11514
11515 // send multi-result set
11516 iSelect = 0;
11517 ARRAY_FOREACH ( i, dStmt )
11518 {
11519 SqlStmt_e eStmt = dStmt[i].m_eStmt;
11520
11521 THD_STATE ( THD_QUERY );
11522 if ( pThd )
11523 pThd->m_sCommand = g_dSqlStmts[eStmt];
11524
11525 const CSphQueryResultMeta & tMeta = iSelect-1>=0 ? tHandler.m_dResults[iSelect-1] : tPrevMeta;
11526 bool bMoreResultsFollow = (i+1)<dStmt.GetLength();
11527
11528 if ( eStmt==STMT_SELECT )
11529 {
11530 AggrResult_t & tRes = tHandler.m_dResults[iSelect++];
11531 if ( !sWarning.IsEmpty() )
11532 tRes.m_sWarning = sWarning;
11533 SendMysqlSelectResult ( tOut, uPacketID, dRows, tRes, bMoreResultsFollow );
11534 } else if ( eStmt==STMT_SHOW_WARNINGS )
11535 HandleMysqlWarning ( tOut, uPacketID, tMeta, dRows, bMoreResultsFollow );
11536 else if ( eStmt==STMT_SHOW_STATUS || eStmt==STMT_SHOW_META )
11537 HandleMysqlMeta ( tOut, uPacketID, tMeta, dRows, eStmt==STMT_SHOW_STATUS, bMoreResultsFollow );
11538
11539 if ( g_bGotSigterm )
11540 {
11541 sphLogDebug ( "HandleMultiStmt: got SIGTERM, sending the packet MYSQL_ERR_SERVER_SHUTDOWN" );
11542 SendMysqlErrorPacket ( tOut, uPacketID, NULL, "Server shutdown in progress", MYSQL_ERR_SERVER_SHUTDOWN );
11543 return;
11544 }
11545 }
11546 }
11547
11548
11549 struct SessionVars_t
11550 {
11551 bool m_bAutoCommit;
11552 bool m_bInTransaction;
11553 ESphCollation m_eCollation;
11554
SessionVars_tSessionVars_t11555 SessionVars_t ()
11556 : m_bAutoCommit ( true )
11557 , m_bInTransaction ( false )
11558 , m_eCollation ( g_eCollation )
11559 {}
11560 };
11561
sphCollationFromName(const CSphString & sName,CSphString * pError)11562 static ESphCollation sphCollationFromName ( const CSphString & sName, CSphString * pError )
11563 {
11564 assert ( pError );
11565
11566 // FIXME! replace with a hash lookup?
11567 if ( sName=="libc_ci" )
11568 return SPH_COLLATION_LIBC_CI;
11569 else if ( sName=="libc_cs" )
11570 return SPH_COLLATION_LIBC_CS;
11571 else if ( sName=="utf8_general_ci" )
11572 return SPH_COLLATION_UTF8_GENERAL_CI;
11573 else if ( sName=="binary" )
11574 return SPH_COLLATION_BINARY;
11575
11576 pError->SetSprintf ( "Unknown collation: '%s'", sName.cstr() );
11577 return SPH_COLLATION_DEFAULT;
11578 }
11579
11580
HandleMysqlSet(NetOutputBuffer_c & tOut,BYTE & uPacketID,SqlStmt_t & tStmt,SessionVars_t & tVars)11581 void HandleMysqlSet ( NetOutputBuffer_c & tOut, BYTE & uPacketID, SqlStmt_t & tStmt, SessionVars_t & tVars )
11582 {
11583 MEMORY ( SPH_MEM_COMMIT_SET_SQL );
11584 CSphString sError;
11585
11586 tStmt.m_sSetName.ToLower();
11587 switch ( tStmt.m_eSet )
11588 {
11589 case SET_LOCAL:
11590 if ( tStmt.m_sSetName=="autocommit" )
11591 {
11592 // per-session AUTOCOMMIT
11593 tVars.m_bAutoCommit = ( tStmt.m_iSetValue!=0 );
11594 tVars.m_bInTransaction = false;
11595
11596 // commit all pending changes
11597 if ( tVars.m_bAutoCommit )
11598 {
11599 ISphRtIndex * pIndex = sphGetCurrentIndexRT();
11600 if ( pIndex )
11601 pIndex->Commit();
11602 }
11603 } else if ( tStmt.m_sSetName=="collation_connection" )
11604 {
11605 // per-session COLLATION_CONNECTION
11606 CSphString & sVal = tStmt.m_sSetValue;
11607 sVal.ToLower();
11608
11609 tVars.m_eCollation = sphCollationFromName ( sVal, &sError );
11610 if ( !sError.IsEmpty() )
11611 {
11612 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, sError.cstr() );
11613 return;
11614 }
11615 } else if ( tStmt.m_sSetName=="character_set_results"
11616 || tStmt.m_sSetName=="sql_auto_is_null"
11617 || tStmt.m_sSetName=="sql_mode" )
11618 {
11619 // per-session CHARACTER_SET_RESULTS et al; just ignore for now
11620
11621 } else
11622 {
11623 // unknown variable, return error
11624 sError.SetSprintf ( "Unknown session variable '%s' in SET statement", tStmt.m_sSetName.cstr() );
11625 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, sError.cstr() );
11626 return;
11627 }
11628 break;
11629
11630 case SET_GLOBAL_UVAR:
11631 {
11632 // global user variable
11633 if ( g_eWorkers!=MPM_THREADS )
11634 {
11635 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, "SET GLOBAL currently requires workers=threads" );
11636 return;
11637 }
11638
11639 // INT_SET type must be sorted
11640 tStmt.m_dSetValues.Sort();
11641
11642 // create or update the variable
11643 g_tUservarsMutex.Lock();
11644 Uservar_t * pVar = g_hUservars ( tStmt.m_sSetName );
11645 if ( pVar )
11646 {
11647 // variable exists, release previous value
11648 // actual destruction of the value (aka data) might happen later
11649 // as the concurrent queries might still be using and holding that data
11650 // from here, the old value becomes nameless, though
11651 assert ( pVar->m_eType==USERVAR_INT_SET );
11652 assert ( pVar->m_pVal );
11653 pVar->m_pVal->Release();
11654 pVar->m_pVal = NULL;
11655 } else
11656 {
11657 // create a shiny new variable
11658 Uservar_t tVar;
11659 g_hUservars.Add ( tVar, tStmt.m_sSetName );
11660 pVar = g_hUservars ( tStmt.m_sSetName );
11661 }
11662
11663 // swap in the new value
11664 assert ( pVar );
11665 assert ( !pVar->m_pVal );
11666 pVar->m_eType = USERVAR_INT_SET;
11667 pVar->m_pVal = new UservarIntSet_c();
11668 pVar->m_pVal->SwapData ( tStmt.m_dSetValues );
11669 g_tUservarsMutex.Unlock();
11670 break;
11671 }
11672
11673 case SET_GLOBAL_SVAR:
11674 // global server variable
11675 if ( g_eWorkers!=MPM_THREADS )
11676 {
11677 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, "SET GLOBAL currently requires workers=threads" );
11678 return;
11679 }
11680
11681 if ( tStmt.m_sSetName=="query_log_format" )
11682 {
11683 if ( tStmt.m_sSetValue=="plain" )
11684 g_eLogFormat = LOG_FORMAT_PLAIN;
11685 else if ( tStmt.m_sSetValue=="sphinxql" )
11686 g_eLogFormat = LOG_FORMAT_SPHINXQL;
11687 else
11688 {
11689 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, "Unknown query_log_format value (must be plain or sphinxql)" );
11690 return;
11691 }
11692 } else if ( tStmt.m_sSetName=="log_level" )
11693 {
11694 if ( tStmt.m_sSetValue=="info" )
11695 g_eLogLevel = SPH_LOG_INFO;
11696 else if ( tStmt.m_sSetValue=="debug" )
11697 g_eLogLevel = SPH_LOG_DEBUG;
11698 else if ( tStmt.m_sSetValue=="debugv" )
11699 g_eLogLevel = SPH_LOG_VERBOSE_DEBUG;
11700 else if ( tStmt.m_sSetValue=="debugvv" )
11701 g_eLogLevel = SPH_LOG_VERY_VERBOSE_DEBUG;
11702 else
11703 {
11704 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, "Unknown log_level value (must be one of info, debug, debugv, debugvv)" );
11705 return;
11706 }
11707 } else
11708 {
11709 sError.SetSprintf ( "Unknown system variable '%s'", tStmt.m_sSetName.cstr() );
11710 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, sError.cstr() );
11711 return;
11712 }
11713 break;
11714
11715 default:
11716 sError.SetSprintf ( "INTERNAL ERROR: unhandle SET mode %d", (int)tStmt.m_eSet );
11717 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, sError.cstr() );
11718 return;
11719 }
11720
11721 // it went ok
11722 SendMysqlOkPacket ( tOut, uPacketID );
11723 }
11724
11725
11726 // fwd
11727 void PreCreatePlainIndex ( ServedDesc_t & tServed, const char * sName );
11728 bool PrereadNewIndex ( ServedIndex_t & tIdx, const CSphConfigSection & hIndex, const char * szIndexName );
11729
11730
HandleMysqlAttach(const SqlStmt_t & tStmt,NetOutputBuffer_c & tOut,BYTE uPacketID)11731 void HandleMysqlAttach ( const SqlStmt_t & tStmt, NetOutputBuffer_c & tOut, BYTE uPacketID )
11732 {
11733 const CSphString & sFrom = tStmt.m_sIndex;
11734 const CSphString & sTo = tStmt.m_sSetName;
11735 CSphString sError;
11736
11737 ServedIndex_t * pFrom = g_pIndexes->GetWlockedEntry ( sFrom );
11738 const ServedIndex_t * pTo = g_pIndexes->GetRlockedEntry ( sTo );
11739
11740 if ( !pFrom || !pFrom->m_bEnabled
11741 || !pTo || !pTo->m_bEnabled
11742 || pFrom->m_bRT
11743 || !pTo->m_bRT )
11744 {
11745 if ( !pFrom || !pFrom->m_bEnabled )
11746 SendMysqlErrorPacketEx ( tOut, uPacketID, MYSQL_ERR_PARSE_ERROR, "no such index '%s'", sFrom.cstr() );
11747 else if ( !pTo || !pTo->m_bEnabled )
11748 SendMysqlErrorPacketEx ( tOut, uPacketID, MYSQL_ERR_PARSE_ERROR, "no such index '%s'", sTo.cstr() );
11749 else if ( pFrom->m_bRT )
11750 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, "1st argument to ATTACH must be a plain index" );
11751 else if ( pTo->m_bRT )
11752 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, "2nd argument to ATTACH must be a RT index" );
11753
11754 if ( pFrom )
11755 pFrom->Unlock();
11756 if ( pTo )
11757 pTo->Unlock();
11758 return;
11759 }
11760
11761 ISphRtIndex * pRtTo = dynamic_cast<ISphRtIndex*> ( pTo->m_pIndex );
11762 assert ( pRtTo );
11763
11764 if ( !pRtTo->AttachDiskIndex ( pFrom->m_pIndex, sError ) )
11765 {
11766 pFrom->Unlock();
11767 pTo->Unlock();
11768 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, sError.cstr() );
11769 return;
11770 }
11771
11772 pTo->Unlock();
11773
11774 // after a successfull Attach() RT index owns it
11775 // so we need to create dummy disk index until further notice
11776 pFrom->m_pIndex = NULL;
11777 pFrom->m_bEnabled = false;
11778 PreCreatePlainIndex ( *pFrom, sFrom.cstr() );
11779 if ( pFrom->m_pIndex )
11780 pFrom->m_bEnabled = PrereadNewIndex ( *pFrom, g_pCfg.m_tConf["index"][sFrom], sFrom.cstr() );
11781 pFrom->Unlock();
11782
11783 SendMysqlOkPacket ( tOut, uPacketID );
11784 }
11785
11786
HandleMysqlFlush(const SqlStmt_t & tStmt,NetOutputBuffer_c & tOut,BYTE uPacketID)11787 void HandleMysqlFlush ( const SqlStmt_t & tStmt, NetOutputBuffer_c & tOut, BYTE uPacketID )
11788 {
11789 CSphString sError;
11790 const ServedIndex_t * pIndex = g_pIndexes->GetRlockedEntry ( tStmt.m_sIndex );
11791
11792 if ( !pIndex || !pIndex->m_bEnabled || !pIndex->m_bRT )
11793 {
11794 if ( pIndex )
11795 pIndex->Unlock();
11796 SendMysqlErrorPacket ( tOut, uPacketID, tStmt.m_sStmt, "FLUSH RTINDEX requires an existing RT index" );
11797 return;
11798 }
11799
11800
11801 ISphRtIndex * pRt = dynamic_cast<ISphRtIndex*> ( pIndex->m_pIndex );
11802 assert ( pRt );
11803
11804 pRt->ForceRamFlush();
11805 pIndex->Unlock();
11806 SendMysqlOkPacket ( tOut, uPacketID );
11807 }
11808
11809
SendMysqlPair(NetOutputBuffer_c & tOut,BYTE & uPacketID,SqlRowBuffer_c & dRows,const char * sKey,const char * sValue)11810 void SendMysqlPair ( NetOutputBuffer_c & tOut, BYTE & uPacketID, SqlRowBuffer_c & dRows, const char * sKey, const char * sValue )
11811 {
11812 dRows.PutString ( sKey );
11813 dRows.PutString ( sValue );
11814 tOut.SendLSBDword ( ((uPacketID++)<<24) + ( dRows.Length() ) );
11815 tOut.SendBytes ( dRows.Off ( 0 ), dRows.Length() );
11816 dRows.Reset();
11817 }
11818
11819
sphCollationToName(ESphCollation eColl)11820 const char * sphCollationToName ( ESphCollation eColl )
11821 {
11822 switch ( eColl )
11823 {
11824 case SPH_COLLATION_LIBC_CI: return "libc_ci";
11825 case SPH_COLLATION_LIBC_CS: return "libc_cs";
11826 case SPH_COLLATION_UTF8_GENERAL_CI: return "utf8_general_ci";
11827 case SPH_COLLATION_BINARY: return "binary";
11828 default: return "unknown";
11829 }
11830 }
11831
11832
LogLevelName(ESphLogLevel eLevel)11833 static const char * LogLevelName ( ESphLogLevel eLevel )
11834 {
11835 switch ( eLevel )
11836 {
11837 case SPH_LOG_FATAL: return "fatal";
11838 case SPH_LOG_WARNING: return "warning";
11839 case SPH_LOG_INFO: return "info";
11840 case SPH_LOG_DEBUG: return "debug";
11841 case SPH_LOG_VERBOSE_DEBUG: return "debugv";
11842 case SPH_LOG_VERY_VERBOSE_DEBUG: return "debugvv";
11843 default: return "unknown";
11844 }
11845 }
11846
11847
HandleMysqlShowVariables(const SqlStmt_t &,NetOutputBuffer_c & tOut,BYTE uPacketID,SqlRowBuffer_c & dRows,SessionVars_t & tVars)11848 void HandleMysqlShowVariables ( const SqlStmt_t &, NetOutputBuffer_c & tOut, BYTE uPacketID, SqlRowBuffer_c & dRows, SessionVars_t & tVars )
11849 {
11850 // result set header packet
11851 tOut.SendLSBDword ( ((uPacketID++)<<24) + 2 );
11852 tOut.SendByte ( 2 ); // field count (level+code+message)
11853 tOut.SendByte ( 0 ); // extra
11854
11855 // field packets
11856 SendMysqlFieldPacket ( tOut, uPacketID++, "Variable_name", MYSQL_COL_STRING );
11857 SendMysqlFieldPacket ( tOut, uPacketID++, "Value", MYSQL_COL_STRING );
11858 SendMysqlEofPacket ( tOut, uPacketID++, 0 );
11859
11860 // send rows
11861 dRows.Reset();
11862
11863 // sessions vars
11864 SendMysqlPair ( tOut, uPacketID, dRows, "autocommit", tVars.m_bAutoCommit ? "1" : "0" );
11865 SendMysqlPair ( tOut, uPacketID, dRows, "collation_connection", sphCollationToName ( tVars.m_eCollation ) );
11866
11867 // server vars
11868 SendMysqlPair ( tOut, uPacketID, dRows, "query_log_format", g_eLogFormat==LOG_FORMAT_PLAIN ? "plain" : "sphinxql" );
11869 SendMysqlPair ( tOut, uPacketID, dRows, "log_level", LogLevelName ( g_eLogLevel ) );
11870
11871 // cleanup
11872 SendMysqlEofPacket ( tOut, uPacketID++, 0 );
11873 }
11874
11875
11876 class CSphinxqlSession : public ISphNoncopyable
11877 {
11878 CSphString & m_sError;
11879
11880 public:
11881 CSphQueryResultMeta m_tLastMeta;
11882 SessionVars_t m_tVars;
11883
11884 public:
CSphinxqlSession(CSphString & sError)11885 explicit CSphinxqlSession ( CSphString & sError ) :
11886 m_sError ( sError )
11887 {}
11888
11889 // just execute one sphinxql statement
Execute(const CSphString & sQuery,NetOutputBuffer_c & tOut,BYTE & uPacketID,ThdDesc_t * pThd=NULL)11890 void Execute ( const CSphString & sQuery, NetOutputBuffer_c & tOut, BYTE & uPacketID, ThdDesc_t * pThd=NULL )
11891 {
11892 // set on query guard
11893 CrashQuery_t tCrashQuery;
11894 tCrashQuery.m_pQuery = (const BYTE *)sQuery.cstr();
11895 tCrashQuery.m_iSize = sQuery.Length();
11896 tCrashQuery.m_bMySQL = true;
11897 SphCrashLogger_c::SetLastQuery ( tCrashQuery );
11898
11899 // parse SQL query
11900 CSphVector<SqlStmt_t> dStmt;
11901 bool bParsedOK = ParseSqlQuery ( sQuery, dStmt, m_sError, m_tVars.m_eCollation );
11902
11903 SqlStmt_e eStmt = STMT_PARSE_ERROR;
11904 if ( bParsedOK )
11905 {
11906 eStmt = dStmt[0].m_eStmt;
11907 dStmt[0].m_sStmt = sQuery.cstr();
11908 }
11909
11910 SqlStmt_t * pStmt = dStmt.Begin();
11911 assert ( !bParsedOK || pStmt );
11912
11913 if ( pThd )
11914 pThd->m_sCommand = g_dSqlStmts[eStmt];
11915 THD_STATE ( THD_QUERY );
11916
11917 SqlRowBuffer_c dRows;
11918
11919 // handle multi SQL query
11920 if ( bParsedOK && dStmt.GetLength()>1 )
11921 {
11922 m_sError = "";
11923 HandleMysqlMultiStmt ( tOut, uPacketID, dStmt, m_tLastMeta, dRows, pThd, m_sError );
11924 return;
11925 }
11926
11927 // handle SQL query
11928 switch ( eStmt )
11929 {
11930 case STMT_PARSE_ERROR:
11931 m_tLastMeta = CSphQueryResultMeta();
11932 m_tLastMeta.m_sError = m_sError;
11933 m_tLastMeta.m_sWarning = "";
11934 SendMysqlErrorPacket ( tOut, uPacketID, sQuery.cstr(), m_sError.cstr() );
11935 return;
11936
11937 case STMT_SELECT:
11938 {
11939 MEMORY ( SPH_MEM_SELECT_SQL );
11940
11941 StatCountCommand ( SEARCHD_COMMAND_SEARCH );
11942 SearchHandler_c tHandler ( 1, true );
11943 tHandler.m_dQueries[0] = dStmt.Begin()->m_tQuery;
11944
11945 if ( HandleMysqlSelect ( tOut, uPacketID, tHandler ) )
11946 {
11947 // query just completed ok; reset out error message
11948 m_sError = "";
11949 AggrResult_t & tLast = tHandler.m_dResults.Last();
11950 SendMysqlSelectResult ( tOut, uPacketID, dRows, tLast, false );
11951 }
11952
11953 // save meta for SHOW META
11954 m_tLastMeta = tHandler.m_dResults.Last();
11955 return;
11956 }
11957 case STMT_SHOW_WARNINGS:
11958 HandleMysqlWarning ( tOut, uPacketID, m_tLastMeta, dRows, false );
11959 return;
11960
11961 case STMT_SHOW_STATUS:
11962 case STMT_SHOW_META:
11963 if ( eStmt==STMT_SHOW_STATUS )
11964 {
11965 StatCountCommand ( SEARCHD_COMMAND_STATUS );
11966 }
11967 HandleMysqlMeta ( tOut, uPacketID, m_tLastMeta, dRows, eStmt==STMT_SHOW_STATUS, false );
11968 return;
11969
11970 case STMT_INSERT:
11971 case STMT_REPLACE:
11972 HandleMysqlInsert ( *pStmt, tOut, uPacketID, eStmt==STMT_REPLACE, m_tVars.m_bAutoCommit && !m_tVars.m_bInTransaction );
11973 return;
11974
11975 case STMT_DELETE:
11976 HandleMysqlDelete ( tOut, uPacketID, *pStmt, m_tVars.m_bAutoCommit && !m_tVars.m_bInTransaction );
11977 return;
11978
11979 case STMT_SET:
11980 HandleMysqlSet ( tOut, uPacketID, *pStmt, m_tVars );
11981 return;
11982
11983 case STMT_BEGIN:
11984 {
11985 MEMORY ( SPH_MEM_COMMIT_BEGIN_SQL );
11986
11987 m_tVars.m_bInTransaction = true;
11988 ISphRtIndex * pIndex = sphGetCurrentIndexRT();
11989 if ( pIndex )
11990 pIndex->Commit();
11991 SendMysqlOkPacket ( tOut, uPacketID );
11992 return;
11993 }
11994 case STMT_COMMIT:
11995 case STMT_ROLLBACK:
11996 {
11997 MEMORY ( SPH_MEM_COMMIT_SQL );
11998
11999 m_tVars.m_bInTransaction = false;
12000 ISphRtIndex * pIndex = sphGetCurrentIndexRT();
12001 if ( pIndex )
12002 {
12003 if ( eStmt==STMT_COMMIT )
12004 pIndex->Commit();
12005 else
12006 pIndex->RollBack();
12007 }
12008 SendMysqlOkPacket ( tOut, uPacketID );
12009 return;
12010 }
12011 case STMT_CALL:
12012 pStmt->m_sCallProc.ToUpper();
12013 if ( pStmt->m_sCallProc=="SNIPPETS" )
12014 {
12015 StatCountCommand ( SEARCHD_COMMAND_EXCERPT );
12016 HandleMysqlCallSnippets ( tOut, uPacketID, *pStmt );
12017 } else if ( pStmt->m_sCallProc=="KEYWORDS" )
12018 {
12019 StatCountCommand ( SEARCHD_COMMAND_KEYWORDS );
12020 HandleMysqlCallKeywords ( tOut, uPacketID, *pStmt );
12021 } else
12022 {
12023 m_sError.SetSprintf ( "no such builtin procedure %s", pStmt->m_sCallProc.cstr() );
12024 SendMysqlErrorPacket ( tOut, uPacketID, sQuery.cstr(), m_sError.cstr() );
12025 }
12026 return;
12027
12028 case STMT_DESC:
12029 HandleMysqlDescribe ( tOut, uPacketID, *pStmt );
12030 return;
12031
12032 case STMT_SHOW_TABLES:
12033 HandleMysqlShowTables ( tOut, uPacketID );
12034 return;
12035
12036 case STMT_UPDATE:
12037 StatCountCommand ( SEARCHD_COMMAND_UPDATE );
12038 HandleMysqlUpdate ( tOut, uPacketID, *pStmt, sQuery, m_tVars.m_bAutoCommit && !m_tVars.m_bInTransaction );
12039 return;
12040
12041 case STMT_DUMMY:
12042 SendMysqlOkPacket ( tOut, uPacketID );
12043 return;
12044
12045 case STMT_CREATE_FUNC:
12046 if ( !sphUDFCreate ( pStmt->m_sUdfLib.cstr(), pStmt->m_sUdfName.cstr(), pStmt->m_eUdfType, m_sError ) )
12047 SendMysqlErrorPacket ( tOut, uPacketID, sQuery.cstr(), m_sError.cstr() );
12048 else
12049 SendMysqlOkPacket ( tOut, uPacketID );
12050 return;
12051
12052 case STMT_DROP_FUNC:
12053 if ( !sphUDFDrop ( pStmt->m_sUdfName.cstr(), m_sError ) )
12054 SendMysqlErrorPacket ( tOut, uPacketID, sQuery.cstr(), m_sError.cstr() );
12055 else
12056 SendMysqlOkPacket ( tOut, uPacketID );
12057 return;
12058
12059 case STMT_ATTACH_INDEX:
12060 HandleMysqlAttach ( *pStmt, tOut, uPacketID );
12061 return;
12062
12063 case STMT_FLUSH_RTINDEX:
12064 HandleMysqlFlush ( *pStmt, tOut, uPacketID );
12065 return;
12066
12067 case STMT_SHOW_VARIABLES:
12068 HandleMysqlShowVariables ( *pStmt, tOut, uPacketID, dRows, m_tVars );
12069 return;
12070
12071 default:
12072 m_sError.SetSprintf ( "internal error: unhandled statement type (value=%d)", eStmt );
12073 SendMysqlErrorPacket ( tOut, uPacketID, sQuery.cstr(), m_sError.cstr() );
12074 return;
12075 } // switch
12076 }
12077 };
12078
12079
12080 /// sphinxql command over API
HandleCommandSphinxql(int iSock,int iVer,InputBuffer_c & tReq)12081 void HandleCommandSphinxql ( int iSock, int iVer, InputBuffer_c & tReq )
12082 {
12083 if ( !CheckCommandVersion ( iVer, VER_COMMAND_SPHINXQL, tReq ) )
12084 return;
12085
12086 // parse request
12087 CSphString sCommand = tReq.GetString ();
12088
12089 NetOutputBuffer_c tOut ( iSock );
12090 BYTE uDummy = 0;
12091 CSphString sError;
12092
12093 // todo: move upper, if the session variables are also necessary in API access mode.
12094 CSphinxqlSession tSession ( sError );
12095
12096 tOut.Flush();
12097 tOut.SendWord ( SEARCHD_OK );
12098 tOut.SendWord ( VER_COMMAND_SPHINXQL );
12099
12100 // assume that the whole answer could fit in output buffer without flush.
12101 // Otherwise the error will be fired.
12102 // SEARCHD_ERROR + strlen (32) + the message
12103 tOut.FreezeBlock ( "\x01\x00\x20\x00\x00\x00The output buffer is overloaded.", 38 );
12104 tSession.Execute ( sCommand, tOut, uDummy );
12105 tOut.Flush ( true );
12106 }
12107
12108
StatCountCommand(int iCmd,int iCount)12109 void StatCountCommand ( int iCmd, int iCount )
12110 {
12111 if ( g_pStats && iCmd>=0 && iCmd<SEARCHD_COMMAND_TOTAL )
12112 {
12113 g_tStatsMutex.Lock();
12114 g_pStats->m_iCommandCount[iCmd] += iCount;
12115 g_tStatsMutex.Unlock();
12116 }
12117 }
12118
12119
HandleClientMySQL(int iSock,const char * sClientIP,ThdDesc_t * pThd)12120 void HandleClientMySQL ( int iSock, const char * sClientIP, ThdDesc_t * pThd )
12121 {
12122 MEMORY ( SPH_MEM_HANDLE_SQL );
12123 THD_STATE ( THD_HANDSHAKE );
12124
12125 const int INTERACTIVE_TIMEOUT = 900;
12126 NetInputBuffer_c tIn ( iSock );
12127 NetOutputBuffer_c tOut ( iSock ); // OPTIMIZE? looks like buffer size matters a lot..
12128 int64_t iCID = ( pThd ? pThd->m_iConnID : g_iConnID );
12129
12130 if ( sphSockSend ( iSock, g_sMysqlHandshake, g_iMysqlHandshake )!=g_iMysqlHandshake )
12131 {
12132 int iErrno = sphSockGetErrno ();
12133 sphWarning ( "failed to send server version (client=%s("INT64_FMT"), error: %d '%s')", sClientIP, iCID, iErrno, sphSockError ( iErrno ) );
12134 return;
12135 }
12136
12137 bool bAuthed = false;
12138 BYTE uPacketID = 1;
12139
12140 CSphString sError;
12141 CSphinxqlSession tSession ( sError ); // session variables and state
12142
12143 CSphString sQuery; // to keep data alive for SphCrashQuery_c
12144 for ( ;; )
12145 {
12146 // set off query guard
12147 CrashQuery_t tCrashQuery;
12148 tCrashQuery.m_bMySQL = true;
12149 SphCrashLogger_c::SetLastQuery ( tCrashQuery );
12150
12151 // send the packet formed on the previous cycle
12152 THD_STATE ( THD_NET_WRITE );
12153 if ( !tOut.Flush() )
12154 break;
12155
12156 // get next packet
12157 // we want interruptible calls here, so that shutdowns could be honoured
12158 THD_STATE ( THD_NET_READ );
12159 if ( !tIn.ReadFrom ( 4, INTERACTIVE_TIMEOUT, true ) )
12160 break;
12161
12162 const int MAX_PACKET_LEN = 0xffffffL; // 16777215 bytes, max low level packet size
12163 DWORD uPacketHeader = tIn.GetLSBDword ();
12164 int iPacketLen = ( uPacketHeader & MAX_PACKET_LEN );
12165 if ( !tIn.ReadFrom ( iPacketLen, INTERACTIVE_TIMEOUT, true ) )
12166 break;
12167
12168 // handle it!
12169 uPacketID = 1 + (BYTE)( uPacketHeader>>24 ); // client will expect this id
12170
12171 // handle big packets
12172 if ( iPacketLen==MAX_PACKET_LEN )
12173 {
12174 NetInputBuffer_c tIn2 ( iSock );
12175 int iAddonLen = -1;
12176 do
12177 {
12178 if ( !tIn2.ReadFrom ( 4, INTERACTIVE_TIMEOUT, true ) )
12179 break;
12180
12181 DWORD uAddon = tIn2.GetLSBDword();
12182 uPacketID = 1 + (BYTE)( uAddon>>24 );
12183 iAddonLen = ( uAddon & MAX_PACKET_LEN );
12184 if ( !tIn.ReadFrom ( iAddonLen, INTERACTIVE_TIMEOUT, true, true ) )
12185 {
12186 iAddonLen = -1;
12187 break;
12188 }
12189 iPacketLen += iAddonLen;
12190 } while ( iAddonLen==MAX_PACKET_LEN );
12191 if ( iAddonLen<0 )
12192 break;
12193 if ( iPacketLen<0 || iPacketLen>g_iMaxPacketSize )
12194 {
12195 sphWarning ( "ill-formed client request (length=%d out of bounds)", iPacketLen );
12196 break;
12197 }
12198 }
12199
12200 // handle auth packet
12201 if ( !bAuthed )
12202 {
12203 bAuthed = true;
12204 SendMysqlOkPacket ( tOut, uPacketID );
12205 continue;
12206 }
12207
12208 // get command, handle special packets
12209 const BYTE uMysqlCmd = tIn.GetByte ();
12210 if ( uMysqlCmd==MYSQL_COM_QUIT )
12211 {
12212 // client is done
12213 break;
12214
12215 } else if ( uMysqlCmd==MYSQL_COM_PING || uMysqlCmd==MYSQL_COM_INIT_DB )
12216 {
12217 // client wants a pong
12218 SendMysqlOkPacket ( tOut, uPacketID );
12219 continue;
12220
12221 } else if ( uMysqlCmd==MYSQL_COM_SET_OPTION )
12222 {
12223 // bMulti = ( tIn.GetWord()==MYSQL_OPTION_MULTI_STATEMENTS_ON ); // that's how we could double check and validate multi query
12224 // server reporting success in response to COM_SET_OPTION and COM_DEBUG
12225 SendMysqlEofPacket ( tOut, uPacketID, 0 );
12226 continue;
12227
12228 } else if ( uMysqlCmd!=MYSQL_COM_QUERY )
12229 {
12230 // default case, unknown command
12231 // (and query is handled just below)
12232 sError.SetSprintf ( "unknown command (code=%d)", uMysqlCmd );
12233 SendMysqlErrorPacket ( tOut, uPacketID, sQuery.cstr(), sError.cstr(), MYSQL_ERR_UNKNOWN_COM_ERROR );
12234 continue;
12235 }
12236
12237 // handle query packet
12238 assert ( uMysqlCmd==MYSQL_COM_QUERY );
12239 sQuery = tIn.GetRawString ( iPacketLen-1 );
12240 assert ( !tIn.GetError() );
12241 tSession.Execute ( sQuery, tOut, uPacketID, pThd );
12242 } // for (;;)
12243
12244 // set off query guard
12245 SphCrashLogger_c::SetLastQuery ( CrashQuery_t() );
12246 }
12247
12248 //////////////////////////////////////////////////////////////////////////
12249 // HANDLE-BY-LISTENER
12250 //////////////////////////////////////////////////////////////////////////
12251
HandleClient(ProtocolType_e eProto,int iSock,const char * sClientIP,ThdDesc_t * pThd)12252 void HandleClient ( ProtocolType_e eProto, int iSock, const char * sClientIP, ThdDesc_t * pThd )
12253 {
12254 switch ( eProto )
12255 {
12256 case PROTO_SPHINX: HandleClientSphinx ( iSock, sClientIP, pThd ); break;
12257 case PROTO_MYSQL41: HandleClientMySQL ( iSock, sClientIP, pThd ); break;
12258 default: assert ( 0 && "unhandled protocol type" ); break;
12259 }
12260 }
12261
12262 /////////////////////////////////////////////////////////////////////////////
12263 // INDEX ROTATION
12264 /////////////////////////////////////////////////////////////////////////////
12265
TryRename(const char * sIndex,const char * sPrefix,const char * sFromPostfix,const char * sToPostfix,bool bFatal,bool bCheckExist=true)12266 bool TryRename ( const char * sIndex, const char * sPrefix, const char * sFromPostfix, const char * sToPostfix, bool bFatal, bool bCheckExist=true )
12267 {
12268 char sFrom [ SPH_MAX_FILENAME_LEN ];
12269 char sTo [ SPH_MAX_FILENAME_LEN ];
12270
12271 snprintf ( sFrom, sizeof(sFrom), "%s%s", sPrefix, sFromPostfix );
12272 snprintf ( sTo, sizeof(sTo), "%s%s", sPrefix, sToPostfix );
12273
12274 #if USE_WINDOWS
12275 ::unlink ( sTo );
12276 #endif
12277
12278 // if there is no file we have nothing to do
12279 if ( !bCheckExist && !sphIsReadable ( sFrom ) )
12280 return true;
12281
12282 if ( rename ( sFrom, sTo ) )
12283 {
12284 if ( bFatal )
12285 {
12286 sphFatal ( "rotating index '%s': rollback rename '%s' to '%s' failed: %s",
12287 sIndex, sFrom, sTo, strerror(errno) );
12288 } else
12289 {
12290 sphWarning ( "rotating index '%s': rename '%s' to '%s' failed: %s",
12291 sIndex, sFrom, sTo, strerror(errno) );
12292 }
12293 return false;
12294 }
12295
12296 return true;
12297 }
12298
12299
HasFiles(const ServedIndex_t & tIndex,const char ** dExts)12300 bool HasFiles ( const ServedIndex_t & tIndex, const char ** dExts )
12301 {
12302 char sFile [ SPH_MAX_FILENAME_LEN ];
12303 const char * sPath = tIndex.m_sIndexPath.cstr();
12304
12305 for ( int i=0; i<EXT_COUNT; i++ )
12306 {
12307 snprintf ( sFile, sizeof(sFile), "%s%s", sPath, dExts[i] );
12308 if ( !sphIsReadable ( sFile ) )
12309 return false;
12310 }
12311
12312 return true;
12313 }
12314
12315 /// returns true if any version of the index (old or new one) has been preread
RotateIndexGreedy(ServedIndex_t & tIndex,const char * sIndex)12316 bool RotateIndexGreedy ( ServedIndex_t & tIndex, const char * sIndex )
12317 {
12318 sphLogDebug ( "RotateIndexGreedy for '%s' invoked", sIndex );
12319 char sFile [ SPH_MAX_FILENAME_LEN ];
12320 const char * sPath = tIndex.m_sIndexPath.cstr();
12321
12322
12323 for ( int i=0; i<EXT_COUNT; i++ )
12324 {
12325 snprintf ( sFile, sizeof(sFile), "%s%s", sPath, g_dNewExts[i] );
12326 if ( !sphIsReadable ( sFile ) )
12327 {
12328 if ( i>0 )
12329 {
12330 if ( tIndex.m_bOnlyNew )
12331 sphWarning ( "rotating index '%s': '%s' unreadable: %s; NOT SERVING", sIndex, sFile, strerror(errno) );
12332 else
12333 sphWarning ( "rotating index '%s': '%s' unreadable: %s; using old index", sIndex, sFile, strerror(errno) );
12334 }
12335 return false;
12336 }
12337 }
12338 sphLogDebug ( "RotateIndexGreedy: new index is readable" );
12339
12340 if ( !tIndex.m_bOnlyNew )
12341 {
12342 // rename current to old
12343 for ( int i=0; i<EXT_COUNT; i++ )
12344 {
12345 snprintf ( sFile, sizeof(sFile), "%s%s", sPath, g_dCurExts[i] );
12346 if ( !sphIsReadable ( sFile ) || TryRename ( sIndex, sPath, g_dCurExts[i], g_dOldExts[i], false ) )
12347 continue;
12348
12349 // rollback
12350 for ( int j=0; j<i; j++ )
12351 TryRename ( sIndex, sPath, g_dOldExts[j], g_dCurExts[j], true );
12352
12353 sphWarning ( "rotating index '%s': rename to .old failed; using old index", sIndex );
12354 return false;
12355 }
12356
12357 // holding the persistent MVA updates (.mvp).
12358 for ( ;; )
12359 {
12360 char sBuf [ SPH_MAX_FILENAME_LEN ];
12361 snprintf ( sBuf, sizeof(sBuf), "%s%s", sPath, g_dCurExts[EXT_MVP] );
12362
12363 CSphString sFakeError;
12364 CSphAutofile fdTest ( sBuf, SPH_O_READ, sFakeError );
12365 bool bNoMVP = ( fdTest.GetFD()<0 );
12366 fdTest.Close();
12367 if ( bNoMVP )
12368 break; ///< no file, nothing to hold
12369
12370 if ( TryRename ( sIndex, sPath, g_dCurExts[EXT_MVP], g_dOldExts[EXT_MVP], false, false ) )
12371 break;
12372
12373 // rollback
12374 for ( int j=0; j<EXT_COUNT; j++ )
12375 TryRename ( sIndex, sPath, g_dOldExts[j], g_dCurExts[j], true );
12376
12377 break;
12378 }
12379 sphLogDebug ( "RotateIndexGreedy: Current index renamed to .old" );
12380 }
12381
12382 // rename new to current
12383 for ( int i=0; i<EXT_COUNT; i++ )
12384 {
12385 if ( TryRename ( sIndex, sPath, g_dNewExts[i], g_dCurExts[i], false ) )
12386 continue;
12387
12388 // rollback new ones we already renamed
12389 for ( int j=0; j<i; j++ )
12390 TryRename ( sIndex, sPath, g_dCurExts[j], g_dNewExts[j], true );
12391
12392 // rollback old ones
12393 if ( !tIndex.m_bOnlyNew )
12394 for ( int j=0; j<=EXT_COUNT; j++ ) ///< <=, not <, since we have the .mvp at the end of these lists
12395 TryRename ( sIndex, sPath, g_dOldExts[j], g_dCurExts[j], true );
12396
12397 return false;
12398 }
12399 sphLogDebug ( "RotateIndexGreedy: New renamed to current" );
12400
12401 bool bPreread = false;
12402
12403 // try to use new index
12404 CSphString sWarning;
12405 ISphTokenizer * pTokenizer = tIndex.m_pIndex->LeakTokenizer (); // FIXME! disable support of that old indexes and remove this bullshit
12406 CSphDict * pDictionary = tIndex.m_pIndex->LeakDictionary ();
12407
12408 if ( !tIndex.m_pIndex->Prealloc ( tIndex.m_bMlock, g_bStripPath, sWarning ) || !tIndex.m_pIndex->Preread() )
12409 {
12410 if ( tIndex.m_bOnlyNew )
12411 {
12412 sphWarning ( "rotating index '%s': .new preload failed: %s; NOT SERVING", sIndex, tIndex.m_pIndex->GetLastError().cstr() );
12413 return false;
12414
12415 } else
12416 {
12417 sphWarning ( "rotating index '%s': .new preload failed: %s", sIndex, tIndex.m_pIndex->GetLastError().cstr() );
12418
12419 // try to recover
12420 for ( int j=0; j<EXT_COUNT; j++ )
12421 {
12422 TryRename ( sIndex, sPath, g_dCurExts[j], g_dNewExts[j], true );
12423 TryRename ( sIndex, sPath, g_dOldExts[j], g_dCurExts[j], true );
12424 }
12425 TryRename ( sIndex, sPath, g_dOldExts[EXT_MVP], g_dCurExts[EXT_MVP], false, false );
12426 sphLogDebug ( "RotateIndexGreedy: has recovered" );
12427
12428 if ( !tIndex.m_pIndex->Prealloc ( tIndex.m_bMlock, g_bStripPath, sWarning ) || !tIndex.m_pIndex->Preread() )
12429 {
12430 sphWarning ( "rotating index '%s': .new preload failed; ROLLBACK FAILED; INDEX UNUSABLE", sIndex );
12431 tIndex.m_bEnabled = false;
12432
12433 } else
12434 {
12435 tIndex.m_bEnabled = true;
12436 bPreread = true;
12437 sphWarning ( "rotating index '%s': .new preload failed; using old index", sIndex );
12438 }
12439
12440 if ( !sWarning.IsEmpty() )
12441 sphWarning ( "rotating index '%s': %s", sIndex, sWarning.cstr() );
12442
12443 if ( !tIndex.m_pIndex->GetTokenizer () )
12444 tIndex.m_pIndex->SetTokenizer ( pTokenizer );
12445 else
12446 SafeDelete ( pTokenizer );
12447
12448 if ( !tIndex.m_pIndex->GetDictionary () )
12449 tIndex.m_pIndex->SetDictionary ( pDictionary );
12450 else
12451 SafeDelete ( pDictionary );
12452 }
12453
12454 return bPreread;
12455
12456 } else
12457 {
12458 bPreread = true;
12459
12460 if ( !sWarning.IsEmpty() )
12461 sphWarning ( "rotating index '%s': %s", sIndex, sWarning.cstr() );
12462 }
12463
12464 if ( !tIndex.m_pIndex->GetTokenizer () )
12465 tIndex.m_pIndex->SetTokenizer ( pTokenizer );
12466 else
12467 SafeDelete ( pTokenizer );
12468
12469 if ( !tIndex.m_pIndex->GetDictionary () )
12470 tIndex.m_pIndex->SetDictionary ( pDictionary );
12471 else
12472 SafeDelete ( pDictionary );
12473
12474 // unlink .old
12475 if ( !tIndex.m_bOnlyNew )
12476 {
12477 snprintf ( sFile, sizeof(sFile), "%s.old", sPath );
12478 sphUnlinkIndex ( sFile, false );
12479 }
12480
12481 sphLogDebug ( "RotateIndexGreedy: the old index unlinked" );
12482
12483 // uff. all done
12484 tIndex.m_bEnabled = true;
12485 tIndex.m_bOnlyNew = false;
12486 sphInfo ( "rotating index '%s': success", sIndex );
12487 return bPreread;
12488 }
12489
12490 /////////////////////////////////////////////////////////////////////////////
12491 // MAIN LOOP
12492 /////////////////////////////////////////////////////////////////////////////
12493
12494 #if USE_WINDOWS
12495
CreatePipe(bool,int)12496 int CreatePipe ( bool, int ) { return -1; }
PipeAndFork(bool,int)12497 int PipeAndFork ( bool, int ) { return -1; }
12498
12499 #else
12500
12501 // open new pipe to be able to receive notifications from children
12502 // adds read-end fd to g_dPipes; returns write-end fd for child
CreatePipe(bool bFatal,int iHandler)12503 int CreatePipe ( bool bFatal, int iHandler )
12504 {
12505 assert ( g_bHeadDaemon );
12506 int dPipe[2] = { -1, -1 };
12507
12508 for ( ;; )
12509 {
12510 if ( pipe(dPipe) )
12511 {
12512 if ( bFatal )
12513 sphFatal ( "pipe() failed (error=%s)", strerror(errno) );
12514 else
12515 sphWarning ( "pipe() failed (error=%s)", strerror(errno) );
12516 break;
12517 }
12518
12519 if ( fcntl ( dPipe[0], F_SETFL, O_NONBLOCK ) )
12520 {
12521 sphWarning ( "fcntl(O_NONBLOCK) on pipe failed (error=%s)", strerror(errno) );
12522 SafeClose ( dPipe[0] );
12523 SafeClose ( dPipe[1] );
12524 break;
12525 }
12526
12527 PipeInfo_t tAdd;
12528 tAdd.m_iFD = dPipe[0];
12529 tAdd.m_iHandler = iHandler;
12530 g_dPipes.Add ( tAdd );
12531 break;
12532 }
12533
12534 return dPipe[1];
12535 }
12536
12537
12538 /// create new worker child
12539 /// creates a pipe to it, forks, and does some post-fork work
12540 //
12541 /// in child, returns write-end pipe fd (might be -1!) and sets g_bHeadDaemon to false
12542 /// in parent, returns -1 and leaves g_bHeadDaemon unaffected
PipeAndFork(bool bFatal,int iHandler)12543 int PipeAndFork ( bool bFatal, int iHandler )
12544 {
12545 int iChildPipe = CreatePipe ( bFatal, iHandler );
12546 int iFork = fork();
12547 switch ( iFork )
12548 {
12549 // fork() failed
12550 case -1:
12551 sphFatal ( "fork() failed (reason: %s)", strerror(errno) );
12552
12553 // child process, handle client
12554 case 0:
12555 g_bHeadDaemon = false;
12556 g_bGotSighup = 0; // just in case.. of a race
12557 g_bGotSigterm = 0;
12558 sphSetProcessInfo ( false );
12559 ARRAY_FOREACH ( i, g_dPipes )
12560 SafeClose ( g_dPipes[i].m_iFD );
12561 break;
12562
12563 // parent process, continue accept()ing
12564 default:
12565 g_dChildren.Add ( iFork );
12566 SafeClose ( iChildPipe );
12567 break;
12568 }
12569 return iChildPipe;
12570 }
12571
12572 #endif // !USE_WINDOWS
12573
DumpMemStat()12574 void DumpMemStat ()
12575 {
12576 #if SPH_ALLOCS_PROFILER
12577 sphMemStatDump ( g_iLogFile );
12578 #endif
12579 }
12580
12581 /// check and report if there were any leaks since last call
CheckLeaks()12582 void CheckLeaks ()
12583 {
12584 #if SPH_DEBUG_LEAKS
12585 static int iHeadAllocs = sphAllocsCount ();
12586 static int iHeadCheckpoint = sphAllocsLastID ();
12587
12588 if ( g_dThd.GetLength()==0 && !g_iRotateCount && iHeadAllocs!=sphAllocsCount() )
12589 {
12590 lseek ( g_iLogFile, 0, SEEK_END );
12591 sphAllocsDump ( g_iLogFile, iHeadCheckpoint );
12592
12593 iHeadAllocs = sphAllocsCount ();
12594 iHeadCheckpoint = sphAllocsLastID ();
12595 }
12596 #endif
12597
12598 #if SPH_ALLOCS_PROFILER
12599 int iAllocLogPeriod = 60 * 1000000;
12600 static int64_t tmLastLog = -iAllocLogPeriod*10;
12601
12602 const int iAllocCount = sphAllocsCount();
12603 const float fMemTotal = (float)sphAllocBytes();
12604
12605 if ( iAllocLogPeriod>0 && tmLastLog+iAllocLogPeriod<sphMicroTimer() )
12606 {
12607 tmLastLog = sphMicroTimer ();
12608 const int iThdsCount = g_dThd.GetLength ();
12609 const float fMB = 1024.0f*1024.0f;
12610 sphInfo ( "--- allocs-count=%d, mem-total=%.4f Mb, active-threads=%d", iAllocCount, fMemTotal/fMB, iThdsCount );
12611 DumpMemStat ();
12612 }
12613 #endif
12614 }
12615
CheckIndex(const CSphIndex * pIndex,CSphString & sError)12616 bool CheckIndex ( const CSphIndex * pIndex, CSphString & sError )
12617 {
12618 const CSphIndexSettings & tSettings = pIndex->GetSettings ();
12619
12620 if ( ( tSettings.m_iMinPrefixLen>0 || tSettings.m_iMinInfixLen>0 ) && !pIndex->IsStarEnabled() )
12621 {
12622 CSphDict * pDict = pIndex->GetDictionary ();
12623 assert ( pDict );
12624 if ( pDict->HasMorphology () )
12625 {
12626 sError = "infixes and morphology are enabled, enable_star=0";
12627 return false;
12628 }
12629 }
12630
12631 return true;
12632 }
12633
12634
CheckServedEntry(const ServedIndex_t * pEntry,const char * sIndex)12635 static bool CheckServedEntry ( const ServedIndex_t * pEntry, const char * sIndex )
12636 {
12637 if ( !pEntry )
12638 {
12639 sphWarning ( "rotating index '%s': INTERNAL ERROR, index went AWOL", sIndex );
12640 return false;
12641 }
12642
12643 if ( pEntry->m_bToDelete || !pEntry->m_pIndex )
12644 {
12645 if ( pEntry->m_bToDelete )
12646 sphWarning ( "rotating index '%s': INTERNAL ERROR, entry marked for deletion", sIndex );
12647
12648 if ( !pEntry->m_pIndex )
12649 sphWarning ( "rotating index '%s': INTERNAL ERROR, entry does not have an index", sIndex );
12650
12651 return false;
12652 }
12653
12654 return true;
12655 }
12656
12657 #define SPH_RT_AUTO_FLUSH_CHECK_PERIOD ( 5000000 )
12658
RtFlushThreadFunc(void *)12659 static void RtFlushThreadFunc ( void * )
12660 {
12661 int64_t tmNextCheck = sphMicroTimer() + SPH_RT_AUTO_FLUSH_CHECK_PERIOD;
12662 while ( !g_bRtFlushShutdown )
12663 {
12664 // stand still till save time
12665 if ( tmNextCheck>sphMicroTimer() )
12666 {
12667 sphSleepMsec ( 50 );
12668 continue;
12669 }
12670
12671 // collecting available rt indexes at save time
12672 CSphVector<CSphString> dRtIndexes;
12673 for ( IndexHashIterator_c it ( g_pIndexes ); it.Next(); )
12674 if ( it.Get().m_bRT )
12675 dRtIndexes.Add ( it.GetKey() );
12676
12677 // do check+save
12678 ARRAY_FOREACH_COND ( i, dRtIndexes, !g_bRtFlushShutdown )
12679 {
12680 const ServedIndex_t * pServed = g_pIndexes->GetRlockedEntry ( dRtIndexes[i] );
12681 if ( !pServed )
12682 continue;
12683
12684 if ( !pServed->m_bEnabled )
12685 {
12686 pServed->Unlock();
12687 continue;
12688 }
12689
12690 ISphRtIndex * pRT = (ISphRtIndex *)pServed->m_pIndex;
12691 pRT->CheckRamFlush();
12692
12693 pServed->Unlock();
12694 }
12695
12696 tmNextCheck = sphMicroTimer() + SPH_RT_AUTO_FLUSH_CHECK_PERIOD;
12697 }
12698 }
12699
12700
RotateIndexMT(const CSphString & sIndex)12701 static void RotateIndexMT ( const CSphString & sIndex )
12702 {
12703 assert ( g_eWorkers==MPM_THREADS );
12704 //////////////////
12705 // load new index
12706 //////////////////
12707
12708 // create new index, copy some settings from existing one
12709 const ServedIndex_t * pRotating = g_pIndexes->GetRlockedEntry ( sIndex );
12710 if ( !CheckServedEntry ( pRotating, sIndex.cstr() ) )
12711 {
12712 if ( pRotating )
12713 pRotating->Unlock();
12714 return;
12715 }
12716
12717 sphInfo ( "rotating index '%s': started", sIndex.cstr() );
12718
12719 ServedDesc_t tNewIndex;
12720 tNewIndex.m_bOnlyNew = pRotating->m_bOnlyNew;
12721
12722 tNewIndex.m_pIndex = sphCreateIndexPhrase ( sIndex.cstr(), NULL );
12723 tNewIndex.m_pIndex->SetEnableStar ( pRotating->m_bStar );
12724 tNewIndex.m_pIndex->m_bExpandKeywords = pRotating->m_bExpand;
12725 tNewIndex.m_pIndex->m_iExpansionLimit = g_iExpansionLimit;
12726 tNewIndex.m_pIndex->SetPreopen ( pRotating->m_bPreopen || g_bPreopenIndexes );
12727 tNewIndex.m_pIndex->SetWordlistPreload ( !pRotating->m_bOnDiskDict && !g_bOnDiskDicts );
12728
12729 // rebase new index
12730 char sNewPath [ SPH_MAX_FILENAME_LEN ];
12731 snprintf ( sNewPath, sizeof(sNewPath), "%s.new", pRotating->m_sIndexPath.cstr() );
12732 tNewIndex.m_pIndex->SetBase ( sNewPath );
12733
12734 // don't need to hold the existing index any more now
12735 pRotating->Unlock();
12736 pRotating = NULL;
12737
12738 // prealloc enough RAM and lock new index
12739 sphLogDebug ( "prealloc enough RAM and lock new index" );
12740 CSphString sWarn, sError;
12741 if ( !tNewIndex.m_pIndex->Prealloc ( tNewIndex.m_bMlock, g_bStripPath, sWarn ) )
12742 {
12743 sphWarning ( "rotating index '%s': prealloc: %s; using old index", sIndex.cstr(), tNewIndex.m_pIndex->GetLastError().cstr() );
12744 return;
12745 }
12746
12747 if ( !tNewIndex.m_pIndex->Lock() )
12748 {
12749 sphWarning ( "rotating index '%s': lock: %s; using old index", sIndex.cstr (), tNewIndex.m_pIndex->GetLastError().cstr() );
12750 return;
12751 }
12752
12753 // fixup settings if needed
12754 sphLogDebug ( "fixup settings if needed" );
12755 g_tRotateConfigMutex.Lock ();
12756 if ( tNewIndex.m_bOnlyNew && g_pCfg.m_tConf ( "index" ) && g_pCfg.m_tConf["index"]( sIndex.cstr() ) )
12757 {
12758 if ( !sphFixupIndexSettings ( tNewIndex.m_pIndex, g_pCfg.m_tConf["index"][sIndex.cstr()], sError ) )
12759 {
12760 sphWarning ( "rotating index '%s': fixup: %s; using old index", sIndex.cstr(), sError.cstr() );
12761 g_tRotateConfigMutex.Unlock ();
12762 return;
12763 }
12764 }
12765 g_tRotateConfigMutex.Unlock();
12766
12767 if ( !CheckIndex ( tNewIndex.m_pIndex, sError ) )
12768 {
12769 sphWarning ( "rotating index '%s': check: %s; using old index", sIndex.cstr(), sError.cstr() );
12770 return;
12771 }
12772
12773 if ( !tNewIndex.m_pIndex->Preread() )
12774 {
12775 sphWarning ( "rotating index '%s': preread failed: %s; using old index", sIndex.cstr(), tNewIndex.m_pIndex->GetLastError().cstr() );
12776 return;
12777 }
12778
12779 //////////////////////
12780 // activate new index
12781 //////////////////////
12782
12783 sphLogDebug ( "activate new index" );
12784
12785 ServedIndex_t * pServed = g_pIndexes->GetWlockedEntry ( sIndex );
12786 if ( !CheckServedEntry ( pServed, sIndex.cstr() ) )
12787 {
12788 if ( pServed )
12789 pServed->Unlock();
12790 return;
12791 }
12792
12793 CSphIndex * pOld = pServed->m_pIndex;
12794 CSphIndex * pNew = tNewIndex.m_pIndex;
12795
12796 // rename files
12797 // FIXME! factor out a common function w/ non-threaded rotation code
12798 char sOld [ SPH_MAX_FILENAME_LEN ];
12799 snprintf ( sOld, sizeof(sOld), "%s.old", pServed->m_sIndexPath.cstr() );
12800 char sCurTest [ SPH_MAX_FILENAME_LEN ];
12801 snprintf ( sCurTest, sizeof(sCurTest), "%s.sph", pServed->m_sIndexPath.cstr() );
12802
12803 if ( !pServed->m_bOnlyNew && sphIsReadable ( sCurTest ) && !pOld->Rename ( sOld ) )
12804 {
12805 // FIXME! rollback inside Rename() call potentially fail
12806 sphWarning ( "rotating index '%s': cur to old rename failed: %s", sIndex.cstr(), pOld->GetLastError().cstr() );
12807
12808 } else
12809 {
12810 // FIXME! at this point there's no cur lock file; ie. potential race
12811 sphLogDebug ( "no cur lock file; ie. potential race" );
12812 if ( !pNew->Rename ( pServed->m_sIndexPath.cstr() ) )
12813 {
12814 sphWarning ( "rotating index '%s': new to cur rename failed: %s", sIndex.cstr(), pNew->GetLastError().cstr() );
12815 if ( !pServed->m_bOnlyNew && !pOld->Rename ( pServed->m_sIndexPath.cstr() ) )
12816 {
12817 sphWarning ( "rotating index '%s': old to cur rename failed: %s; INDEX UNUSABLE", sIndex.cstr(), pOld->GetLastError().cstr() );
12818 pServed->m_bEnabled = false;
12819 }
12820 } else
12821 {
12822 // all went fine; swap them
12823 sphLogDebug ( "all went fine; swap them" );
12824
12825 tNewIndex.m_pIndex->m_iTID = pServed->m_pIndex->m_iTID;
12826 if ( g_pBinlog )
12827 g_pBinlog->NotifyIndexFlush ( sIndex.cstr(), pServed->m_pIndex->m_iTID, false );
12828
12829 if ( !tNewIndex.m_pIndex->GetTokenizer() )
12830 tNewIndex.m_pIndex->SetTokenizer ( pServed->m_pIndex->LeakTokenizer() );
12831
12832 if ( !tNewIndex.m_pIndex->GetDictionary() )
12833 tNewIndex.m_pIndex->SetDictionary ( pServed->m_pIndex->LeakDictionary() );
12834
12835 Swap ( pServed->m_pIndex, tNewIndex.m_pIndex );
12836 pServed->m_bEnabled = true;
12837
12838 // rename current MVP to old one to unlink it
12839 TryRename ( sIndex.cstr(), pServed->m_sIndexPath.cstr(), g_dCurExts[EXT_MVP], g_dOldExts[EXT_MVP], false, false );
12840 // unlink .old
12841 sphLogDebug ( "unlink .old" );
12842 if ( !pServed->m_bOnlyNew )
12843 {
12844 sphUnlinkIndex ( sOld, false );
12845 }
12846
12847 pServed->m_bOnlyNew = false;
12848 sphInfo ( "rotating index '%s': success", sIndex.cstr() );
12849 }
12850 }
12851
12852 pServed->Unlock();
12853 }
12854
RotationThreadFunc(void *)12855 void RotationThreadFunc ( void * )
12856 {
12857 assert ( g_eWorkers==MPM_THREADS );
12858 while ( !g_bRotateShutdown )
12859 {
12860 // check if we have work to do
12861 if ( !g_iRotateCount )
12862 {
12863 sphSleepMsec ( 50 );
12864 continue;
12865 }
12866 g_tRotateQueueMutex.Lock();
12867 if ( !g_dRotateQueue.GetLength() )
12868 {
12869 g_tRotateQueueMutex.Unlock();
12870 sphSleepMsec ( 50 );
12871 continue;
12872 }
12873
12874 CSphString sIndex = g_dRotateQueue.Pop();
12875 g_sPrereading = sIndex.cstr();
12876 g_tRotateQueueMutex.Unlock();
12877
12878 RotateIndexMT ( sIndex );
12879
12880 g_tRotateQueueMutex.Lock();
12881 if ( !g_dRotateQueue.GetLength() )
12882 {
12883 g_iRotateCount = Max ( 0, g_iRotateCount-1 );
12884 sphInfo ( "rotating index: all indexes done" );
12885 }
12886 g_sPrereading = NULL;
12887 g_tRotateQueueMutex.Unlock();
12888 }
12889 }
12890
12891
IndexRotationDone()12892 void IndexRotationDone ()
12893 {
12894 #if !USE_WINDOWS
12895 if ( g_iRotationThrottle && g_eWorkers==MPM_PREFORK )
12896 {
12897 ARRAY_FOREACH ( i, g_dChildren )
12898 g_dHupChildren.Add ( g_dChildren[i] );
12899 } else
12900 {
12901 // forcibly restart children serving persistent connections and/or preforked ones
12902 // FIXME! check how both signals are handled in case of FORK and PREFORK
12903 ARRAY_FOREACH ( i, g_dChildren )
12904 kill ( g_dChildren[i], SIGHUP );
12905 }
12906 #endif
12907
12908 g_iRotateCount = Max ( 0, g_iRotateCount-1 );
12909 sphInfo ( "rotating finished" );
12910 }
12911
12912
SeamlessTryToForkPrereader()12913 void SeamlessTryToForkPrereader ()
12914 {
12915 sphLogDebug ( "Invoked SeamlessTryToForkPrereader" );
12916
12917 // next in line
12918 const char * sPrereading = g_dRotating.Pop ();
12919 if ( !sPrereading || !g_pIndexes->Exists ( sPrereading ) )
12920 {
12921 sphWarning ( "INTERNAL ERROR: preread attempt on unknown index '%s'", sPrereading ? sPrereading : "(NULL)" );
12922 return;
12923 }
12924 const ServedIndex_t & tServed = g_pIndexes->GetUnlockedEntry ( sPrereading );
12925
12926 // alloc buffer index (once per run)
12927 if ( !g_pPrereading )
12928 g_pPrereading = sphCreateIndexPhrase ( sPrereading, NULL );
12929 else
12930 g_pPrereading->SetName ( sPrereading );
12931
12932 g_pPrereading->SetEnableStar ( tServed.m_bStar );
12933 g_pPrereading->m_bExpandKeywords = tServed.m_bExpand;
12934 g_pPrereading->m_iExpansionLimit = g_iExpansionLimit;
12935 g_pPrereading->SetPreopen ( tServed.m_bPreopen || g_bPreopenIndexes );
12936 g_pPrereading->SetWordlistPreload ( !tServed.m_bOnDiskDict && !g_bOnDiskDicts );
12937
12938 // rebase buffer index
12939 char sNewPath [ SPH_MAX_FILENAME_LEN ];
12940 snprintf ( sNewPath, sizeof(sNewPath), "%s.new", tServed.m_sIndexPath.cstr() );
12941 g_pPrereading->SetBase ( sNewPath );
12942
12943 // prealloc enough RAM and lock new index
12944 sphLogDebug ( "prealloc enough RAM and lock new index" );
12945 CSphString sWarn, sError;
12946 if ( !g_pPrereading->Prealloc ( tServed.m_bMlock, g_bStripPath, sWarn ) )
12947 {
12948 sphWarning ( "rotating index '%s': prealloc: %s; using old index", sPrereading, g_pPrereading->GetLastError().cstr() );
12949 if ( !sWarn.IsEmpty() )
12950 sphWarning ( "rotating index: %s", sWarn.cstr() );
12951 return;
12952 }
12953 if ( !sWarn.IsEmpty() )
12954 sphWarning ( "rotating index: %s: %s", sPrereading, sWarn.cstr() );
12955
12956 if ( !g_pPrereading->Lock() )
12957 {
12958 sphWarning ( "rotating index '%s': lock: %s; using old index", sPrereading, g_pPrereading->GetLastError().cstr() );
12959 g_pPrereading->Dealloc ();
12960 return;
12961 }
12962
12963 if ( tServed.m_bOnlyNew && g_pCfg.m_tConf.Exists ( "index" ) && g_pCfg.m_tConf["index"].Exists ( sPrereading ) )
12964 if ( !sphFixupIndexSettings ( g_pPrereading, g_pCfg.m_tConf["index"][sPrereading], sError ) )
12965 {
12966 sphWarning ( "rotating index '%s': fixup: %s; using old index", sPrereading, sError.cstr() );
12967 return;
12968 }
12969
12970 if ( !CheckIndex ( g_pPrereading, sError ) )
12971 {
12972 sphWarning ( "rotating index '%s': check: %s; using old index", sPrereading, sError.cstr() );
12973 return;
12974 }
12975
12976 // fork async reader
12977 sphLogDebug ( "fork async reader" );
12978 g_sPrereading = sPrereading;
12979 int iPipeFD = PipeAndFork ( true, SPH_PIPE_PREREAD );
12980
12981 // in parent, wait for prereader process to finish
12982 if ( g_bHeadDaemon )
12983 return;
12984
12985 // in child, do preread
12986 bool bRes = g_pPrereading->Preread ();
12987 if ( !bRes )
12988 sphWarning ( "rotating index '%s': preread failed: %s; using old index", g_sPrereading, g_pPrereading->GetLastError().cstr() );
12989 // report and exit
12990 DWORD uTmp = SPH_PIPE_PREREAD;
12991 sphWrite ( iPipeFD, &uTmp, sizeof(DWORD) ); // FIXME? add buffering/checks?
12992
12993 uTmp = bRes;
12994 sphWrite ( iPipeFD, &uTmp, sizeof(DWORD) );
12995
12996 ::close ( iPipeFD );
12997 sphLogDebug ( "SeamlessTryToForkPrereader: finishing the fork and invoking exit ( 0 )" );
12998 exit ( 0 );
12999 }
13000
13001
SeamlessForkPrereader()13002 void SeamlessForkPrereader ()
13003 {
13004 sphLogDebug ( "Invoked SeamlessForkPrereader" );
13005 // sanity checks
13006 if ( g_sPrereading )
13007 {
13008 sphWarning ( "INTERNAL ERROR: preread attempt before previous completion" );
13009 return;
13010 }
13011
13012 // try candidates one by one
13013 while ( g_dRotating.GetLength() && !g_sPrereading )
13014 SeamlessTryToForkPrereader ();
13015
13016 // if there's no more candidates, and nothing in the works, we're done
13017 if ( !g_sPrereading && !g_dRotating.GetLength() )
13018 IndexRotationDone ();
13019 }
13020
13021
13022 /// simple wrapper to simplify reading from pipes
13023 struct PipeReader_t
13024 {
PipeReader_tPipeReader_t13025 explicit PipeReader_t ( int iFD )
13026 : m_iFD ( iFD )
13027 , m_bError ( false )
13028 {
13029 #if !USE_WINDOWS
13030 if ( fcntl ( iFD, F_SETFL, 0 )<0 )
13031 sphWarning ( "fcntl(0) on pipe failed (error=%s)", strerror(errno) );
13032 #endif
13033 }
13034
~PipeReader_tPipeReader_t13035 ~PipeReader_t ()
13036 {
13037 SafeClose ( m_iFD );
13038 }
13039
GetFDPipeReader_t13040 int GetFD () const
13041 {
13042 return m_iFD;
13043 }
13044
IsErrorPipeReader_t13045 bool IsError () const
13046 {
13047 return m_bError;
13048 }
13049
GetIntPipeReader_t13050 int GetInt ()
13051 {
13052 int iTmp;
13053 if ( !GetBytes ( &iTmp, sizeof(iTmp) ) )
13054 iTmp = 0;
13055 return iTmp;
13056 }
13057
GetStringPipeReader_t13058 CSphString GetString ()
13059 {
13060 int iLen = GetInt ();
13061 CSphString sRes;
13062 sRes.Reserve ( iLen );
13063 if ( !GetBytes ( const_cast<char*> ( sRes.cstr() ), iLen ) )
13064 sRes = "";
13065 return sRes;
13066 }
13067
13068 protected:
GetBytesPipeReader_t13069 bool GetBytes ( void * pBuf, int iCount )
13070 {
13071 if ( m_bError )
13072 return false;
13073
13074 if ( m_iFD<0 )
13075 {
13076 m_bError = true;
13077 sphWarning ( "invalid pipe fd" );
13078 return false;
13079 }
13080
13081 for ( ;; )
13082 {
13083 int iRes = ::read ( m_iFD, pBuf, iCount );
13084 if ( iRes<0 && errno==EINTR )
13085 continue;
13086
13087 if ( iRes!=iCount )
13088 {
13089 m_bError = true;
13090 sphWarning ( "pipe read failed (exp=%d, res=%d, error=%s)",
13091 iCount, iRes, iRes>0 ? "(none)" : strerror(errno) );
13092 return false;
13093 }
13094 return true;
13095 }
13096 }
13097
13098 protected:
13099 int m_iFD;
13100 bool m_bError;
13101 };
13102
13103
13104 /// handle pipe notifications from prereading
HandlePipePreread(PipeReader_t & tPipe,bool bFailure)13105 void HandlePipePreread ( PipeReader_t & tPipe, bool bFailure )
13106 {
13107 if ( bFailure )
13108 {
13109 // clean up previous one and launch next one
13110 g_sPrereading = NULL;
13111
13112 // in any case, buffer index should now be deallocated
13113 g_pPrereading->Dealloc ();
13114 g_pPrereading->Unlock ();
13115
13116 // work next one
13117 SeamlessForkPrereader ();
13118 return;
13119 }
13120
13121 assert ( g_iRotateCount && g_bSeamlessRotate && g_sPrereading );
13122
13123 // whatever the outcome, we will be done with this one
13124 const char * sPrereading = g_sPrereading;
13125 g_sPrereading = NULL;
13126
13127 // notice that this will block!
13128 int iRes = tPipe.GetInt();
13129 if ( !tPipe.IsError() && iRes )
13130 {
13131 // if preread was successful, exchange served index and prereader buffer index
13132 ServedIndex_t & tServed = g_pIndexes->GetUnlockedEntry ( sPrereading );
13133 CSphIndex * pOld = tServed.m_pIndex;
13134 CSphIndex * pNew = g_pPrereading;
13135
13136 char sOld [ SPH_MAX_FILENAME_LEN ];
13137 snprintf ( sOld, sizeof(sOld), "%s.old", tServed.m_sIndexPath.cstr() );
13138 char sCurTest [ SPH_MAX_FILENAME_LEN ];
13139 snprintf ( sCurTest, sizeof(sCurTest), "%s.sph", tServed.m_sIndexPath.cstr() );
13140
13141 if ( !tServed.m_bOnlyNew && sphIsReadable ( sCurTest ) && !pOld->Rename ( sOld ) )
13142 {
13143 // FIXME! rollback inside Rename() call potentially fail
13144 sphWarning ( "rotating index '%s': cur to old rename failed: %s", sPrereading, pOld->GetLastError().cstr() );
13145
13146 } else
13147 {
13148 // FIXME! at this point there's no cur lock file; ie. potential race
13149 if ( !pNew->Rename ( tServed.m_sIndexPath.cstr() ) )
13150 {
13151 sphWarning ( "rotating index '%s': new to cur rename failed: %s", sPrereading, pNew->GetLastError().cstr() );
13152 if ( !tServed.m_bOnlyNew && !pOld->Rename ( tServed.m_sIndexPath.cstr() ) )
13153 {
13154 sphWarning ( "rotating index '%s': old to cur rename failed: %s; INDEX UNUSABLE", sPrereading, pOld->GetLastError().cstr() );
13155 tServed.m_bEnabled = false;
13156 }
13157 } else
13158 {
13159 // all went fine; swap them
13160 g_pPrereading->m_iTID = tServed.m_pIndex->m_iTID;
13161
13162 if ( !g_pPrereading->GetTokenizer () )
13163 g_pPrereading->SetTokenizer ( tServed.m_pIndex->LeakTokenizer () );
13164
13165 if ( !g_pPrereading->GetDictionary () )
13166 g_pPrereading->SetDictionary ( tServed.m_pIndex->LeakDictionary () );
13167
13168 Swap ( tServed.m_pIndex, g_pPrereading );
13169 tServed.m_bEnabled = true;
13170
13171 // rename current MVP to old one to unlink it
13172 TryRename ( sPrereading, tServed.m_sIndexPath.cstr(), g_dCurExts[EXT_MVP], g_dOldExts[EXT_MVP], false, false );
13173 // unlink .old
13174 if ( !tServed.m_bOnlyNew )
13175 {
13176 sphUnlinkIndex ( sOld, false );
13177 }
13178
13179 tServed.m_bOnlyNew = false;
13180 sphInfo ( "rotating index '%s': success", sPrereading );
13181 }
13182 }
13183
13184 } else
13185 {
13186 if ( tPipe.IsError() )
13187 sphWarning ( "rotating index '%s': pipe read failed", sPrereading );
13188 else
13189 sphWarning ( "rotating index '%s': preread failure reported", sPrereading );
13190 }
13191
13192 // in any case, buffer index should now be deallocated
13193 g_pPrereading->Dealloc ();
13194 g_pPrereading->Unlock ();
13195
13196 // work next one
13197 SeamlessForkPrereader ();
13198 }
13199
13200
13201 /// check if there are any notifications from the children and handle them
CheckPipes()13202 void CheckPipes ()
13203 {
13204 ARRAY_FOREACH ( i, g_dPipes )
13205 {
13206 // try to get status code
13207 DWORD uStatus;
13208 int iRes = ::read ( g_dPipes[i].m_iFD, &uStatus, sizeof(DWORD) );
13209
13210 // no data yet?
13211 if ( iRes==-1 && errno==EAGAIN )
13212 continue;
13213
13214 // either if there's eof, or error, or valid data - this pipe is over
13215 PipeReader_t tPipe ( g_dPipes[i].m_iFD );
13216 int iHandler = g_dPipes[i].m_iHandler;
13217 g_dPipes.Remove ( i-- );
13218
13219 // check for eof/error
13220 bool bFailure = false;
13221 if ( iRes!=sizeof(DWORD) )
13222 {
13223 bFailure = true;
13224
13225 if ( iHandler<0 )
13226 continue; // no handler; we're not expecting anything
13227
13228 if ( iRes!=0 || iHandler>=0 )
13229 sphWarning ( "pipe status read failed (handler=%d)", iHandler );
13230 }
13231
13232 // check for handler/status mismatch
13233 if ( !bFailure && ( iHandler>=0 && (int)uStatus!=iHandler ) )
13234 {
13235 bFailure = true;
13236 sphWarning ( "INTERNAL ERROR: pipe status mismatch (handler=%d, status=%d)", iHandler, uStatus );
13237 }
13238
13239 // check for handler promotion (ie: we did not expect anything particular, but something happened anyway)
13240 if ( !bFailure && iHandler<0 )
13241 iHandler = (int)uStatus;
13242
13243 // run the proper handler
13244 switch ( iHandler )
13245 {
13246 case SPH_PIPE_PREREAD: HandlePipePreread ( tPipe, bFailure ); break;
13247 default: if ( !bFailure ) sphWarning ( "INTERNAL ERROR: unknown pipe handler (handler=%d, status=%d)", iHandler, uStatus ); break;
13248 }
13249 }
13250 }
13251
13252
ConfigureIndex(ServedDesc_t & tIdx,const CSphConfigSection & hIndex)13253 void ConfigureIndex ( ServedDesc_t & tIdx, const CSphConfigSection & hIndex )
13254 {
13255 tIdx.m_bMlock = ( hIndex.GetInt ( "mlock", 0 )!=0 ) && !g_bOptNoLock;
13256 tIdx.m_bStar = ( hIndex.GetInt ( "enable_star", 0 )!=0 );
13257 tIdx.m_bExpand = ( hIndex.GetInt ( "expand_keywords", 0 )!=0 );
13258 tIdx.m_bPreopen = ( hIndex.GetInt ( "preopen", 0 )!=0 );
13259 tIdx.m_bOnDiskDict = ( hIndex.GetInt ( "ondisk_dict", 0 )!=0 );
13260 }
13261
13262
PrereadNewIndex(ServedIndex_t & tIdx,const CSphConfigSection & hIndex,const char * szIndexName)13263 bool PrereadNewIndex ( ServedIndex_t & tIdx, const CSphConfigSection & hIndex, const char * szIndexName )
13264 {
13265 CSphString sWarning;
13266
13267 if ( !tIdx.m_pIndex->Prealloc ( tIdx.m_bMlock, g_bStripPath, sWarning ) || !tIdx.m_pIndex->Preread() )
13268 {
13269 sphWarning ( "index '%s': preload: %s; NOT SERVING", szIndexName, tIdx.m_pIndex->GetLastError().cstr() );
13270 return false;
13271 }
13272
13273 if ( !sWarning.IsEmpty() )
13274 sphWarning ( "index '%s': %s", szIndexName, sWarning.cstr() );
13275
13276 CSphString sError;
13277 if ( !sphFixupIndexSettings ( tIdx.m_pIndex, hIndex, sError ) )
13278 {
13279 sphWarning ( "index '%s': %s - NOT SERVING", szIndexName, sError.cstr() );
13280 return false;
13281 }
13282
13283 // try to lock it
13284 if ( !g_bOptNoLock && !tIdx.m_pIndex->Lock() )
13285 {
13286 sphWarning ( "index '%s': lock: %s; NOT SERVING", szIndexName, tIdx.m_pIndex->GetLastError().cstr() );
13287 return false;
13288 }
13289
13290 return true;
13291 }
13292
13293
ConfigureAgent(AgentDesc_t & tAgent,const CSphVariant * pAgent,const char * szIndexName,bool bBlackhole)13294 bool ConfigureAgent ( AgentDesc_t & tAgent, const CSphVariant * pAgent, const char * szIndexName, bool bBlackhole )
13295 {
13296 // extract host name or path
13297 const char * p = pAgent->cstr();
13298 while ( sphIsAlpha(*p) || *p=='.' || *p=='-' || *p=='/' ) p++;
13299 if ( p==pAgent->cstr() )
13300 {
13301 sphWarning ( "index '%s': agent '%s': host name or path expected - SKIPPING AGENT",
13302 szIndexName, pAgent->cstr() );
13303 return false;
13304 }
13305 if ( *p++!=':' )
13306 {
13307 sphWarning ( "index '%s': agent '%s': colon expected near '%s' - SKIPPING AGENT",
13308 szIndexName, pAgent->cstr(), p );
13309 return false;
13310 }
13311
13312 CSphString sSub = pAgent->SubString ( 0, p-1-pAgent->cstr() );
13313 if ( sSub.cstr()[0]=='/' )
13314 {
13315 #if USE_WINDOWS
13316 sphWarning ( "index '%s': agent '%s': UNIX sockets are not supported on Windows - SKIPPING AGENT",
13317 szIndexName, pAgent->cstr() );
13318 return false;
13319 #else
13320 if ( strlen ( sSub.cstr() ) + 1 > sizeof(((struct sockaddr_un *)0)->sun_path) )
13321 {
13322 sphWarning ( "index '%s': agent '%s': UNIX socket path is too long - SKIPPING AGENT",
13323 szIndexName, pAgent->cstr() );
13324 return false;
13325 }
13326
13327 tAgent.m_iFamily = AF_UNIX;
13328 tAgent.m_sPath = sSub;
13329 p--;
13330 #endif
13331 } else
13332 {
13333 tAgent.m_iFamily = AF_INET;
13334 tAgent.m_sHost = sSub;
13335
13336 // extract port
13337 if ( !isdigit(*p) )
13338 {
13339 sphWarning ( "index '%s': agent '%s': port number expected near '%s' - SKIPPING AGENT",
13340 szIndexName, pAgent->cstr(), p );
13341 return false;
13342 }
13343 tAgent.m_iPort = atoi(p);
13344
13345 if ( !IsPortInRange ( tAgent.m_iPort ) )
13346 {
13347 sphWarning ( "index '%s': agent '%s': invalid port number near '%s' - SKIPPING AGENT",
13348 szIndexName, pAgent->cstr(), p );
13349 return false;
13350 }
13351
13352 while ( isdigit(*p) ) p++;
13353 }
13354
13355 // extract index list
13356 if ( *p++!=':' )
13357 {
13358 sphWarning ( "index '%s': agent '%s': colon expected near '%s' - SKIPPING AGENT",
13359 szIndexName, pAgent->cstr(), p );
13360 return false;
13361 }
13362 while ( isspace(*p) )
13363 p++;
13364 const char * sIndexList = p;
13365 while ( sphIsAlpha(*p) || isspace(*p) || *p==',' )
13366 p++;
13367 if ( *p )
13368 {
13369 sphWarning ( "index '%s': agent '%s': index list expected near '%s' - SKIPPING AGENT",
13370 szIndexName, pAgent->cstr(), p );
13371 return false;
13372 }
13373 tAgent.m_sIndexes = sIndexList;
13374
13375 // lookup address (if needed)
13376 if ( tAgent.m_iFamily==AF_INET )
13377 {
13378 tAgent.m_uAddr = sphGetAddress ( tAgent.m_sHost.cstr() );
13379 if ( tAgent.m_uAddr==0 )
13380 {
13381 sphWarning ( "index '%s': agent '%s': failed to lookup host name '%s' (error=%s) - SKIPPING AGENT",
13382 szIndexName, pAgent->cstr(), tAgent.m_sHost.cstr(), sphSockError() );
13383 return false;
13384 }
13385 }
13386
13387 tAgent.m_bBlackhole = bBlackhole;
13388
13389 // allocate stats slot
13390 if ( g_pStats )
13391 {
13392 g_tStatsMutex.Lock();
13393 for ( int i=0; i<STATS_MAX_AGENTS/32; i++ )
13394 if ( g_pStats->m_bmAgentStats[i]!=0xffffffffUL )
13395 {
13396 int j = FindBit ( g_pStats->m_bmAgentStats[i] );
13397 g_pStats->m_bmAgentStats[i] |= ( 1<<j );
13398 tAgent.m_iStatsIndex = i*32 + j;
13399 memset ( &g_pStats->m_dAgentStats[tAgent.m_iStatsIndex], 0, sizeof(AgentStats_t) );
13400 break;
13401 }
13402 g_tStatsMutex.Unlock();
13403 }
13404
13405 return true;
13406 }
13407
ConfigureDistributedIndex(const char * szIndexName,const CSphConfigSection & hIndex)13408 static DistributedIndex_t ConfigureDistributedIndex ( const char * szIndexName, const CSphConfigSection & hIndex )
13409 {
13410 assert ( hIndex("type") && hIndex["type"]=="distributed" );
13411
13412 DistributedIndex_t tIdx;
13413
13414 // add local agents
13415 for ( CSphVariant * pLocal = hIndex("local"); pLocal; pLocal = pLocal->m_pNext )
13416 {
13417 if ( !g_pIndexes->Exists ( pLocal->cstr() ) )
13418 {
13419 sphWarning ( "index '%s': no such local index '%s' - SKIPPING LOCAL INDEX",
13420 szIndexName, pLocal->cstr() );
13421 continue;
13422 }
13423 tIdx.m_dLocal.Add ( pLocal->cstr() );
13424 }
13425
13426 // add remote agents
13427 for ( CSphVariant * pAgent = hIndex("agent"); pAgent; pAgent = pAgent->m_pNext )
13428 {
13429 AgentDesc_t tAgent;
13430 if ( ConfigureAgent ( tAgent, pAgent, szIndexName, false ) )
13431 tIdx.m_dAgents.Add ( tAgent );
13432 }
13433
13434 for ( CSphVariant * pAgent = hIndex("agent_blackhole"); pAgent; pAgent = pAgent->m_pNext )
13435 {
13436 AgentDesc_t tAgent;
13437 if ( ConfigureAgent ( tAgent, pAgent, szIndexName, true ) )
13438 tIdx.m_dAgents.Add ( tAgent );
13439 }
13440
13441 // configure options
13442 if ( hIndex("agent_connect_timeout") )
13443 {
13444 if ( hIndex["agent_connect_timeout"].intval()<=0 )
13445 sphWarning ( "index '%s': connect_timeout must be positive, ignored", szIndexName );
13446 else
13447 tIdx.m_iAgentConnectTimeout = hIndex["agent_connect_timeout"].intval();
13448 }
13449
13450 if ( hIndex("agent_query_timeout") )
13451 {
13452 if ( hIndex["agent_query_timeout"].intval()<=0 )
13453 sphWarning ( "index '%s': query_timeout must be positive, ignored", szIndexName );
13454 else
13455 tIdx.m_iAgentQueryTimeout = hIndex["agent_query_timeout"].intval();
13456 }
13457
13458 return tIdx;
13459 }
13460
13461
FreeAgentStats(DistributedIndex_t & tIndex)13462 void FreeAgentStats ( DistributedIndex_t & tIndex )
13463 {
13464 if ( !g_pStats )
13465 return;
13466
13467 g_tStatsMutex.Lock();
13468 ARRAY_FOREACH ( i, tIndex.m_dAgents )
13469 {
13470 int iIndex = tIndex.m_dAgents[i].m_iStatsIndex;
13471 if ( iIndex<0 || iIndex>=STATS_MAX_AGENTS )
13472 continue;
13473
13474 assert ( g_pStats->m_bmAgentStats[iIndex>>5] & ( 1UL<<( iIndex & 31 ) ) );
13475 g_pStats->m_bmAgentStats[iIndex>>5] &= ~( 1UL<<( iIndex & 31 ) );
13476 }
13477 g_tStatsMutex.Unlock();
13478 }
13479
13480
PreCreatePlainIndex(ServedDesc_t & tServed,const char * sName)13481 void PreCreatePlainIndex ( ServedDesc_t & tServed, const char * sName )
13482 {
13483 tServed.m_pIndex = sphCreateIndexPhrase ( sName, tServed.m_sIndexPath.cstr() );
13484 tServed.m_pIndex->SetEnableStar ( tServed.m_bStar );
13485 tServed.m_pIndex->m_bExpandKeywords = tServed.m_bExpand;
13486 tServed.m_pIndex->m_iExpansionLimit = g_iExpansionLimit;
13487 tServed.m_pIndex->SetPreopen ( tServed.m_bPreopen || g_bPreopenIndexes );
13488 tServed.m_pIndex->SetWordlistPreload ( !tServed.m_bOnDiskDict && !g_bOnDiskDicts );
13489 tServed.m_bEnabled = false;
13490 }
13491
13492
AddIndex(const char * szIndexName,const CSphConfigSection & hIndex)13493 ESphAddIndex AddIndex ( const char * szIndexName, const CSphConfigSection & hIndex )
13494 {
13495 if ( hIndex("type") && hIndex["type"]=="distributed" )
13496 {
13497 ///////////////////////////////
13498 // configure distributed index
13499 ///////////////////////////////
13500
13501 DistributedIndex_t tIdx = ConfigureDistributedIndex ( szIndexName, hIndex );
13502
13503 // finally, check and add distributed index to global table
13504 if ( tIdx.m_dAgents.GetLength()==0 && tIdx.m_dLocal.GetLength()==0 )
13505 {
13506 FreeAgentStats ( tIdx );
13507 sphWarning ( "index '%s': no valid local/remote indexes in distributed index - NOT SERVING", szIndexName );
13508 return ADD_ERROR;
13509
13510 } else
13511 {
13512 g_tDistLock.Lock ();
13513 if ( !g_hDistIndexes.Add ( tIdx, szIndexName ) )
13514 {
13515 g_tDistLock.Unlock ();
13516 FreeAgentStats ( tIdx );
13517 sphWarning ( "index '%s': duplicate name - NOT SERVING", szIndexName );
13518 return ADD_ERROR;
13519 }
13520 g_tDistLock.Unlock ();
13521 }
13522
13523 return ADD_DISTR;
13524
13525 } else if ( hIndex("type") && hIndex["type"]=="rt" )
13526 {
13527 ////////////////////////////
13528 // configure realtime index
13529 ////////////////////////////
13530
13531 if ( g_eWorkers!=MPM_THREADS )
13532 {
13533 sphWarning ( "index '%s': RT index requires workers=threads - NOT SERVING", szIndexName );
13534 return ADD_ERROR;
13535 }
13536
13537 CSphString sError;
13538 CSphSchema tSchema ( szIndexName );
13539 if ( !sphRTSchemaConfigure ( hIndex, &tSchema, &sError ) )
13540 {
13541 sphWarning ( "index '%s': %s - NOT SERVING", szIndexName, sError.cstr() );
13542 return ADD_ERROR;
13543 }
13544
13545 // path
13546 if ( !hIndex("path") )
13547 {
13548 sphWarning ( "index '%s': path must be specified - NOT SERVING", szIndexName );
13549 return ADD_ERROR;
13550 }
13551
13552 // RAM chunk size
13553 DWORD uRamSize = hIndex.GetSize ( "rt_mem_limit", 32*1024*1024 );
13554 if ( uRamSize<128*1024 )
13555 {
13556 sphWarning ( "index '%s': rt_mem_limit extremely low, using 128K instead", szIndexName );
13557 uRamSize = 128*1024;
13558 } else if ( uRamSize<8*1024*1024 )
13559 sphWarning ( "index '%s': rt_mem_limit very low (under 8 MB)", szIndexName );
13560
13561 // index
13562 ServedDesc_t tIdx;
13563 bool bWordDict = strcmp ( hIndex.GetStr ( "dict", "" ), "keywords" )==0;
13564 tIdx.m_pIndex = sphCreateIndexRT ( tSchema, szIndexName, uRamSize, hIndex["path"].cstr(), bWordDict );
13565 tIdx.m_bEnabled = false;
13566 tIdx.m_sIndexPath = hIndex["path"];
13567 tIdx.m_bRT = true;
13568
13569 ConfigureIndex ( tIdx, hIndex );
13570 tIdx.m_pIndex->SetEnableStar ( tIdx.m_bStar );
13571 tIdx.m_pIndex->m_iExpansionLimit = g_iExpansionLimit;
13572 tIdx.m_pIndex->SetPreopen ( tIdx.m_bPreopen || g_bPreopenIndexes );
13573 tIdx.m_pIndex->SetWordlistPreload ( !tIdx.m_bOnDiskDict && !g_bOnDiskDicts );
13574
13575 // pick config settings
13576 // they should be overriden later by Preload() if needed
13577 CSphIndexSettings tSettings;
13578 if ( !sphConfIndex ( hIndex, tSettings, sError ) )
13579 {
13580 sphWarning ( "ERROR: index '%s': %s - NOT SERVING", szIndexName, sError.cstr() );
13581 return ADD_ERROR;
13582 }
13583
13584 tIdx.m_pIndex->Setup ( tSettings );
13585
13586 // hash it
13587 if ( !g_pIndexes->Add ( tIdx, szIndexName ) )
13588 {
13589 sphWarning ( "INTERNAL ERROR: index '%s': hash add failed - NOT SERVING", szIndexName );
13590 return ADD_ERROR;
13591 }
13592
13593 // leak pointer, so it's destructor won't delete it
13594 tIdx.m_pIndex = NULL;
13595
13596 return ADD_RT;
13597
13598 } else if ( !hIndex("type") || hIndex["type"]=="plain" )
13599 {
13600 /////////////////////////
13601 // configure local index
13602 /////////////////////////
13603
13604 ServedDesc_t tIdx;
13605
13606 // check path
13607 if ( !hIndex.Exists ( "path" ) )
13608 {
13609 sphWarning ( "index '%s': key 'path' not found - NOT SERVING", szIndexName );
13610 return ADD_ERROR;
13611 }
13612
13613 // check name
13614 if ( g_pIndexes->Exists ( szIndexName ) )
13615 {
13616 sphWarning ( "index '%s': duplicate name - NOT SERVING", szIndexName );
13617 return ADD_ERROR;
13618 }
13619
13620 // configure memlocking, star
13621 ConfigureIndex ( tIdx, hIndex );
13622
13623 // try to create index
13624 tIdx.m_sIndexPath = hIndex["path"];
13625 PreCreatePlainIndex ( tIdx, szIndexName );
13626
13627 // done
13628 if ( !g_pIndexes->Add ( tIdx, szIndexName ) )
13629 {
13630 sphWarning ( "INTERNAL ERROR: index '%s': hash add failed - NOT SERVING", szIndexName );
13631 return ADD_ERROR;
13632 }
13633
13634 // leak pointer, so it's destructor won't delete it
13635 tIdx.m_pIndex = NULL;
13636
13637 return ADD_LOCAL;
13638
13639 } else
13640 {
13641 // unknown type
13642 sphWarning ( "index '%s': unknown type '%s' - NOT SERVING", szIndexName, hIndex["type"].cstr() );
13643 return ADD_ERROR;
13644 }
13645 }
13646
13647
CheckConfigChanges()13648 bool CheckConfigChanges ()
13649 {
13650 struct stat tStat;
13651 memset ( &tStat, 0, sizeof ( tStat ) );
13652 if ( stat ( g_sConfigFile.cstr (), &tStat ) < 0 )
13653 memset ( &tStat, 0, sizeof ( tStat ) );
13654
13655 DWORD uCRC32 = 0;
13656
13657 #if !USE_WINDOWS
13658 char sBuf [ 8192 ];
13659 FILE * fp = NULL;
13660
13661 fp = fopen ( g_sConfigFile.cstr (), "rb" );
13662 if ( !fp )
13663 return true;
13664 fgets ( sBuf, sizeof(sBuf), fp );
13665 fclose ( fp );
13666
13667 char * p = sBuf;
13668 while ( isspace(*p) )
13669 p++;
13670 if ( p[0]=='#' && p[1]=='!' )
13671 {
13672 p += 2;
13673
13674 CSphVector<char> dContent;
13675 char sError [ 1024 ];
13676 if ( !TryToExec ( p, g_sConfigFile.cstr(), dContent, sError, sizeof(sError) ) )
13677 return true;
13678
13679 uCRC32 = sphCRC32 ( (const BYTE*)dContent.Begin(), dContent.GetLength() );
13680 } else
13681 sphCalcFileCRC32 ( g_sConfigFile.cstr (), uCRC32 );
13682 #else
13683 sphCalcFileCRC32 ( g_sConfigFile.cstr (), uCRC32 );
13684 #endif
13685
13686 if ( g_uCfgCRC32==uCRC32 && tStat.st_mtime==g_tCfgStat.st_mtime && tStat.st_ctime==g_tCfgStat.st_ctime && tStat.st_size==g_tCfgStat.st_size )
13687 return false;
13688
13689 g_uCfgCRC32 = uCRC32;
13690 g_tCfgStat = tStat;
13691
13692 return true;
13693 }
13694
13695
ReloadIndexSettings(CSphConfigParser & tCP)13696 void ReloadIndexSettings ( CSphConfigParser & tCP )
13697 {
13698 if ( !tCP.ReParse ( g_sConfigFile.cstr () ) )
13699 {
13700 sphWarning ( "failed to parse config file '%s'; using previous settings", g_sConfigFile.cstr () );
13701 return;
13702 }
13703
13704 g_bDoDelete = false;
13705
13706 for ( IndexHashIterator_c it ( g_pIndexes ); it.Next(); )
13707 it.Get().m_bToDelete = true; ///< FIXME! What about write lock before doing this?
13708
13709 g_hDistIndexes.IterateStart ();
13710 while ( g_hDistIndexes.IterateNext () )
13711 g_hDistIndexes.IterateGet().m_bToDelete = true;
13712
13713 int nTotalIndexes = g_pIndexes->GetLength () + g_hDistIndexes.GetLength ();
13714 int nChecked = 0;
13715
13716 const CSphConfig & hConf = tCP.m_tConf;
13717 hConf["index"].IterateStart ();
13718 while ( hConf["index"].IterateNext() )
13719 {
13720 const CSphConfigSection & hIndex = hConf["index"].IterateGet();
13721 const char * sIndexName = hConf["index"].IterateGetKey().cstr();
13722
13723 ServedIndex_t * pServedIndex = g_pIndexes->GetWlockedEntry ( sIndexName );
13724 if ( pServedIndex )
13725 {
13726 ConfigureIndex ( *pServedIndex, hIndex );
13727 pServedIndex->m_bToDelete = false;
13728 nChecked++;
13729 pServedIndex->Unlock();
13730
13731 } else if ( g_hDistIndexes.Exists ( sIndexName ) && hIndex.Exists("type") && hIndex["type"]=="distributed" )
13732 {
13733 DistributedIndex_t tIdx = ConfigureDistributedIndex ( sIndexName, hIndex );
13734
13735 // finally, check and add distributed index to global table
13736 if ( tIdx.m_dAgents.GetLength()==0 && tIdx.m_dLocal.GetLength()==0 )
13737 {
13738 FreeAgentStats ( tIdx );
13739 sphWarning ( "index '%s': no valid local/remote indexes in distributed index; using last valid definition", sIndexName );
13740 g_hDistIndexes[sIndexName].m_bToDelete = false;
13741
13742 } else
13743 {
13744 g_tDistLock.Lock();
13745 FreeAgentStats ( g_hDistIndexes[sIndexName] );
13746 g_hDistIndexes[sIndexName] = tIdx;
13747 g_tDistLock.Unlock();
13748 }
13749
13750 nChecked++;
13751
13752 } else if ( AddIndex ( sIndexName, hIndex )==ADD_LOCAL )
13753 {
13754 ServedIndex_t * pIndex = g_pIndexes->GetWlockedEntry ( sIndexName );
13755 if ( pIndex )
13756 {
13757 pIndex->m_bOnlyNew = true;
13758 pIndex->Unlock();
13759 }
13760 }
13761 }
13762
13763 if ( nChecked < nTotalIndexes )
13764 g_bDoDelete = true;
13765 }
13766
13767
CheckDelete()13768 void CheckDelete ()
13769 {
13770 if ( !g_bDoDelete )
13771 return;
13772
13773 if ( g_dChildren.GetLength() )
13774 return;
13775
13776 CSphVector<const CSphString *> dToDelete;
13777 CSphVector<const CSphString *> dDistToDelete;
13778 dToDelete.Reserve ( 8 );
13779 dDistToDelete.Reserve ( 8 );
13780
13781 for ( IndexHashIterator_c it ( g_pIndexes ); it.Next(); )
13782 {
13783 ServedIndex_t & tIndex = it.Get();
13784 if ( tIndex.m_bToDelete )
13785 dToDelete.Add ( &it.GetKey() );
13786 }
13787
13788 g_hDistIndexes.IterateStart ();
13789 while ( g_hDistIndexes.IterateNext () )
13790 {
13791 DistributedIndex_t & tIndex = g_hDistIndexes.IterateGet ();
13792 if ( tIndex.m_bToDelete )
13793 dDistToDelete.Add ( &g_hDistIndexes.IterateGetKey () );
13794 }
13795
13796 ARRAY_FOREACH ( i, dToDelete )
13797 g_pIndexes->Delete ( *dToDelete[i] ); // should result in automatic CSphIndex::Unlock() via dtor call
13798
13799 g_tDistLock.Lock();
13800
13801 ARRAY_FOREACH ( i, dDistToDelete )
13802 {
13803 FreeAgentStats ( g_hDistIndexes [ *dDistToDelete[i] ] );
13804 g_hDistIndexes.Delete ( *dDistToDelete[i] );
13805 }
13806
13807 g_tDistLock.Unlock();
13808
13809 g_bDoDelete = false;
13810 }
13811
13812
CheckRotate()13813 void CheckRotate ()
13814 {
13815 // do we need to rotate now?
13816 if ( !g_iRotateCount )
13817 return;
13818
13819 sphLogDebug ( "CheckRotate invoked" );
13820
13821 /////////////////////
13822 // RAM-greedy rotate
13823 /////////////////////
13824
13825 if ( !g_bSeamlessRotate || g_eWorkers==MPM_PREFORK )
13826 {
13827 // wait until there's no running queries
13828 if ( g_dChildren.GetLength() && g_eWorkers!=MPM_PREFORK )
13829 return;
13830
13831 if ( CheckConfigChanges () )
13832 {
13833 ReloadIndexSettings ( g_pCfg );
13834 }
13835
13836 for ( IndexHashIterator_c it ( g_pIndexes ); it.Next(); )
13837 {
13838 ServedIndex_t & tIndex = it.Get();
13839 tIndex.WriteLock();
13840 const char * sIndex = it.GetKey().cstr();
13841 assert ( tIndex.m_pIndex );
13842
13843 bool bWasAdded = tIndex.m_bOnlyNew;
13844 RotateIndexGreedy ( tIndex, sIndex );
13845 if ( bWasAdded && tIndex.m_bEnabled )
13846 {
13847 const CSphConfigType & hConf = g_pCfg.m_tConf ["index"];
13848 if ( hConf.Exists ( sIndex ) )
13849 {
13850 CSphString sError;
13851 if ( !sphFixupIndexSettings ( tIndex.m_pIndex, hConf [sIndex], sError ) )
13852 {
13853 sphWarning ( "index '%s': %s - NOT SERVING", sIndex, sError.cstr() );
13854 tIndex.m_bEnabled = false;
13855 }
13856
13857 if ( tIndex.m_bEnabled && !CheckIndex ( tIndex.m_pIndex, sError ) )
13858 {
13859 sphWarning ( "index '%s': %s - NOT SERVING", sIndex, sError.cstr() );
13860 tIndex.m_bEnabled = false;
13861 }
13862 }
13863 }
13864 tIndex.Unlock();
13865 }
13866
13867 IndexRotationDone ();
13868 return;
13869 }
13870
13871 ///////////////////
13872 // seamless rotate
13873 ///////////////////
13874
13875 if ( g_dRotating.GetLength() || g_dRotateQueue.GetLength() || g_sPrereading )
13876 return; // rotate in progress already; will be handled in CheckPipes()
13877
13878 g_tRotateConfigMutex.Lock();
13879 if ( CheckConfigChanges() )
13880 {
13881 ReloadIndexSettings ( g_pCfg );
13882 }
13883 g_tRotateConfigMutex.Unlock();
13884
13885 int iRotIndexes = 0;
13886 // check what indexes need to be rotated
13887 for ( IndexHashIterator_c it ( g_pIndexes ); it.Next(); )
13888 {
13889 const ServedIndex_t & tIndex = it.Get();
13890 const CSphString & sIndex = it.GetKey();
13891 assert ( tIndex.m_pIndex );
13892
13893 CSphString sNewPath;
13894 sNewPath.SetSprintf ( "%s.new", tIndex.m_sIndexPath.cstr() );
13895
13896 // check if there's a .new index incoming
13897 // FIXME? move this code to index, and also check for exists-but-not-readable
13898 CSphString sTmp;
13899 sTmp.SetSprintf ( "%s.sph", sNewPath.cstr() );
13900 if ( !sphIsReadable ( sTmp.cstr() ) )
13901 {
13902 sphLogDebug ( "%s.sph is not readable. Skipping", sNewPath.cstr() );
13903 continue;
13904 }
13905
13906 if ( g_eWorkers==MPM_THREADS )
13907 {
13908 g_tRotateQueueMutex.Lock();
13909 g_dRotateQueue.Add ( sIndex );
13910 g_tRotateQueueMutex.Unlock();
13911 } else
13912 {
13913 g_dRotating.Add ( sIndex.cstr() );
13914
13915 if ( !( tIndex.m_bPreopen || g_bPreopenIndexes ) )
13916 sphWarning ( "rotating index '%s' without preopen option; use per-index propen=1 or searchd preopen_indexes=1", sIndex.cstr() );
13917 }
13918
13919 iRotIndexes++;
13920 }
13921
13922 if ( !iRotIndexes )
13923 {
13924 g_iRotateCount = Max ( 0, g_iRotateCount-1 );
13925 sphWarning ( "nothing to rotate after SIGHUP ( in queue=%d )", g_iRotateCount );
13926 }
13927
13928 if ( g_eWorkers!=MPM_THREADS && iRotIndexes )
13929 SeamlessForkPrereader ();
13930 }
13931
13932
CheckReopen()13933 void CheckReopen ()
13934 {
13935 if ( !g_bGotSigusr1 )
13936 return;
13937
13938 // reopen searchd log
13939 if ( g_iLogFile>=0 && !g_bLogTty )
13940 {
13941 int iFD = ::open ( g_sLogFile.cstr(), O_CREAT | O_RDWR | O_APPEND, S_IREAD | S_IWRITE );
13942 if ( iFD<0 )
13943 {
13944 sphWarning ( "failed to reopen log file '%s': %s", g_sLogFile.cstr(), strerror(errno) );
13945 } else
13946 {
13947 ::close ( g_iLogFile );
13948 g_iLogFile = iFD;
13949 g_bLogTty = ( isatty ( g_iLogFile )!=0 );
13950 sphInfo ( "log reopened" );
13951 }
13952 }
13953
13954 // reopen query log
13955 if ( !g_bQuerySyslog && g_iQueryLogFile!=g_iLogFile && g_iQueryLogFile>=0 && !isatty ( g_iQueryLogFile ) )
13956 {
13957 int iFD = ::open ( g_sQueryLogFile.cstr(), O_CREAT | O_RDWR | O_APPEND, S_IREAD | S_IWRITE );
13958 if ( iFD<0 )
13959 {
13960 sphWarning ( "failed to reopen query log file '%s': %s", g_sQueryLogFile.cstr(), strerror(errno) );
13961 } else
13962 {
13963 ::close ( g_iQueryLogFile );
13964 g_iQueryLogFile = iFD;
13965 sphInfo ( "query log reopened" );
13966 }
13967 }
13968
13969 #if !USE_WINDOWS
13970 if ( g_eWorkers==MPM_PREFORK )
13971 ARRAY_FOREACH ( i, g_dChildren )
13972 kill ( g_dChildren[i], SIGUSR1 );
13973 #endif
13974
13975
13976 g_bGotSigusr1 = 0;
13977 }
13978
13979
ThdSaveIndexes(void *)13980 static void ThdSaveIndexes ( void * )
13981 {
13982 SaveIndexes ();
13983
13984 // we're no more flushing
13985 g_pFlush->m_bFlushing = false;
13986 }
13987
13988 #if !USE_WINDOWS
13989 int PreforkChild ();
13990 #endif
13991
CheckFlush()13992 void CheckFlush ()
13993 {
13994 if ( g_pFlush->m_bFlushing )
13995 return;
13996
13997 // do a periodic check, unless we have a forced check
13998 if ( !g_pFlush->m_bForceCheck )
13999 {
14000 static int64_t tmLastCheck = -1000;
14001 int64_t tmNow = sphMicroTimer();
14002
14003 if ( !g_iAttrFlushPeriod || ( tmLastCheck + int64_t(g_iAttrFlushPeriod)*I64C(1000000) )>=tmNow )
14004 return;
14005
14006 tmLastCheck = tmNow;
14007 sphLogDebug ( "attrflush: doing periodic check" );
14008 } else
14009 {
14010 sphLogDebug ( "attrflush: doing forced check" );
14011 }
14012
14013 // check if there are dirty indexes
14014 bool bDirty = false;
14015 for ( IndexHashIterator_c it ( g_pIndexes ); it.Next(); )
14016 {
14017 const ServedIndex_t & tServed = it.Get();
14018 if ( tServed.m_bEnabled && tServed.m_pIndex->GetAttributeStatus() )
14019 {
14020 bDirty = true;
14021 break;
14022 }
14023 }
14024
14025 // need to set this before clearing check flag
14026 if ( bDirty )
14027 g_pFlush->m_bFlushing = true;
14028
14029 // if there was a forced check in progress, it no longer is
14030 if ( g_pFlush->m_bForceCheck )
14031 g_pFlush->m_bForceCheck = false;
14032
14033 // nothing to do, no indexes were updated
14034 if ( !bDirty )
14035 {
14036 sphLogDebug ( "attrflush: no dirty indexes found" );
14037 return;
14038 }
14039
14040 // launch the flush!
14041 g_pFlush->m_iFlushTag++;
14042
14043 sphLogDebug ( "attrflush: starting writer, tag ( %d )", g_pFlush->m_iFlushTag );
14044
14045 #if !USE_WINDOWS
14046 if ( g_eWorkers==MPM_FORK || g_eWorkers==MPM_PREFORK )
14047 {
14048 PreforkChild(); // FIXME! gracefully handle fork() failures, Windows, etc
14049 if ( g_bHeadDaemon )
14050 {
14051 return;
14052 }
14053
14054 // child process, do the work
14055 SaveIndexes ();
14056 g_pFlush->m_bFlushing = false;
14057 exit ( 0 );
14058 } else
14059 #endif
14060 {
14061 ThdDesc_t tThd;
14062 if ( !sphThreadCreate ( &tThd.m_tThd, ThdSaveIndexes, NULL, true ) )
14063 sphWarning ( "failed to create attribute save thread, error[%d] %s", errno, strerror(errno) );
14064 }
14065 }
14066
14067
14068 #if !USE_WINDOWS
14069 #define WINAPI
14070 #else
14071
14072 SERVICE_STATUS g_ss;
14073 SERVICE_STATUS_HANDLE g_ssHandle;
14074
14075
MySetServiceStatus(DWORD dwCurrentState,DWORD dwWin32ExitCode,DWORD dwWaitHint)14076 void MySetServiceStatus ( DWORD dwCurrentState, DWORD dwWin32ExitCode, DWORD dwWaitHint )
14077 {
14078 static DWORD dwCheckPoint = 1;
14079
14080 if ( dwCurrentState==SERVICE_START_PENDING )
14081 g_ss.dwControlsAccepted = 0;
14082 else
14083 g_ss.dwControlsAccepted = SERVICE_ACCEPT_STOP;
14084
14085 g_ss.dwCurrentState = dwCurrentState;
14086 g_ss.dwWin32ExitCode = dwWin32ExitCode;
14087 g_ss.dwWaitHint = dwWaitHint;
14088
14089 if ( dwCurrentState==SERVICE_RUNNING || dwCurrentState==SERVICE_STOPPED )
14090 g_ss.dwCheckPoint = 0;
14091 else
14092 g_ss.dwCheckPoint = dwCheckPoint++;
14093
14094 SetServiceStatus ( g_ssHandle, &g_ss );
14095 }
14096
14097
ServiceControl(DWORD dwControlCode)14098 void WINAPI ServiceControl ( DWORD dwControlCode )
14099 {
14100 switch ( dwControlCode )
14101 {
14102 case SERVICE_CONTROL_STOP:
14103 MySetServiceStatus ( SERVICE_STOP_PENDING, NO_ERROR, 0 );
14104 g_bServiceStop = true;
14105 break;
14106
14107 default:
14108 MySetServiceStatus ( g_ss.dwCurrentState, NO_ERROR, 0 );
14109 break;
14110 }
14111 }
14112
14113
14114 // warning! static buffer, non-reentrable
WinErrorInfo()14115 const char * WinErrorInfo ()
14116 {
14117 static char sBuf[1024];
14118
14119 DWORD uErr = ::GetLastError ();
14120 snprintf ( sBuf, sizeof(sBuf), "code=%d, error=", uErr );
14121
14122 int iLen = strlen(sBuf);
14123 if ( !FormatMessage ( FORMAT_MESSAGE_FROM_SYSTEM, NULL, uErr, 0, sBuf+iLen, sizeof(sBuf)-iLen, NULL ) ) // FIXME? force US-english langid?
14124 snprintf ( sBuf+iLen, sizeof(sBuf)-iLen, "(no message)" );
14125
14126 return sBuf;
14127 }
14128
14129
ServiceOpenManager()14130 SC_HANDLE ServiceOpenManager ()
14131 {
14132 SC_HANDLE hSCM = OpenSCManager (
14133 NULL, // local computer
14134 NULL, // ServicesActive database
14135 SC_MANAGER_ALL_ACCESS ); // full access rights
14136
14137 if ( hSCM==NULL )
14138 sphFatal ( "OpenSCManager() failed: %s", WinErrorInfo() );
14139
14140 return hSCM;
14141 }
14142
14143
AppendArg(char * sBuf,int iBufLimit,const char * sArg)14144 void AppendArg ( char * sBuf, int iBufLimit, const char * sArg )
14145 {
14146 char * sBufMax = sBuf + iBufLimit - 2; // reserve place for opening space and trailing zero
14147 sBuf += strlen(sBuf);
14148
14149 if ( sBuf>=sBufMax )
14150 return;
14151
14152 int iArgLen = strlen(sArg);
14153 bool bQuote = false;
14154 for ( int i=0; i<iArgLen && !bQuote; i++ )
14155 if ( sArg[i]==' ' || sArg[i]=='"' )
14156 bQuote = true;
14157
14158 *sBuf++ = ' ';
14159 if ( !bQuote )
14160 {
14161 // just copy
14162 int iToCopy = Min ( sBufMax-sBuf, iArgLen );
14163 memcpy ( sBuf, sArg, iToCopy );
14164 sBuf[iToCopy] = '\0';
14165
14166 } else
14167 {
14168 // quote
14169 sBufMax -= 2; // reserve place for quotes
14170 if ( sBuf>=sBufMax )
14171 return;
14172
14173 *sBuf++ = '"';
14174 while ( sBuf<sBufMax && *sArg )
14175 {
14176 if ( *sArg=='"' )
14177 {
14178 // quote
14179 if ( sBuf<sBufMax-1 )
14180 {
14181 *sBuf++ = '\\';
14182 *sBuf++ = *sArg++;
14183 }
14184 } else
14185 {
14186 // copy
14187 *sBuf++ = *sArg++;
14188 }
14189 }
14190 *sBuf++ = '"';
14191 *sBuf++ = '\0';
14192 }
14193 }
14194
14195
ServiceInstall(int argc,char ** argv)14196 void ServiceInstall ( int argc, char ** argv )
14197 {
14198 if ( g_bService )
14199 return;
14200
14201 sphInfo ( "Installing service..." );
14202
14203 char szBinary[MAX_PATH];
14204 if ( !GetModuleFileName ( NULL, szBinary, MAX_PATH ) )
14205 sphFatal ( "GetModuleFileName() failed: %s", WinErrorInfo() );
14206
14207 char szPath[MAX_PATH];
14208 szPath[0] = '\0';
14209
14210 AppendArg ( szPath, sizeof(szPath), szBinary );
14211 AppendArg ( szPath, sizeof(szPath), "--ntservice" );
14212 for ( int i=1; i<argc; i++ )
14213 if ( strcmp ( argv[i], "--install" ) )
14214 AppendArg ( szPath, sizeof(szPath), argv[i] );
14215
14216 SC_HANDLE hSCM = ServiceOpenManager ();
14217 SC_HANDLE hService = CreateService (
14218 hSCM, // SCM database
14219 g_sServiceName, // name of service
14220 g_sServiceName, // service name to display
14221 SERVICE_ALL_ACCESS, // desired access
14222 SERVICE_WIN32_OWN_PROCESS, // service type
14223 SERVICE_AUTO_START, // start type
14224 SERVICE_ERROR_NORMAL, // error control type
14225 szPath+1, // path to service's binary
14226 NULL, // no load ordering group
14227 NULL, // no tag identifier
14228 NULL, // no dependencies
14229 NULL, // LocalSystem account
14230 NULL ); // no password
14231
14232 if ( !hService )
14233 {
14234 CloseServiceHandle ( hSCM );
14235 sphFatal ( "CreateService() failed: %s", WinErrorInfo() );
14236
14237 } else
14238 {
14239 sphInfo ( "Service '%s' installed successfully.", g_sServiceName );
14240 }
14241
14242 CSphString sDesc;
14243 sDesc.SetSprintf ( "%s-%s", g_sServiceName, SPHINX_VERSION );
14244
14245 SERVICE_DESCRIPTION tDesc;
14246 tDesc.lpDescription = (LPSTR) sDesc.cstr();
14247 if ( !ChangeServiceConfig2 ( hService, SERVICE_CONFIG_DESCRIPTION, &tDesc ) )
14248 sphWarning ( "failed to set service description" );
14249
14250 CloseServiceHandle ( hService );
14251 CloseServiceHandle ( hSCM );
14252 }
14253
14254
ServiceDelete()14255 void ServiceDelete ()
14256 {
14257 if ( g_bService )
14258 return;
14259
14260 sphInfo ( "Deleting service..." );
14261
14262 // open manager
14263 SC_HANDLE hSCM = ServiceOpenManager ();
14264
14265 // open service
14266 SC_HANDLE hService = OpenService ( hSCM, g_sServiceName, DELETE );
14267 if ( !hService )
14268 {
14269 CloseServiceHandle ( hSCM );
14270 sphFatal ( "OpenService() failed: %s", WinErrorInfo() );
14271 }
14272
14273 // do delete
14274 bool bRes = !!DeleteService ( hService );
14275 CloseServiceHandle ( hService );
14276 CloseServiceHandle ( hSCM );
14277
14278 if ( !bRes )
14279 sphFatal ( "DeleteService() failed: %s", WinErrorInfo() );
14280 else
14281 sphInfo ( "Service '%s' deleted successfully.", g_sServiceName );
14282 }
14283 #endif // USE_WINDOWS
14284
14285
ShowHelp()14286 void ShowHelp ()
14287 {
14288 fprintf ( stdout,
14289 "Usage: searchd [OPTIONS]\n"
14290 "\n"
14291 "Options are:\n"
14292 "-h, --help\t\tdisplay this help message\n"
14293 "-c, --config <file>\tread configuration from specified file\n"
14294 "\t\t\t(default is sphinx.conf)\n"
14295 "--stop\t\t\tsend SIGTERM to currently running searchd\n"
14296 "--stopwait\t\tsend SIGTERM and wait until actual exit\n"
14297 "--status\t\tget ant print status variables\n"
14298 "\t\t\t(PID is taken from pid_file specified in config file)\n"
14299 "--iostats\t\tlog per-query io stats\n"
14300 #ifdef HAVE_CLOCK_GETTIME
14301 "--cpustats\t\tlog per-query cpu stats\n"
14302 #endif
14303 #if USE_WINDOWS
14304 "--install\t\tinstall as Windows service\n"
14305 "--delete\t\tdelete Windows service\n"
14306 "--servicename <name>\tuse given service name (default is 'searchd')\n"
14307 "--ntservice\t\tinternal option used to invoke a Windows service\n"
14308 #endif
14309 "--strip-path\t\tstrip paths from stopwords, wordforms, exceptions\n"
14310 "\t\t\tand other file names stored in the index header\n"
14311 "--replay-flags=<OPTIONS>\n"
14312 "\t\t\textra binary log replay options (the only current one\n"
14313 "\t\t\tis 'accept-desc-timestamp')\n"
14314 "\n"
14315 "Debugging options are:\n"
14316 "--console\t\trun in console mode (do not fork, do not log to files)\n"
14317 "-p, --port <port>\tlisten on given port (overrides config setting)\n"
14318 "-l, --listen <spec>\tlisten on given address, port or path (overrides\n"
14319 "\t\t\tconfig settings)\n"
14320 "-i, --index <index>\tonly serve one given index\n"
14321 #if !USE_WINDOWS
14322 "--nodetach\t\tdo not detach into background\n"
14323 #endif
14324 "--logdebug, --logdebugv, --logdebugvv\n"
14325 "\t\t\tenable additional debug information logging\n"
14326 "\t\t\t(with different verboseness)\n"
14327 "--pidfile\t\tforce using the PID file (useful with --console)\n"
14328 "--safetrace\t\tonly use system backtrace() call in crash reports\n"
14329 "\n"
14330 "Examples:\n"
14331 "searchd --config /usr/local/sphinx/etc/sphinx.conf\n"
14332 #if USE_WINDOWS
14333 "searchd --install --config c:\\sphinx\\sphinx.conf\n"
14334 #endif
14335 );
14336 }
14337
14338
14339 template<typename T>
InitSharedBuffer(CSphSharedBuffer<T> & tBuffer,int iLen)14340 T * InitSharedBuffer ( CSphSharedBuffer<T> & tBuffer, int iLen )
14341 {
14342 CSphString sError, sWarning;
14343 if ( !tBuffer.Alloc ( iLen, sError, sWarning ) )
14344 sphDie ( "failed to allocate shared buffer (msg=%s)", sError.cstr() );
14345
14346 T * pRes = tBuffer.GetWritePtr();
14347 memset ( pRes, 0, iLen*sizeof(T) ); // reset
14348 return pRes;
14349 }
14350
14351
14352 #if USE_WINDOWS
CtrlHandler(DWORD)14353 BOOL WINAPI CtrlHandler ( DWORD )
14354 {
14355 if ( !g_bService )
14356 {
14357 g_bGotSigterm = 1;
14358 sphInterruptNow();
14359 }
14360 return TRUE;
14361 }
14362 #endif
14363
14364
14365 #if !USE_WINDOWS
PreforkChild()14366 int PreforkChild ()
14367 {
14368 // next one
14369 int iRes = fork();
14370 if ( iRes==-1 )
14371 sphFatal ( "fork() failed during prefork (error=%s)", strerror(errno) );
14372
14373 // child process
14374 if ( iRes==0 )
14375 {
14376 g_bHeadDaemon = false;
14377 sphSetProcessInfo ( false );
14378 return iRes;
14379 }
14380
14381 // parent process
14382 g_dChildren.Add ( iRes );
14383 return iRes;
14384 }
14385
14386
14387 // returns 'true' only once - at the very start, to show it beatiful way.
SetWatchDog(int iDevNull)14388 bool SetWatchDog ( int iDevNull )
14389 {
14390 InitSharedBuffer ( g_bDaemonAtShutdown, 1 );
14391
14392 // Fork #1 - detach from controlling terminal
14393 switch ( fork() )
14394 {
14395 case -1:
14396 // error
14397 Shutdown ();
14398 sphFatal ( "fork() failed (reason: %s)", strerror ( errno ) );
14399 exit ( 1 );
14400 case 0:
14401 // daemonized child - or new and free watchdog :)
14402 break;
14403
14404 default:
14405 // tty-controlled parent
14406 while ( g_tHaveTTY.ReadValue() )
14407 sphSleepMsec ( 100 );
14408
14409 sphSetProcessInfo ( false );
14410 exit ( 0 );
14411 }
14412
14413 // became the session leader
14414 if ( setsid()==-1 )
14415 {
14416 Shutdown ();
14417 sphFatal ( "setsid() failed (reason: %s)", strerror ( errno ) );
14418 exit ( 1 );
14419 }
14420
14421 // Fork #2 - detach from session leadership (may be not necessary, however)
14422 switch ( fork() )
14423 {
14424 case -1:
14425 // error
14426 Shutdown ();
14427 sphFatal ( "fork() failed (reason: %s)", strerror ( errno ) );
14428 exit ( 1 );
14429 case 0:
14430 // daemonized child - or new and free watchdog :)
14431 break;
14432
14433 default:
14434 // tty-controlled parent
14435 sphSetProcessInfo ( false );
14436 exit ( 0 );
14437 }
14438
14439 // now we are the watchdog. Let us fork the actual process
14440 int iReincarnate = 1;
14441 bool bShutdown = false;
14442 bool bStreamsActive = true;
14443 int iRes = 0;
14444 for ( ;; )
14445 {
14446 if ( iReincarnate!=0 )
14447 iRes = fork();
14448
14449 if ( iRes==-1 )
14450 {
14451 Shutdown ();
14452 sphFatal ( "fork() failed during watchdog setup (error=%s)", strerror(errno) );
14453 }
14454
14455 // child process; return true to show that we have to reload everything
14456 if ( iRes==0 )
14457 {
14458 atexit ( &ReleaseTTYFlag );
14459 return bStreamsActive;
14460 }
14461
14462 // parent process, watchdog
14463 // close the io files
14464 if ( bStreamsActive )
14465 {
14466 close ( STDIN_FILENO );
14467 close ( STDOUT_FILENO );
14468 close ( STDERR_FILENO );
14469 dup2 ( iDevNull, STDIN_FILENO );
14470 dup2 ( iDevNull, STDOUT_FILENO );
14471 dup2 ( iDevNull, STDERR_FILENO );
14472 bStreamsActive = false;
14473 }
14474
14475 sphInfo ( "Child process %d has been forked", iRes );
14476
14477 SetSignalHandlers();
14478
14479 iReincarnate = 0;
14480 int iPid, iStatus;
14481 bool bDaemonAtShutdown = 0;
14482 while ( ( iPid = wait ( &iStatus ) )>0 )
14483 {
14484 bDaemonAtShutdown = ( g_bDaemonAtShutdown[0]!=0 );
14485 const char * sWillRestart = ( bDaemonAtShutdown ? "will not be restarted ( daemon is shutting down )" : "will be restarted" );
14486
14487 assert ( iPid==iRes );
14488 if ( WIFEXITED ( iStatus ) )
14489 {
14490 int iExit = WEXITSTATUS ( iStatus );
14491 if ( iExit==2 || iExit==6 ) // really crash
14492 {
14493 sphInfo ( "Child process %d has been finished by CRASH_EXIT (exit code %d), %s", iPid, iExit, sWillRestart );
14494 iReincarnate = -1;
14495 } else
14496 {
14497 sphInfo ( "Child process %d has been finished, exit code %d. Watchdog finishes also. Good bye!", iPid, iExit );
14498 bShutdown = true;
14499 }
14500 } else if ( WIFSIGNALED ( iStatus ) )
14501 {
14502 if ( WTERMSIG ( iStatus )==SIGINT || WTERMSIG ( iStatus )==SIGTERM
14503 #if WATCHDOG_SIGKILL
14504 || WTERMSIG ( iStatus )==SIGKILL
14505 #endif
14506 )
14507 {
14508 sphInfo ( "Child process %d has been killed with kill or sigterm (%i). Watchdog finishes also. Good bye!", iPid, WTERMSIG ( iStatus ) );
14509 bShutdown = true;
14510 } else
14511 {
14512 if ( WCOREDUMP ( iStatus ) )
14513 sphInfo ( "Child process %i has been killed with signal %i, core dumped, %s", iPid, WTERMSIG ( iStatus ), sWillRestart );
14514 else
14515 sphInfo ( "Child process %i has been killed with signal %i, %s", iPid, WTERMSIG ( iStatus ), sWillRestart );
14516 iReincarnate = -1;
14517 }
14518 } else if ( WIFSTOPPED ( iStatus ) )
14519 sphInfo ( "Child %i stopped with signal %i", iPid, WSTOPSIG ( iStatus ) );
14520 #ifdef WIFCONTINUED
14521 else if ( WIFCONTINUED ( iStatus ) )
14522 sphInfo ( "Child %i resumed", iPid );
14523 #endif
14524 }
14525
14526 if ( bShutdown || g_bGotSigterm || bDaemonAtShutdown )
14527 {
14528 Shutdown();
14529 exit ( 0 );
14530 }
14531 }
14532 }
14533 #endif // !USE_WINDOWS
14534
14535 /// check for incoming signals, and react on them
CheckSignals()14536 void CheckSignals ()
14537 {
14538 #if USE_WINDOWS
14539 if ( g_bService && g_bServiceStop )
14540 {
14541 Shutdown ();
14542 MySetServiceStatus ( SERVICE_STOPPED, NO_ERROR, 0 );
14543 exit ( 0 );
14544 }
14545 #endif
14546
14547 if ( g_bGotSighup )
14548 {
14549 g_tRotateQueueMutex.Lock();
14550 g_iRotateCount++;
14551 g_tRotateQueueMutex.Unlock();
14552 sphInfo ( "caught SIGHUP (seamless=%d, in queue=%d)", (int)g_bSeamlessRotate, g_iRotateCount );
14553 g_bGotSighup = 0;
14554 }
14555
14556 if ( g_bGotSigterm )
14557 {
14558 assert ( g_bHeadDaemon );
14559 sphInfo ( "caught SIGTERM, shutting down" );
14560 Shutdown ();
14561 exit ( 0 );
14562 }
14563
14564 #if !USE_WINDOWS
14565 if ( g_bGotSigchld )
14566 {
14567 // handle gone children
14568 for ( ;; )
14569 {
14570 int iChildPid = waitpid ( -1, NULL, WNOHANG );
14571 sphLogDebugvv ( "gone child %d ( %d )", iChildPid, g_dChildren.GetLength() ); // !COMMIT
14572 if ( iChildPid<=0 )
14573 break;
14574
14575 g_dChildren.RemoveValue ( iChildPid ); // FIXME! OPTIMIZE! can be slow
14576 }
14577 g_bGotSigchld = 0;
14578
14579 // prefork more children, if needed
14580 if ( g_eWorkers==MPM_PREFORK )
14581 while ( g_dChildren.GetLength() < g_iPreforkChildren )
14582 if ( PreforkChild()==0 ) // child process? break from here, go work
14583 return;
14584 }
14585 #endif
14586
14587 #if USE_WINDOWS
14588 BYTE dPipeInBuf [ WIN32_PIPE_BUFSIZE ];
14589 DWORD nBytesRead = 0;
14590 BOOL bSuccess = ReadFile ( g_hPipe, dPipeInBuf, WIN32_PIPE_BUFSIZE, &nBytesRead, NULL );
14591 if ( nBytesRead > 0 && bSuccess )
14592 {
14593 for ( DWORD i=0; i<nBytesRead; i++ )
14594 {
14595 switch ( dPipeInBuf[i] )
14596 {
14597 case 0:
14598 g_bGotSighup = 1;
14599 break;
14600
14601 case 1:
14602 g_bGotSigterm = 1;
14603 sphInterruptNow();
14604 if ( g_bService )
14605 g_bServiceStop = true;
14606 break;
14607 }
14608 }
14609
14610 DisconnectNamedPipe ( g_hPipe );
14611 ConnectNamedPipe ( g_hPipe, NULL );
14612 }
14613 #endif
14614 }
14615
14616
QueryStatus(CSphVariant * v)14617 void QueryStatus ( CSphVariant * v )
14618 {
14619 char sBuf [ SPH_ADDRESS_SIZE ];
14620
14621 for ( ; v; v = v->m_pNext )
14622 {
14623 ListenerDesc_t tDesc = ParseListener ( v->cstr() );
14624 if ( tDesc.m_eProto!=PROTO_SPHINX )
14625 continue;
14626
14627 int iSock = -1;
14628 #if !USE_WINDOWS
14629 if ( !tDesc.m_sUnix.IsEmpty() )
14630 {
14631 // UNIX connection
14632 struct sockaddr_un uaddr;
14633
14634 size_t len = strlen ( tDesc.m_sUnix.cstr() );
14635 if ( len+1 > sizeof(uaddr.sun_path ) )
14636 sphFatal ( "UNIX socket path is too long (len=%d)", (int)len );
14637
14638 memset ( &uaddr, 0, sizeof(uaddr) );
14639 uaddr.sun_family = AF_UNIX;
14640 memcpy ( uaddr.sun_path, tDesc.m_sUnix.cstr(), len+1 );
14641
14642 iSock = socket ( AF_UNIX, SOCK_STREAM, 0 );
14643 if ( iSock<0 )
14644 sphFatal ( "failed to create UNIX socket: %s", sphSockError() );
14645
14646 if ( connect ( iSock, (struct sockaddr*)&uaddr, sizeof(uaddr) )<0 )
14647 {
14648 sphWarning ( "failed to connect to unix://%s: %s\n", tDesc.m_sUnix.cstr(), sphSockError() );
14649 continue;
14650 }
14651
14652 } else
14653 #endif
14654 {
14655 // TCP connection
14656 struct sockaddr_in sin;
14657 memset ( &sin, 0, sizeof(sin) );
14658 sin.sin_family = AF_INET;
14659 sin.sin_addr.s_addr = ( tDesc.m_uIP==htonl ( INADDR_ANY ) )
14660 ? htonl ( INADDR_LOOPBACK )
14661 : tDesc.m_uIP;
14662 sin.sin_port = htons ( (short)tDesc.m_iPort );
14663
14664 iSock = socket ( AF_INET, SOCK_STREAM, 0 );
14665 if ( iSock<0 )
14666 sphFatal ( "failed to create TCP socket: %s", sphSockError() );
14667
14668 if ( connect ( iSock, (struct sockaddr*)&sin, sizeof(sin) )<0 )
14669 {
14670 sphWarning ( "failed to connect to %s:%d: %s\n", sphFormatIP ( sBuf, sizeof(sBuf), tDesc.m_uIP ), tDesc.m_iPort, sphSockError() );
14671 continue;
14672 }
14673 }
14674
14675 // send request
14676 NetOutputBuffer_c tOut ( iSock );
14677 tOut.SendDword ( SPHINX_SEARCHD_PROTO );
14678 tOut.SendWord ( SEARCHD_COMMAND_STATUS );
14679 tOut.SendWord ( VER_COMMAND_STATUS );
14680 tOut.SendInt ( 4 ); // request body length
14681 tOut.SendInt ( 1 ); // dummy body
14682 tOut.Flush ();
14683
14684 // get reply
14685 NetInputBuffer_c tIn ( iSock );
14686 if ( !tIn.ReadFrom ( 12, 5 ) ) // magic_header_size=12, magic_timeout=5
14687 sphFatal ( "handshake failure (no response)" );
14688
14689 DWORD uVer = tIn.GetDword();
14690 if ( uVer!=SPHINX_SEARCHD_PROTO && uVer!=0x01000000UL ) // workaround for all the revisions that sent it in host order...
14691 sphFatal ( "handshake failure (unexpected protocol version=%d)", uVer );
14692
14693 if ( tIn.GetWord()!=SEARCHD_OK )
14694 sphFatal ( "status command failed" );
14695
14696 if ( tIn.GetWord()!=VER_COMMAND_STATUS )
14697 sphFatal ( "status command version mismatch" );
14698
14699 if ( !tIn.ReadFrom ( tIn.GetDword(), 5 ) ) // magic_timeout=5
14700 sphFatal ( "failed to read status reply" );
14701
14702 fprintf ( stdout, "\nsearchd status\n--------------\n" );
14703
14704 int iRows = tIn.GetDword();
14705 int iCols = tIn.GetDword();
14706 for ( int i=0; i<iRows && !tIn.GetError(); i++ )
14707 {
14708 for ( int j=0; j<iCols && !tIn.GetError(); j++ )
14709 {
14710 fprintf ( stdout, "%s", tIn.GetString().cstr() );
14711 fprintf ( stdout, ( j==0 ) ? ": " : " " );
14712 }
14713 fprintf ( stdout, "\n" );
14714 }
14715
14716 // all done
14717 sphSockClose ( iSock );
14718 return;
14719 }
14720 }
14721
14722
ShowProgress(const CSphIndexProgress * pProgress,bool bPhaseEnd)14723 void ShowProgress ( const CSphIndexProgress * pProgress, bool bPhaseEnd )
14724 {
14725 assert ( pProgress );
14726 if ( bPhaseEnd )
14727 {
14728 fprintf ( stdout, "\r \r" );
14729 } else
14730 {
14731 fprintf ( stdout, "%s\r", pProgress->BuildMessage() );
14732 }
14733 fflush ( stdout );
14734 }
14735
14736
FailClient(int iSock,SearchdStatus_e eStatus,const char * sMessage)14737 void FailClient ( int iSock, SearchdStatus_e eStatus, const char * sMessage )
14738 {
14739 assert ( eStatus==SEARCHD_RETRY || eStatus==SEARCHD_ERROR );
14740
14741 int iRespLen = 4 + strlen(sMessage);
14742
14743 NetOutputBuffer_c tOut ( iSock );
14744 tOut.SendInt ( SPHINX_SEARCHD_PROTO );
14745 tOut.SendWord ( (WORD)eStatus );
14746 tOut.SendWord ( 0 ); // version doesn't matter
14747 tOut.SendInt ( iRespLen );
14748 tOut.SendString ( sMessage );
14749 tOut.Flush ();
14750
14751 // FIXME? without some wait, client fails to receive the response on windows
14752 sphSockClose ( iSock );
14753 }
14754
14755
DoAccept(int * pClientSock,char * sClientName)14756 Listener_t * DoAccept ( int * pClientSock, char * sClientName )
14757 {
14758 int iMaxFD = 0;
14759 fd_set fdsAccept;
14760 FD_ZERO ( &fdsAccept );
14761
14762 ARRAY_FOREACH ( i, g_dListeners )
14763 {
14764 sphFDSet ( g_dListeners[i].m_iSock, &fdsAccept );
14765 iMaxFD = Max ( iMaxFD, g_dListeners[i].m_iSock );
14766 }
14767 iMaxFD++;
14768
14769 struct timeval tvTimeout;
14770 tvTimeout.tv_sec = USE_WINDOWS ? 0 : 1;
14771 tvTimeout.tv_usec = USE_WINDOWS ? 50000 : 0;
14772
14773 int iRes = select ( iMaxFD, &fdsAccept, NULL, NULL, &tvTimeout );
14774 if ( iRes==0 )
14775 return NULL;
14776
14777 if ( iRes<0 )
14778 {
14779 int iErrno = sphSockGetErrno();
14780 if ( iErrno==EINTR || iErrno==EAGAIN || iErrno==EWOULDBLOCK )
14781 return NULL;
14782
14783 static int iLastErrno = -1;
14784 if ( iLastErrno!=iErrno )
14785 sphWarning ( "select() failed: %s", sphSockError(iErrno) );
14786 iLastErrno = iErrno;
14787 return NULL;
14788 }
14789
14790 ARRAY_FOREACH ( i, g_dListeners )
14791 {
14792 if ( !FD_ISSET ( g_dListeners[i].m_iSock, &fdsAccept ) )
14793 continue;
14794
14795 // accept
14796 struct sockaddr_storage saStorage;
14797 socklen_t uLength = sizeof(saStorage);
14798 int iClientSock = accept ( g_dListeners[i].m_iSock, (struct sockaddr *)&saStorage, &uLength );
14799
14800 // handle failures
14801 if ( iClientSock<0 )
14802 {
14803 const int iErrno = sphSockGetErrno();
14804 if ( iErrno==EINTR || iErrno==ECONNABORTED || iErrno==EAGAIN || iErrno==EWOULDBLOCK )
14805 return NULL;
14806
14807 sphFatal ( "accept() failed: %s", sphSockError(iErrno) );
14808 }
14809
14810 if ( g_pStats )
14811 {
14812 g_tStatsMutex.Lock();
14813 g_pStats->m_iConnections++;
14814 g_tStatsMutex.Unlock();
14815 }
14816
14817 if ( g_eWorkers==MPM_PREFORK )
14818 {
14819 // protected by accept mutex
14820 if ( ++*g_pConnID<0 )
14821 *g_pConnID = 0;
14822 g_iConnID = *g_pConnID;
14823 } else
14824 {
14825 if ( ++g_iConnID<0 )
14826 g_iConnID = 0;
14827 }
14828
14829 // format client address
14830 if ( sClientName )
14831 {
14832 sClientName[0] = '\0';
14833 if ( saStorage.ss_family==AF_INET )
14834 {
14835 struct sockaddr_in * pSa = ((struct sockaddr_in *)&saStorage);
14836 sphFormatIP ( sClientName, SPH_ADDRESS_SIZE, pSa->sin_addr.s_addr );
14837
14838 char * d = sClientName;
14839 while ( *d )
14840 d++;
14841 snprintf ( d, 7, ":%d", (int)ntohs ( pSa->sin_port ) ); //NOLINT
14842 }
14843 if ( saStorage.ss_family==AF_UNIX )
14844 strncpy ( sClientName, "(local)", SPH_ADDRESS_SIZE );
14845 }
14846
14847 // accepted!
14848 #if !USE_WINDOWS
14849 // FIXME!!! either get git of select() or allocate list of FD (with dup2 back instead close for thouse FD)
14850 // with threads workers to prevent dup2 closes valid FD
14851
14852 if ( SPH_FDSET_OVERFLOW ( iClientSock ) )
14853 {
14854 if ( ( g_eWorkers==MPM_FORK || g_eWorkers==MPM_PREFORK ) )
14855 {
14856 iClientSock = dup2 ( iClientSock, g_iClientFD );
14857 } else
14858 {
14859 FailClient ( iClientSock, SEARCHD_RETRY, "server maxed out, retry in a second" );
14860 sphWarning ( "maxed out, dismissing client (socket=%d)", iClientSock );
14861 sphSockClose ( iClientSock );
14862 return NULL;
14863 }
14864 }
14865 #endif
14866
14867 *pClientSock = iClientSock;
14868 return &g_dListeners[i];
14869 }
14870
14871 return NULL;
14872 }
14873
14874
TickPreforked(CSphProcessSharedMutex * pAcceptMutex)14875 void TickPreforked ( CSphProcessSharedMutex * pAcceptMutex )
14876 {
14877 assert ( !g_bHeadDaemon );
14878 assert ( pAcceptMutex );
14879
14880 if ( g_bGotSigterm || g_bGotSighup )
14881 exit ( 0 );
14882
14883 int iClientSock = -1;
14884 char sClientIP[SPH_ADDRPORT_SIZE];
14885 Listener_t * pListener = NULL;
14886
14887 for ( ; !g_bGotSigterm && !pListener && !g_bGotSighup; )
14888 {
14889 if ( pAcceptMutex->TimedLock ( 100 ) )
14890 {
14891 if ( !g_bGotSigterm && !g_bGotSighup )
14892 pListener = DoAccept ( &iClientSock, sClientIP );
14893
14894 pAcceptMutex->Unlock();
14895 }
14896 }
14897
14898 if ( g_bGotSigterm )
14899 exit ( 0 ); // clean shutdown (after mutex unlock)
14900
14901 if ( pListener )
14902 {
14903 HandleClient ( pListener->m_eProto, iClientSock, sClientIP, NULL );
14904 sphSockClose ( iClientSock );
14905 }
14906 }
14907
14908
HandlerThread(void * pArg)14909 void HandlerThread ( void * pArg )
14910 {
14911 // setup query guard for threaded mode
14912 SphCrashLogger_c tQueryTLS;
14913 tQueryTLS.SetupTLS ();
14914
14915 // handle that client
14916 ThdDesc_t * pThd = (ThdDesc_t*) pArg;
14917 sphThreadSet ( g_tConnKey, &pThd->m_iConnID );
14918 HandleClient ( pThd->m_eProto, pThd->m_iClientSock, pThd->m_sClientName.cstr(), pThd );
14919 sphSockClose ( pThd->m_iClientSock );
14920
14921 // done; remove myself from the table
14922 g_tThdMutex.Lock ();
14923 ARRAY_FOREACH ( i, g_dThd )
14924 if ( g_dThd[i]==pThd )
14925 {
14926 #if USE_WINDOWS
14927 // FIXME? this is sort of automatic on UNIX (pthread_exit() gets implicitly called on return)
14928 CloseHandle ( pThd->m_tThd );
14929 #endif
14930 SafeDelete ( pThd );
14931 g_dThd.RemoveFast(i);
14932 break;
14933 }
14934 g_tThdMutex.Unlock ();
14935
14936 // something went wrong while removing; report
14937 if ( pThd )
14938 {
14939 sphWarning ( "thread missing from thread table" );
14940 #if USE_WINDOWS
14941 // FIXME? this is sort of automatic on UNIX (pthread_exit() gets implicitly called on return)
14942 CloseHandle ( pThd->m_tThd );
14943 #endif
14944 SafeDelete ( pThd );
14945 }
14946 }
14947
14948
CheckChildrenHup()14949 static void CheckChildrenHup ()
14950 {
14951 #if !USE_WINDOWS
14952 if ( g_eWorkers!=MPM_PREFORK || !g_dHupChildren.GetLength() || g_tmRotateChildren>sphMicroTimer() )
14953 return;
14954
14955 sphLogDebugvv ( "sending sighup to child %d ( %d )", g_dHupChildren.Last(), g_dHupChildren.GetLength() );
14956 kill ( g_dHupChildren.Pop(), SIGHUP );
14957 g_tmRotateChildren = sphMicroTimer() + g_iRotationThrottle*1000;
14958 #endif
14959 }
14960
14961
TickHead(CSphProcessSharedMutex * pAcceptMutex)14962 void TickHead ( CSphProcessSharedMutex * pAcceptMutex )
14963 {
14964 CheckSignals ();
14965 if ( !g_bHeadDaemon )
14966 return;
14967
14968 CheckLeaks ();
14969 CheckReopen ();
14970 CheckPipes ();
14971 CheckDelete ();
14972 CheckRotate ();
14973 CheckFlush ();
14974 CheckChildrenHup();
14975
14976 sphInfo ( NULL ); // flush dupes
14977
14978 if ( pAcceptMutex )
14979 {
14980 // FIXME! what if all children are busy; we might want to accept here and temp fork more
14981 sphSleepMsec ( 1000 );
14982 return;
14983 }
14984
14985 int iClientSock;
14986 char sClientName[SPH_ADDRPORT_SIZE];
14987 Listener_t * pListener = DoAccept ( &iClientSock, sClientName );
14988 if ( !pListener )
14989 return;
14990
14991 if ( ( g_iMaxChildren && ( g_dChildren.GetLength()>=g_iMaxChildren || g_dThd.GetLength()>=g_iMaxChildren ) )
14992 || ( g_iRotateCount && !g_bSeamlessRotate ) )
14993 {
14994 FailClient ( iClientSock, SEARCHD_RETRY, "server maxed out, retry in a second" );
14995 sphWarning ( "maxed out, dismissing client" );
14996
14997 if ( g_pStats )
14998 g_pStats->m_iMaxedOut++;
14999 return;
15000 }
15001
15002 // handle the client
15003 if ( g_eWorkers==MPM_NONE )
15004 {
15005 HandleClient ( pListener->m_eProto, iClientSock, sClientName, NULL );
15006 sphSockClose ( iClientSock );
15007 return;
15008 }
15009
15010 #if !USE_WINDOWS
15011 if ( g_eWorkers==MPM_FORK )
15012 {
15013 sphLogDebugv ( "conn %s: accepted, socket %d", sClientName, iClientSock );
15014 int iChildPipe = PipeAndFork ( false, -1 );
15015 SafeClose ( iChildPipe );
15016 if ( !g_bHeadDaemon )
15017 {
15018 // child process, handle client
15019 sphLogDebugv ( "conn %s: forked handler, socket %d", sClientName, iClientSock );
15020 HandleClient ( pListener->m_eProto, iClientSock, sClientName, NULL );
15021 sphSockClose ( iClientSock );
15022 exit ( 0 );
15023 } else
15024 {
15025 // parent process, continue accept()ing
15026 sphSockClose ( iClientSock );
15027 }
15028 }
15029 #endif // !USE_WINDOWS
15030
15031 if ( g_eWorkers==MPM_THREADS )
15032 {
15033 ThdDesc_t * pThd = new ThdDesc_t ();
15034 pThd->m_eProto = pListener->m_eProto;
15035 pThd->m_iClientSock = iClientSock;
15036 pThd->m_sClientName = sClientName;
15037 pThd->m_iConnID = g_iConnID;
15038
15039 g_tThdMutex.Lock ();
15040 g_dThd.Add ( pThd );
15041 if ( !sphThreadCreate ( &pThd->m_tThd, HandlerThread, pThd, true ) )
15042 {
15043 int iErr = errno;
15044 g_dThd.Pop();
15045 SafeDelete ( pThd );
15046
15047 FailClient ( iClientSock, SEARCHD_RETRY, "failed to create worker thread" );
15048 sphWarning ( "failed to create worker thread, threads(%d), error[%d] %s", g_dThd.GetLength(), iErr, strerror(iErr) );
15049 }
15050 g_tThdMutex.Unlock ();
15051 return;
15052 }
15053
15054 // default (should not happen)
15055 sphSockClose ( iClientSock );
15056 }
15057
15058
ConfigureSearchd(const CSphConfig & hConf,bool bOptPIDFile)15059 void ConfigureSearchd ( const CSphConfig & hConf, bool bOptPIDFile )
15060 {
15061 if ( !hConf.Exists ( "searchd" ) || !hConf["searchd"].Exists ( "searchd" ) )
15062 sphFatal ( "'searchd' config section not found in '%s'", g_sConfigFile.cstr () );
15063
15064 const CSphConfigSection & hSearchd = hConf["searchd"]["searchd"];
15065
15066 if ( !hConf.Exists ( "index" ) )
15067 sphFatal ( "no indexes found in '%s'", g_sConfigFile.cstr () );
15068
15069 if ( bOptPIDFile )
15070 if ( !hSearchd ( "pid_file" ) )
15071 sphFatal ( "mandatory option 'pid_file' not found in 'searchd' section" );
15072
15073 if ( hSearchd.Exists ( "read_timeout" ) && hSearchd["read_timeout"].intval()>=0 )
15074 g_iReadTimeout = hSearchd["read_timeout"].intval();
15075
15076 if ( hSearchd.Exists ( "client_timeout" ) && hSearchd["client_timeout"].intval()>=0 )
15077 g_iClientTimeout = hSearchd["client_timeout"].intval();
15078
15079 if ( hSearchd.Exists ( "max_children" ) && hSearchd["max_children"].intval()>=0 )
15080 g_iMaxChildren = hSearchd["max_children"].intval();
15081
15082 g_bPreopenIndexes = hSearchd.GetInt ( "preopen_indexes", (int)g_bPreopenIndexes )!=0;
15083 g_bOnDiskDicts = hSearchd.GetInt ( "ondisk_dict_default", (int)g_bOnDiskDicts )!=0;
15084 sphSetUnlinkOld ( hSearchd.GetInt ( "unlink_old", 1 )!=0 );
15085 g_iExpansionLimit = hSearchd.GetInt ( "expansion_limit", 0 );
15086 g_bCompatResults = hSearchd.GetInt ( "compat_sphinxql_magics", (int)g_bCompatResults )!=0;
15087
15088 if ( g_bCompatResults )
15089 sphWarning ( "compat_sphinxql_magics=1 is deprecated; please update your application and config" );
15090
15091 if ( hSearchd("max_matches") )
15092 {
15093 int iMax = hSearchd["max_matches"].intval();
15094 if ( iMax<0 || iMax>10000000 )
15095 {
15096 sphWarning ( "max_matches=%d out of bounds; using default 1000", iMax );
15097 } else
15098 {
15099 g_iMaxMatches = iMax;
15100 }
15101 }
15102
15103 if ( hSearchd("subtree_docs_cache") )
15104 g_iMaxCachedDocs = hSearchd.GetSize ( "subtree_docs_cache", g_iMaxCachedDocs );
15105
15106 if ( hSearchd("subtree_hits_cache") )
15107 g_iMaxCachedHits = hSearchd.GetSize ( "subtree_hits_cache", g_iMaxCachedHits );
15108
15109 if ( hSearchd("seamless_rotate") )
15110 g_bSeamlessRotate = ( hSearchd["seamless_rotate"].intval()!=0 );
15111
15112 if ( !g_bSeamlessRotate && g_bPreopenIndexes )
15113 sphWarning ( "preopen_indexes=1 has no effect with seamless_rotate=0" );
15114
15115 g_iAttrFlushPeriod = hSearchd.GetInt ( "attr_flush_period", g_iAttrFlushPeriod );
15116 g_iMaxPacketSize = hSearchd.GetSize ( "max_packet_size", g_iMaxPacketSize );
15117 g_iMaxFilters = hSearchd.GetInt ( "max_filters", g_iMaxFilters );
15118 g_iMaxFilterValues = hSearchd.GetInt ( "max_filter_values", g_iMaxFilterValues );
15119 g_iMaxBatchQueries = hSearchd.GetInt ( "max_batch_queries", g_iMaxBatchQueries );
15120 g_iDistThreads = hSearchd.GetInt ( "dist_threads", g_iDistThreads );
15121 g_iPreforkChildren = hSearchd.GetInt ( "prefork", g_iPreforkChildren );
15122
15123 if ( hSearchd ( "collation_libc_locale" ) )
15124 {
15125 const char * sLocale = hSearchd.GetStr ( "collation_libc_locale" );
15126 if ( !setlocale ( LC_COLLATE, sLocale ) )
15127 sphWarning ( "setlocale failed (locale='%s')", sLocale );
15128 }
15129
15130 if ( hSearchd ( "collation_server" ) )
15131 {
15132 CSphString sCollation = hSearchd.GetStr ( "collation_server" );
15133 CSphString sError;
15134 g_eCollation = sphCollationFromName ( sCollation, &sError );
15135 if ( !sError.IsEmpty() )
15136 sphWarning ( "%s", sError.cstr() );
15137 }
15138
15139 if ( hSearchd("thread_stack") )
15140 {
15141 int iThreadStackSizeMin = 65536;
15142 int iThreadStackSizeMax = 2*1024*1024;
15143 int iStackSize = hSearchd.GetSize ( "thread_stack", iThreadStackSizeMin );
15144 if ( iStackSize<iThreadStackSizeMin || iStackSize>iThreadStackSizeMax )
15145 sphWarning ( "thread_stack is %d will be clamped to range ( 65k to 2M )", iStackSize );
15146
15147 iStackSize = Min ( iStackSize, iThreadStackSizeMax );
15148 iStackSize = Max ( iStackSize, iThreadStackSizeMin );
15149 sphSetMyStackSize ( iStackSize );
15150 }
15151
15152 char sHandshake1[] =
15153 "\x00\x00\x00" // packet length
15154 "\x00" // packet id
15155 "\x0A"; // protocol version; v.10
15156
15157 char sHandshake2[] =
15158 "\x01\x00\x00\x00" // thread id
15159 "\x01\x02\x03\x04\x05\x06\x07\x08" // scramble buffer (for auth)
15160 "\x00" // filler
15161 "\x08\x82" // server capabilities; CLIENT_PROTOCOL_41 | CLIENT_CONNECT_WITH_DB | SECURE_CONNECTION
15162 "\x21" // server language; let it be ut8_general_ci to make different clients happy
15163 "\x02\x00" // server status
15164 "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" // filler
15165 "\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d"; // scramble buffer2 (for auth, 4.1+)
15166
15167 const char * sVersion = hSearchd.GetStr ( "mysql_version_string", SPHINX_VERSION );
15168 int iLen = strlen ( sVersion );
15169
15170 g_iMysqlHandshake = sizeof(sHandshake1) + strlen(sVersion) + sizeof(sHandshake2) - 1;
15171 if ( g_iMysqlHandshake>=(int)sizeof(g_sMysqlHandshake) )
15172 {
15173 sphWarning ( "mysql_version_string too long; using default (version=%s)", SPHINX_VERSION );
15174 g_iMysqlHandshake = sizeof(sHandshake1) + strlen(SPHINX_VERSION) + sizeof(sHandshake2) - 1;
15175 assert ( g_iMysqlHandshake < (int)sizeof(g_sMysqlHandshake) );
15176 }
15177
15178 char * p = g_sMysqlHandshake;
15179 memcpy ( p, sHandshake1, sizeof(sHandshake1)-1 );
15180 memcpy ( p+sizeof(sHandshake1)-1, sVersion, iLen+1 );
15181 memcpy ( p+sizeof(sHandshake1)+iLen, sHandshake2, sizeof(sHandshake2)-1 );
15182 g_sMysqlHandshake[0] = (char)(g_iMysqlHandshake-4); // safe, as long as buffer size is 128
15183 }
15184
ConfigureAndPreload(const CSphConfig & hConf,const char * sOptIndex)15185 void ConfigureAndPreload ( const CSphConfig & hConf, const char * sOptIndex )
15186 {
15187 int iCounter = 1;
15188 int iValidIndexes = 0;
15189 int64_t tmLoad = -sphMicroTimer();
15190
15191 hConf["index"].IterateStart ();
15192 while ( hConf["index"].IterateNext() )
15193 {
15194 const CSphConfigSection & hIndex = hConf["index"].IterateGet();
15195 const char * sIndexName = hConf["index"].IterateGetKey().cstr();
15196
15197 if ( g_bOptNoDetach && sOptIndex && strcasecmp ( sIndexName, sOptIndex )!=0 )
15198 continue;
15199
15200 ESphAddIndex eAdd = AddIndex ( sIndexName, hIndex );
15201 if ( eAdd==ADD_LOCAL || eAdd==ADD_RT )
15202 {
15203 ServedIndex_t & tIndex = g_pIndexes->GetUnlockedEntry ( sIndexName );
15204 iCounter++;
15205
15206 fprintf ( stdout, "precaching index '%s'\n", sIndexName );
15207 fflush ( stdout );
15208 tIndex.m_pIndex->SetProgressCallback ( ShowProgress );
15209
15210 if ( HasFiles ( tIndex, g_dNewExts ) )
15211 {
15212 tIndex.m_bOnlyNew = !HasFiles ( tIndex, g_dCurExts );
15213 if ( RotateIndexGreedy ( tIndex, sIndexName ) )
15214 {
15215 CSphString sError;
15216 if ( !sphFixupIndexSettings ( tIndex.m_pIndex, hIndex, sError ) )
15217 {
15218 sphWarning ( "index '%s': %s - NOT SERVING", sIndexName, sError.cstr() );
15219 tIndex.m_bEnabled = false;
15220 }
15221 } else
15222 {
15223 if ( PrereadNewIndex ( tIndex, hIndex, sIndexName ) )
15224 tIndex.m_bEnabled = true;
15225 }
15226 } else
15227 {
15228 tIndex.m_bOnlyNew = false;
15229 if ( PrereadNewIndex ( tIndex, hIndex, sIndexName ) )
15230 tIndex.m_bEnabled = true;
15231 }
15232
15233 CSphString sError;
15234 if ( tIndex.m_bEnabled && !CheckIndex ( tIndex.m_pIndex, sError ) )
15235 {
15236 sphWarning ( "index '%s': %s - NOT SERVING", sIndexName, sError.cstr() );
15237 tIndex.m_bEnabled = false;
15238 }
15239
15240 if ( !tIndex.m_bEnabled )
15241 continue;
15242 }
15243
15244 if ( eAdd!=ADD_ERROR )
15245 iValidIndexes++;
15246 }
15247
15248 tmLoad += sphMicroTimer();
15249 if ( !iValidIndexes )
15250 sphFatal ( "no valid indexes to serve" );
15251 else
15252 fprintf ( stdout, "precached %d indexes in %0.3f sec\n", iCounter-1, float(tmLoad)/1000000 );
15253 }
15254
OpenDaemonLog(const CSphConfigSection & hSearchd,bool bCloseIfOpened=false)15255 void OpenDaemonLog ( const CSphConfigSection & hSearchd, bool bCloseIfOpened=false )
15256 {
15257 // create log
15258 const char * sLog = "searchd.log";
15259 if ( hSearchd.Exists ( "log" ) )
15260 {
15261 if ( hSearchd["log"]=="syslog" )
15262 {
15263 #if !USE_SYSLOG
15264 if ( g_iLogFile<0 )
15265 {
15266 g_iLogFile = STDOUT_FILENO;
15267 sphWarning ( "failed to use syslog for logging. You have to reconfigure --with-syslog and rebuild the daemon!" );
15268 sphInfo ( "will use default file 'searchd.log' for logging." );
15269 }
15270 #else
15271 g_bLogSyslog = true;
15272 #endif
15273 } else
15274 {
15275 sLog = hSearchd["log"].cstr();
15276 }
15277 }
15278
15279 umask ( 066 );
15280 if ( bCloseIfOpened && g_iLogFile!=STDOUT_FILENO )
15281 {
15282 close ( g_iLogFile );
15283 g_iLogFile = STDOUT_FILENO;
15284 }
15285 if ( !g_bLogSyslog )
15286 {
15287 g_iLogFile = open ( sLog, O_CREAT | O_RDWR | O_APPEND, S_IREAD | S_IWRITE );
15288 if ( g_iLogFile<0 )
15289 {
15290 g_iLogFile = STDOUT_FILENO;
15291 sphFatal ( "failed to open log file '%s': %s", sLog, strerror(errno) );
15292 }
15293 }
15294
15295 g_sLogFile = sLog;
15296 g_bLogTty = isatty ( g_iLogFile )!=0;
15297 }
15298
15299
sphPoll(int iSock,int64_t tmTimeout,bool bWrite=false)15300 int sphPoll ( int iSock, int64_t tmTimeout, bool bWrite=false )
15301 {
15302 fd_set fdSet;
15303 FD_ZERO ( &fdSet );
15304 sphFDSet ( iSock, &fdSet );
15305
15306 struct timeval tv;
15307 tv.tv_sec = (int)( tmTimeout / 1000000 );
15308 tv.tv_usec = (int)( tmTimeout % 1000000 );
15309
15310 return ::select ( iSock+1, bWrite ? NULL : &fdSet, bWrite ? &fdSet : NULL, NULL, &tv );
15311 }
15312
15313
ServiceMain(int argc,char ** argv)15314 int WINAPI ServiceMain ( int argc, char **argv )
15315 {
15316 g_bLogTty = isatty ( g_iLogFile )!=0;
15317
15318 #if USE_WINDOWS
15319 CSphVector<char *> dArgs;
15320 if ( g_bService )
15321 {
15322 g_ssHandle = RegisterServiceCtrlHandler ( g_sServiceName, ServiceControl );
15323 if ( !g_ssHandle )
15324 sphFatal ( "failed to start service: RegisterServiceCtrlHandler() failed: %s", WinErrorInfo() );
15325
15326 g_ss.dwServiceType = SERVICE_WIN32_OWN_PROCESS;
15327 MySetServiceStatus ( SERVICE_START_PENDING, NO_ERROR, 4000 );
15328
15329 if ( argc<=1 )
15330 {
15331 dArgs.Resize ( g_dArgs.GetLength() );
15332 ARRAY_FOREACH ( i, g_dArgs )
15333 dArgs[i] = (char*) g_dArgs[i].cstr();
15334
15335 argc = g_dArgs.GetLength();
15336 argv = &dArgs[0];
15337 }
15338 }
15339
15340 char szPipeName[64];
15341 snprintf ( szPipeName, sizeof(szPipeName), "\\\\.\\pipe\\searchd_%d", getpid() );
15342 g_hPipe = CreateNamedPipe ( szPipeName, PIPE_ACCESS_INBOUND, PIPE_TYPE_BYTE | PIPE_READMODE_BYTE | PIPE_NOWAIT, PIPE_UNLIMITED_INSTANCES, 0, WIN32_PIPE_BUFSIZE, NMPWAIT_NOWAIT, NULL );
15343 ConnectNamedPipe ( g_hPipe, NULL );
15344 #endif
15345
15346 tzset();
15347
15348 if ( !g_bService )
15349 fprintf ( stdout, SPHINX_BANNER );
15350
15351 //////////////////////
15352 // parse command line
15353 //////////////////////
15354
15355 CSphConfig conf;
15356 bool bOptStop = false;
15357 bool bOptStopWait = false;
15358 bool bOptStatus = false;
15359 bool bOptPIDFile = false;
15360 const char * sOptIndex = NULL;
15361
15362 int iOptPort = 0;
15363 bool bOptPort = false;
15364
15365 CSphString sOptListen;
15366 bool bOptListen = false;
15367 bool bTestMode = false;
15368
15369 DWORD uReplayFlags = 0;
15370
15371 #define OPT(_a1,_a2) else if ( !strcmp(argv[i],_a1) || !strcmp(argv[i],_a2) )
15372 #define OPT1(_a1) else if ( !strcmp(argv[i],_a1) )
15373
15374 int i;
15375 for ( i=1; i<argc; i++ )
15376 {
15377 // handle non-options
15378 if ( argv[i][0]!='-' ) break;
15379
15380 // handle no-arg options
15381 OPT ( "-h", "--help" ) { ShowHelp(); return 0; }
15382 OPT ( "-?", "--?" ) { ShowHelp(); return 0; }
15383 OPT1 ( "--console" ) { g_eWorkers = MPM_NONE; g_bOptNoLock = true; g_bOptNoDetach = true; }
15384 OPT1 ( "--stop" ) bOptStop = true;
15385 OPT1 ( "--stopwait" ) { bOptStop = true; bOptStopWait = true; }
15386 OPT1 ( "--status" ) bOptStatus = true;
15387 OPT1 ( "--pidfile" ) bOptPIDFile = true;
15388 OPT1 ( "--iostats" ) g_bIOStats = true;
15389 #if !USE_WINDOWS
15390 OPT1 ( "--cpustats" ) g_bCpuStats = true;
15391 #endif
15392 #if USE_WINDOWS
15393 OPT1 ( "--install" ) { if ( !g_bService ) { ServiceInstall ( argc, argv ); return 0; } }
15394 OPT1 ( "--delete" ) { if ( !g_bService ) { ServiceDelete (); return 0; } }
15395 OPT1 ( "--ntservice" ) {} // it's valid but handled elsewhere
15396 #else
15397 OPT1 ( "--nodetach" ) g_bOptNoDetach = true;
15398 #endif
15399 OPT1 ( "--logdebug" ) g_eLogLevel = SPH_LOG_DEBUG;
15400 OPT1 ( "--logdebugv" ) g_eLogLevel = SPH_LOG_VERBOSE_DEBUG;
15401 OPT1 ( "--logdebugvv" ) g_eLogLevel = SPH_LOG_VERY_VERBOSE_DEBUG;
15402 OPT1 ( "--safetrace" ) g_bSafeTrace = true;
15403 OPT1 ( "--test" ) { g_bWatchdog = false; bTestMode = true; } // internal option, do NOT document
15404 OPT1 ( "--strip-path" ) g_bStripPath = true;
15405
15406 // FIXME! add opt=(csv)val handling here
15407 OPT1 ( "--replay-flags=accept-desc-timestamp" ) uReplayFlags |= SPH_REPLAY_ACCEPT_DESC_TIMESTAMP;
15408
15409 // handle 1-arg options
15410 else if ( (i+1)>=argc ) break;
15411 OPT ( "-c", "--config" ) g_sConfigFile = argv[++i];
15412 OPT ( "-p", "--port" ) { bOptPort = true; iOptPort = atoi ( argv[++i] ); }
15413 OPT ( "-l", "--listen" ) { bOptListen = true; sOptListen = argv[++i]; }
15414 OPT ( "-i", "--index" ) sOptIndex = argv[++i];
15415 #if USE_WINDOWS
15416 OPT1 ( "--servicename" ) ++i; // it's valid but handled elsewhere
15417 #endif
15418
15419 // handle unknown options
15420 else
15421 break;
15422 }
15423 if ( i!=argc )
15424 sphFatal ( "malformed or unknown option near '%s'; use '-h' or '--help' to see available options.", argv[i] );
15425
15426 #if USE_WINDOWS
15427 // init WSA on Windows
15428 // we need to do it this early because otherwise gethostbyname() from config parser could fail
15429 WSADATA tWSAData;
15430 int iStartupErr = WSAStartup ( WINSOCK_VERSION, &tWSAData );
15431 if ( iStartupErr )
15432 sphFatal ( "failed to initialize WinSock2: %s", sphSockError ( iStartupErr ) );
15433
15434 #ifndef NDEBUG
15435 // i want my windows debugging sessions to log onto stdout
15436 g_bOptNoDetach = true;
15437 g_bOptNoLock = true;
15438 #endif
15439 #endif
15440
15441 if ( !bOptPIDFile )
15442 bOptPIDFile = !g_bOptNoLock;
15443
15444 // check port and listen arguments early
15445 if ( !g_bOptNoDetach && ( bOptPort || bOptListen ) )
15446 {
15447 sphWarning ( "--listen and --port are only allowed in --console debug mode; switch ignored" );
15448 bOptPort = bOptListen = false;
15449 }
15450
15451 if ( bOptPort )
15452 {
15453 if ( bOptListen )
15454 sphFatal ( "please specify either --port or --listen, not both" );
15455
15456 CheckPort ( iOptPort );
15457 }
15458
15459 /////////////////////
15460 // parse config file
15461 /////////////////////
15462
15463 // fallback to defaults if there was no explicit config specified
15464 while ( !g_sConfigFile.cstr() )
15465 {
15466 #ifdef SYSCONFDIR
15467 g_sConfigFile = SYSCONFDIR "/sphinx.conf";
15468 if ( sphIsReadable ( g_sConfigFile.cstr () ) )
15469 break;
15470 #endif
15471
15472 g_sConfigFile = "./sphinx.conf";
15473 if ( sphIsReadable ( g_sConfigFile.cstr () ) )
15474 break;
15475
15476 g_sConfigFile = NULL;
15477 break;
15478 }
15479
15480 if ( !g_sConfigFile.cstr () )
15481 sphFatal ( "no readable config file (looked in "
15482 #ifdef SYSCONFDIR
15483 SYSCONFDIR "/sphinx.conf, "
15484 #endif
15485 "./sphinx.conf)." );
15486
15487 sphInfo ( "using config file '%s'...", g_sConfigFile.cstr () );
15488
15489 CheckConfigChanges ();
15490
15491 // do parse
15492 if ( !g_pCfg.Parse ( g_sConfigFile.cstr () ) )
15493 sphFatal ( "failed to parse config file '%s'", g_sConfigFile.cstr () );
15494
15495 const CSphConfig & hConf = g_pCfg.m_tConf;
15496
15497 if ( !hConf.Exists ( "searchd" ) || !hConf["searchd"].Exists ( "searchd" ) )
15498 sphFatal ( "'searchd' config section not found in '%s'", g_sConfigFile.cstr () );
15499
15500 const CSphConfigSection & hSearchdpre = hConf["searchd"]["searchd"];
15501
15502 ////////////////////////
15503 // stop running searchd
15504 ////////////////////////
15505
15506 if ( bOptStop )
15507 {
15508 if ( !hSearchdpre("pid_file") )
15509 sphFatal ( "stop: option 'pid_file' not found in '%s' section 'searchd'", g_sConfigFile.cstr () );
15510
15511 const char * sPid = hSearchdpre["pid_file"].cstr(); // shortcut
15512 FILE * fp = fopen ( sPid, "r" );
15513 if ( !fp )
15514 sphFatal ( "stop: pid file '%s' does not exist or is not readable", sPid );
15515
15516 char sBuf[16];
15517 int iLen = (int) fread ( sBuf, 1, sizeof(sBuf)-1, fp );
15518 sBuf[iLen] = '\0';
15519 fclose ( fp );
15520
15521 int iPid = atoi(sBuf);
15522 if ( iPid<=0 )
15523 sphFatal ( "stop: failed to read valid pid from '%s'", sPid );
15524
15525 #if USE_WINDOWS
15526 bool bTerminatedOk = false;
15527
15528 char szPipeName[64];
15529 snprintf ( szPipeName, sizeof(szPipeName), "\\\\.\\pipe\\searchd_%d", iPid );
15530
15531 HANDLE hPipe = INVALID_HANDLE_VALUE;
15532
15533 while ( hPipe==INVALID_HANDLE_VALUE )
15534 {
15535 hPipe = CreateFile ( szPipeName, GENERIC_WRITE, 0, NULL, OPEN_EXISTING, 0, NULL );
15536
15537 if ( hPipe==INVALID_HANDLE_VALUE )
15538 {
15539 if ( GetLastError()!=ERROR_PIPE_BUSY )
15540 {
15541 fprintf ( stdout, "WARNING: could not open pipe (GetLastError()=%d)\n", GetLastError () );
15542 break;
15543 }
15544
15545 if ( !WaitNamedPipe ( szPipeName, 1000 ) )
15546 {
15547 fprintf ( stdout, "WARNING: could not open pipe (GetLastError()=%d)\n", GetLastError () );
15548 break;
15549 }
15550 }
15551 }
15552
15553 if ( hPipe!=INVALID_HANDLE_VALUE )
15554 {
15555 DWORD uWritten = 0;
15556 BYTE uWrite = 1;
15557 BOOL bResult = WriteFile ( hPipe, &uWrite, 1, &uWritten, NULL );
15558 if ( !bResult )
15559 fprintf ( stdout, "WARNING: failed to send SIGHTERM to searchd (pid=%d, GetLastError()=%d)\n", iPid, GetLastError () );
15560
15561 bTerminatedOk = !!bResult;
15562
15563 CloseHandle ( hPipe );
15564 }
15565
15566 if ( bTerminatedOk )
15567 {
15568 sphInfo ( "stop: successfully terminated pid %d", iPid );
15569 exit ( 0 );
15570 } else
15571 sphFatal ( "stop: error terminating pid %d", iPid );
15572 #else
15573 CSphString sPipeName;
15574 int iPipeCreated = -1;
15575 int fdPipe = -1;
15576 if ( bOptStopWait )
15577 {
15578 sPipeName = GetNamedPipeName ( iPid );
15579 iPipeCreated = mkfifo ( sPipeName.cstr(), 0666 );
15580 if ( iPipeCreated!=-1 )
15581 fdPipe = ::open ( sPipeName.cstr(), O_RDONLY | O_NONBLOCK );
15582
15583 if ( iPipeCreated==-1 )
15584 sphWarning ( "mkfifo failed (path=%s, err=%d, msg=%s); will NOT wait", sPipeName.cstr(), errno, strerror(errno) );
15585 else if ( fdPipe<0 )
15586 sphWarning ( "open failed (path=%s, err=%d, msg=%s); will NOT wait", sPipeName.cstr(), errno, strerror(errno) );
15587 }
15588
15589 if ( kill ( iPid, SIGTERM ) )
15590 sphFatal ( "stop: kill() on pid %d failed: %s", iPid, strerror(errno) );
15591 else
15592 sphInfo ( "stop: successfully sent SIGTERM to pid %d", iPid );
15593
15594 int iExitCode = ( bOptStopWait && ( iPipeCreated==-1 || fdPipe<0 ) ) ? 1 : 0;
15595 bool bHandshake = true;
15596 while ( bOptStopWait && fdPipe>=0 )
15597 {
15598 int iReady = sphPoll ( fdPipe, 500000 );
15599
15600 // error on wait
15601 if ( iReady<0 )
15602 {
15603 iExitCode = 3;
15604 sphWarning ( "stopwait%s error '%s'", ( bHandshake ? " handshake" : " " ), strerror(errno) );
15605 break;
15606 }
15607
15608 // timeout
15609 if ( iReady==0 )
15610 {
15611 if ( !bHandshake )
15612 continue;
15613
15614 iExitCode = 1;
15615 break;
15616 }
15617
15618 // reading data
15619 DWORD uStatus = 0;
15620 int iRead = ::read ( fdPipe, &uStatus, sizeof(DWORD) );
15621 if ( iRead!=sizeof(DWORD) )
15622 {
15623 sphWarning ( "stopwait read fifo error '%s'", strerror(errno) );
15624 iExitCode = 3; // stopped demon crashed during stop
15625 break;
15626 } else
15627 {
15628 iExitCode = ( uStatus==1 ? 0 : 2 ); // uStatus == 1 - AttributeSave - ok, other values - error
15629 }
15630
15631 if ( !bHandshake )
15632 break;
15633
15634 bHandshake = false;
15635 }
15636 if ( fdPipe>=0 )
15637 ::close ( fdPipe );
15638 if ( iPipeCreated!=-1 )
15639 ::unlink ( sPipeName.cstr() );
15640
15641 exit ( iExitCode );
15642 #endif
15643 }
15644
15645 ////////////////////////////////
15646 // query running searchd status
15647 ////////////////////////////////
15648
15649 if ( bOptStatus )
15650 {
15651 QueryStatus ( hSearchdpre("listen") );
15652 exit ( 0 );
15653 }
15654
15655 /////////////////////
15656 // configure searchd
15657 /////////////////////
15658
15659 ConfigureSearchd ( hConf, bOptPIDFile );
15660 g_bWatchdog = hSearchdpre.GetInt ( "watchdog", g_bWatchdog )!=0;
15661
15662 if ( hSearchdpre("workers") )
15663 {
15664 if ( hSearchdpre["workers"]=="none" )
15665 g_eWorkers = MPM_NONE;
15666 else if ( hSearchdpre["workers"]=="fork" )
15667 g_eWorkers = MPM_FORK;
15668 else if ( hSearchdpre["workers"]=="prefork" )
15669 g_eWorkers = MPM_PREFORK;
15670 else if ( hSearchdpre["workers"]=="threads" )
15671 g_eWorkers = MPM_THREADS;
15672 else
15673 sphFatal ( "unknown workers=%s value", hSearchdpre["workers"].cstr() );
15674 }
15675 #if USE_WINDOWS
15676 if ( g_eWorkers==MPM_FORK || g_eWorkers==MPM_PREFORK )
15677 sphFatal ( "workers=fork and workers=prefork are not supported on Windows" );
15678 #endif
15679
15680 if ( g_iMaxPacketSize<128*1024 || g_iMaxPacketSize>128*1024*1024 )
15681 sphFatal ( "max_packet_size out of bounds (128K..128M)" );
15682
15683 if ( g_iMaxFilters<1 || g_iMaxFilters>10240 )
15684 sphFatal ( "max_filters out of bounds (1..10240)" );
15685
15686 if ( g_iMaxFilterValues<1 || g_iMaxFilterValues>10485760 )
15687 sphFatal ( "max_filter_values out of bounds (1..10485760)" );
15688
15689 bool bVisualLoad = true;
15690 bool bWatched = false;
15691 #if !USE_WINDOWS
15692 // Let us start watchdog right now, on foreground first.
15693 int iDevNull = open ( "/dev/null", O_RDWR );
15694 if ( g_bWatchdog && g_eWorkers==MPM_THREADS && !g_bOptNoDetach )
15695 {
15696 bWatched = true;
15697 if ( !g_bOptNoLock )
15698 OpenDaemonLog ( hConf["searchd"]["searchd"] );
15699 bVisualLoad = SetWatchDog ( iDevNull );
15700 OpenDaemonLog ( hConf["searchd"]["searchd"], true ); // just the 'IT Happens' magic - switch off, then on.
15701 }
15702 #endif
15703
15704 // create the pid
15705 if ( bOptPIDFile )
15706 {
15707 g_sPidFile = hSearchdpre["pid_file"].cstr();
15708
15709 g_iPidFD = ::open ( g_sPidFile, O_CREAT | O_WRONLY, S_IREAD | S_IWRITE );
15710 if ( g_iPidFD<0 )
15711 sphFatal ( "failed to create pid file '%s': %s", g_sPidFile, strerror(errno) );
15712 }
15713 if ( bOptPIDFile && !sphLockEx ( g_iPidFD, false ) )
15714 sphFatal ( "failed to lock pid file '%s': %s (searchd already running?)", g_sPidFile, strerror(errno) );
15715
15716 // Actions on resurrection
15717 if ( bWatched && !bVisualLoad && CheckConfigChanges() )
15718 {
15719 // reparse the config file
15720 sphInfo ( "Reloading the config" );
15721 if ( !g_pCfg.ReParse ( g_sConfigFile.cstr () ) )
15722 sphFatal ( "failed to parse config file '%s'", g_sConfigFile.cstr () );
15723
15724 sphInfo ( "Reconfigure the daemon" );
15725 ConfigureSearchd ( hConf, bOptPIDFile );
15726 }
15727
15728 // hSearchdpre might be dead if we reloaded the config.
15729 const CSphConfigSection & hSearchd = hConf["searchd"]["searchd"];
15730
15731 // handle my signals
15732 SetSignalHandlers ();
15733
15734 // create logs
15735 if ( !g_bOptNoLock )
15736 {
15737 // create log
15738 OpenDaemonLog ( hSearchd, true );
15739
15740 // create query log if required
15741 if ( hSearchd.Exists ( "query_log" ) )
15742 {
15743 if ( hSearchd["query_log"]=="syslog" )
15744 g_bQuerySyslog = true;
15745 else
15746 {
15747 g_iQueryLogFile = open ( hSearchd["query_log"].cstr(), O_CREAT | O_RDWR | O_APPEND, S_IREAD | S_IWRITE );
15748 if ( g_iQueryLogFile<0 )
15749 sphFatal ( "failed to open query log file '%s': %s", hSearchd["query_log"].cstr(), strerror(errno) );
15750 }
15751 g_sQueryLogFile = hSearchd["query_log"].cstr();
15752 }
15753 }
15754
15755 //////////////////////////////////////////////////
15756 // shared stuff (perf counters, flushing) startup
15757 //////////////////////////////////////////////////
15758
15759 g_pStats = InitSharedBuffer ( g_tStatsBuffer, 1 );
15760 g_pFlush = InitSharedBuffer ( g_tFlushBuffer, 1 );
15761 g_pStats->m_uStarted = (DWORD)time(NULL);
15762
15763 if ( g_eWorkers==MPM_PREFORK )
15764 g_pConnID = (int*) InitSharedBuffer ( g_dConnID, sizeof(g_iConnID) );
15765
15766 if ( g_eWorkers==MPM_THREADS )
15767 {
15768 if ( !sphThreadKeyCreate ( &g_tConnKey ) )
15769 sphFatal ( "failed to create TLS for connection ID" );
15770
15771 // for simplicity, UDFs are going to be available in threaded mode only for now
15772 sphUDFInit ( hSearchd.GetStr ( "plugin_dir" ) );
15773 }
15774
15775 ////////////////////
15776 // network startup
15777 ////////////////////
15778
15779 Listener_t tListener;
15780 tListener.m_eProto = PROTO_SPHINX;
15781
15782 // command line arguments override config (but only in --console)
15783 if ( bOptListen )
15784 {
15785 AddListener ( sOptListen );
15786
15787 } else if ( bOptPort )
15788 {
15789 tListener.m_iSock = sphCreateInetSocket ( htonl ( INADDR_ANY ), iOptPort );
15790 g_dListeners.Add ( tListener );
15791
15792 } else
15793 {
15794 // listen directives in configuration file
15795 for ( CSphVariant * v = hSearchd("listen"); v; v = v->m_pNext )
15796 AddListener ( *v );
15797
15798 // handle deprecated directives
15799 if ( hSearchd("port") )
15800 {
15801 DWORD uAddr = hSearchd.Exists("address") ?
15802 sphGetAddress ( hSearchd["address"].cstr(), GETADDR_STRICT ) : htonl ( INADDR_ANY );
15803
15804 int iPort = hSearchd["port"].intval();
15805 CheckPort(iPort);
15806
15807 tListener.m_iSock = sphCreateInetSocket ( uAddr, iPort );
15808 g_dListeners.Add ( tListener );
15809 }
15810
15811 // still nothing? default is to listen on our two ports
15812 if ( !g_dListeners.GetLength() )
15813 {
15814 tListener.m_iSock = sphCreateInetSocket ( htonl ( INADDR_ANY ), SPHINXAPI_PORT );
15815 tListener.m_eProto = PROTO_SPHINX;
15816 g_dListeners.Add ( tListener );
15817
15818 tListener.m_iSock = sphCreateInetSocket ( htonl ( INADDR_ANY ), SPHINXQL_PORT );
15819 tListener.m_eProto = PROTO_MYSQL41;
15820 g_dListeners.Add ( tListener );
15821 }
15822 }
15823
15824 #if !USE_WINDOWS
15825 // reserve an fd for clients
15826
15827 g_iClientFD = dup ( iDevNull );
15828 #endif
15829
15830 g_pIndexes = new IndexHash_c();
15831
15832 //////////////////////
15833 // build indexes hash
15834 //////////////////////
15835
15836 // setup mva updates arena here, since we could have saved persistent mva updates
15837 const char * sArenaError = sphArenaInit ( hSearchd.GetSize ( "mva_updates_pool", MVA_UPDATES_POOL ) );
15838 if ( sArenaError )
15839 sphWarning ( "process shared mutex unsupported, MVA update disabled ( %s )", sArenaError );
15840
15841 // configure and preload
15842
15843 ConfigureAndPreload ( hConf, sOptIndex );
15844
15845 ///////////
15846 // startup
15847 ///////////
15848
15849 if ( g_eWorkers==MPM_THREADS )
15850 sphRTInit ( hSearchd, bTestMode );
15851
15852 if ( !strcmp ( hSearchd.GetStr ( "query_log_format", "plain" ), "sphinxql" ) )
15853 g_eLogFormat = LOG_FORMAT_SPHINXQL;
15854
15855 // prepare to detach
15856 if ( !g_bOptNoDetach )
15857 {
15858 #if !USE_WINDOWS
15859 if ( !bWatched || bVisualLoad )
15860 {
15861 close ( STDIN_FILENO );
15862 close ( STDOUT_FILENO );
15863 close ( STDERR_FILENO );
15864 dup2 ( iDevNull, STDIN_FILENO );
15865 dup2 ( iDevNull, STDOUT_FILENO );
15866 dup2 ( iDevNull, STDERR_FILENO );
15867 }
15868 #endif
15869 ReleaseTTYFlag();
15870
15871 // explicitly unlock everything in parent immediately before fork
15872 //
15873 // there's a race in case another instance is started before
15874 // child re-acquires all locks; but let's hope that's rare
15875 if ( !bWatched )
15876 {
15877 for ( IndexHashIterator_c it ( g_pIndexes ); it.Next(); )
15878 {
15879 ServedIndex_t & tServed = it.Get();
15880 if ( tServed.m_bEnabled )
15881 tServed.m_pIndex->Unlock();
15882 }
15883 }
15884 }
15885
15886 if ( bOptPIDFile && !bWatched )
15887 sphLockUn ( g_iPidFD );
15888
15889 #if !USE_WINDOWS
15890 if ( !g_bOptNoDetach && !bWatched )
15891 {
15892 switch ( fork() )
15893 {
15894 case -1:
15895 // error
15896 Shutdown ();
15897 sphFatal ( "fork() failed (reason: %s)", strerror ( errno ) );
15898 exit ( 1 );
15899
15900 case 0:
15901 // daemonized child
15902 break;
15903
15904 default:
15905 // tty-controlled parent
15906 sphSetProcessInfo ( false );
15907 exit ( 0 );
15908 }
15909 }
15910 #endif
15911
15912 if ( g_eWorkers==MPM_THREADS )
15913 sphRTConfigure ( hSearchd, bTestMode );
15914
15915 if ( bOptPIDFile )
15916 {
15917 #if !USE_WINDOWS
15918 // re-lock pid
15919 // FIXME! there's a potential race here
15920 if ( !sphLockEx ( g_iPidFD, true ) )
15921 sphFatal ( "failed to re-lock pid file '%s': %s", g_sPidFile, strerror(errno) );
15922 #endif
15923
15924 char sPid[16];
15925 snprintf ( sPid, sizeof(sPid), "%d\n", (int)getpid() );
15926 int iPidLen = strlen(sPid);
15927
15928 lseek ( g_iPidFD, 0, SEEK_SET );
15929 if ( !sphWrite ( g_iPidFD, sPid, iPidLen ) )
15930 sphFatal ( "failed to write to pid file '%s' (errno=%d, msg=%s)", g_sPidFile,
15931 errno, strerror(errno) );
15932
15933 if ( ::ftruncate ( g_iPidFD, iPidLen ) )
15934 sphFatal ( "failed to truncate pid file '%s' (errno=%d, msg=%s)", g_sPidFile,
15935 errno, strerror(errno) );
15936 }
15937
15938 #if USE_WINDOWS
15939 SetConsoleCtrlHandler ( CtrlHandler, TRUE );
15940 #endif
15941
15942 if ( !g_bOptNoDetach && !bWatched )
15943 {
15944 // re-lock indexes
15945 for ( IndexHashIterator_c it ( g_pIndexes ); it.Next(); )
15946 {
15947 ServedIndex_t & tServed = it.Get();
15948 if ( !tServed.m_bEnabled )
15949 continue;
15950
15951 // obtain exclusive lock
15952 if ( !tServed.m_pIndex->Lock() )
15953 {
15954 sphWarning ( "index '%s': lock: %s; INDEX UNUSABLE", it.GetKey().cstr(),
15955 tServed.m_pIndex->GetLastError().cstr() );
15956 tServed.m_bEnabled = false;
15957 continue;
15958 }
15959
15960 // try to mlock again because mlock does not survive over fork
15961 if ( !tServed.m_pIndex->Mlock() )
15962 {
15963 sphWarning ( "index '%s': %s", it.GetKey().cstr(),
15964 tServed.m_pIndex->GetLastError().cstr() );
15965 }
15966 }
15967 }
15968
15969 // if we're running in console mode, dump queries to tty as well
15970 if ( g_bOptNoLock && hSearchd ( "query_log" ) )
15971 {
15972 g_bQuerySyslog = false;
15973 g_bLogSyslog = false;
15974 g_iQueryLogFile = g_iLogFile;
15975 }
15976
15977 #if USE_SYSLOG
15978 if ( g_bLogSyslog || g_bQuerySyslog )
15979 {
15980 openlog ( "searchd", LOG_PID, LOG_DAEMON );
15981 }
15982 #else
15983 if ( g_bQuerySyslog )
15984 sphFatal ( "Wrong query_log file! You have to reconfigure --with-syslog and rebuild daemon if you want to use syslog there." );
15985 #endif
15986 /////////////////
15987 // serve clients
15988 /////////////////
15989
15990 g_bHeadDaemon = true;
15991
15992 #if USE_WINDOWS
15993 if ( g_bService )
15994 MySetServiceStatus ( SERVICE_RUNNING, NO_ERROR, 0 );
15995 #endif
15996
15997 sphSetReadBuffers ( hSearchd.GetSize ( "read_buffer", 0 ), hSearchd.GetSize ( "read_unhinted", 0 ) );
15998
15999 CSphProcessSharedMutex * pAcceptMutex = NULL;
16000 #if !USE_WINDOWS
16001 if ( g_eWorkers==MPM_PREFORK )
16002 {
16003 pAcceptMutex = new CSphProcessSharedMutex();
16004 if ( !pAcceptMutex )
16005 sphFatal ( "failed to create process-shared mutex" );
16006
16007 if ( !pAcceptMutex->GetError() )
16008 {
16009 while ( g_dChildren.GetLength() < g_iPreforkChildren )
16010 {
16011 if ( PreforkChild()==0 ) // child process? break from here, go work
16012 break;
16013 }
16014
16015 g_iRotationThrottle = hSearchd.GetInt ( "prefork_rotation_throttle", 0 );
16016 } else
16017 {
16018 sphWarning ( "process shared mutex unsupported, switching to 'workers = fork' ( %s )", pAcceptMutex->GetError() );
16019 g_eWorkers = MPM_FORK;
16020 SafeDelete ( pAcceptMutex );
16021 }
16022 }
16023 #endif
16024
16025 // in threaded mode, create a dedicated rotation thread
16026 if ( g_eWorkers==MPM_THREADS )
16027 {
16028 if ( g_bSeamlessRotate && !sphThreadCreate ( &g_tRotateThread, RotationThreadFunc , 0 ) )
16029 sphDie ( "failed to create rotation thread" );
16030
16031 // reserving max to keep memory consumption constant between frames
16032 g_dThd.Reserve ( g_iMaxChildren*2 );
16033 }
16034
16035 // replay last binlog
16036 SmallStringHash_T<CSphIndex*> hIndexes;
16037 for ( IndexHashIterator_c it ( g_pIndexes ); it.Next(); )
16038 if ( it.Get().m_bEnabled )
16039 hIndexes.Add ( it.Get().m_pIndex, it.GetKey() );
16040
16041 if ( g_eWorkers==MPM_THREADS )
16042 sphReplayBinlog ( hIndexes, uReplayFlags, DumpMemStat );
16043
16044 if ( !g_bOptNoDetach )
16045 g_bLogStdout = false;
16046
16047 // create flush-rt thread
16048 if ( g_eWorkers==MPM_THREADS && !sphThreadCreate ( &g_tRtFlushThread, RtFlushThreadFunc, 0 ) )
16049 sphDie ( "failed to create rt-flush thread" );
16050
16051 if ( g_bIOStats && !sphInitIOStats () )
16052 sphWarning ( "unable to init IO statistics" );
16053
16054 // almost ready, time to start listening
16055 int iBacklog = hSearchd.GetInt ( "listen_backlog", SEARCHD_BACKLOG );
16056 ARRAY_FOREACH ( j, g_dListeners )
16057 if ( listen ( g_dListeners[j].m_iSock, iBacklog )==-1 )
16058 sphFatal ( "listen() failed: %s", sphSockError() );
16059
16060 sphInfo ( "accepting connections" );
16061
16062 for ( ;; )
16063 {
16064 SphCrashLogger_c::SetupTimePID();
16065
16066 if ( !g_bHeadDaemon && pAcceptMutex )
16067 TickPreforked ( pAcceptMutex );
16068 else
16069 TickHead ( pAcceptMutex );
16070 }
16071 } // NOLINT function length
16072
16073
DieCallback(const char * sMessage)16074 bool DieCallback ( const char * sMessage )
16075 {
16076 sphLogFatal ( "%s", sMessage );
16077 return false; // caller should not log
16078 }
16079
16080
16081 extern UservarIntSet_c * ( *g_pUservarsHook )( const CSphString & sUservar );
16082
UservarsHook(const CSphString & sUservar)16083 UservarIntSet_c * UservarsHook ( const CSphString & sUservar )
16084 {
16085 CSphScopedLock<StaticThreadsOnlyMutex_t> tLock ( g_tUservarsMutex );
16086
16087 Uservar_t * pVar = g_hUservars ( sUservar );
16088 if ( !pVar )
16089 return NULL;
16090
16091 assert ( pVar->m_eType==USERVAR_INT_SET );
16092 pVar->m_pVal->AddRef();
16093 return pVar->m_pVal;
16094 }
16095
16096
main(int argc,char ** argv)16097 int main ( int argc, char **argv )
16098 {
16099 // threads should be initialized before memory allocations
16100 char cTopOfMainStack;
16101 sphThreadInit();
16102 MemorizeStack ( &cTopOfMainStack );
16103
16104 sphSetDieCallback ( DieCallback );
16105 sphSetLogger ( sphLog );
16106 g_pUservarsHook = UservarsHook;
16107 sphCollationInit ();
16108
16109 #if USE_WINDOWS
16110 int iNameIndex = -1;
16111 for ( int i=1; i<argc; i++ )
16112 {
16113 if ( strcmp ( argv[i], "--ntservice" )==0 )
16114 g_bService = true;
16115
16116 if ( strcmp ( argv[i], "--servicename" )==0 && (i+1)<argc )
16117 {
16118 iNameIndex = i+1;
16119 g_sServiceName = argv[iNameIndex];
16120 }
16121 }
16122
16123 if ( g_bService )
16124 {
16125 for ( int i=0; i<argc; i++ )
16126 g_dArgs.Add ( argv[i] );
16127
16128 if ( iNameIndex>=0 )
16129 g_sServiceName = g_dArgs[iNameIndex].cstr ();
16130
16131 SERVICE_TABLE_ENTRY dDispatcherTable[] =
16132 {
16133 { (LPSTR) g_sServiceName, (LPSERVICE_MAIN_FUNCTION)ServiceMain },
16134 { NULL, NULL }
16135 };
16136 if ( !StartServiceCtrlDispatcher ( dDispatcherTable ) )
16137 sphFatal ( "StartServiceCtrlDispatcher() failed: %s", WinErrorInfo() );
16138 } else
16139 #endif
16140
16141 return ServiceMain ( argc, argv );
16142 }
16143
16144 //
16145 // $Id: searchd.cpp 4113 2013-08-26 07:43:28Z deogar $
16146 //
16147