1 /* Copyright (C) 2000-2015 Lavtech.com corp. All rights reserved. 2 3 This program is free software; you can redistribute it and/or modify 4 it under the terms of the GNU General Public License as published by 5 the Free Software Foundation; either version 2 of the License, or 6 (at your option) any later version. 7 8 This program is distributed in the hope that it will be useful, 9 but WITHOUT ANY WARRANTY; without even the implied warranty of 10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 GNU General Public License for more details. 12 13 You should have received a copy of the GNU General Public License 14 along with this program; if not, write to the Free Software 15 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 16 */ 17 18 #ifndef _UDM_DB_INT_H 19 #define _UDM_DB_INT_H 20 21 #include <limits.h> 22 #include "udm_sqldbms.h" 23 #include "udm_indexcache.h" 24 #include "udm_searchtool.h" /* UDM_QUERY_PARAM */ 25 26 /* Multi-dict mode defines */ 27 #define MULTI_DICTS 0xff 28 29 #define URL_DELETE_CACHE 256 30 #define URL_SELECT_CACHE 256 31 #define URL_LOCK_TIME 4*60*60 32 #define UDM_MAX_MULTI_INSERT_QSIZE 16*1024 33 #define NDOCS_QUERY "SELECT count(*) FROM url" 34 35 36 /********************************************/ 37 38 typedef struct 39 { 40 char *word; 41 UDM_URLID_COORD urlid_coord; 42 unsigned char seed; 43 } UDM_WORD_CACHE_WORD; 44 45 46 typedef struct 47 { 48 size_t nbytes; 49 size_t nwords; 50 size_t awords; 51 UDM_WORD_CACHE_WORD *words; 52 size_t nurls; 53 size_t aurls; 54 urlid_t *urls; 55 } UDM_WORD_CACHE; 56 57 typedef struct 58 { 59 char *url[UDM_FINDURL_CACHE_SIZE]; 60 urlid_t id[UDM_FINDURL_CACHE_SIZE]; 61 size_t pURLCache; 62 } UDM_URLID_CACHE; 63 64 65 /*******************************************/ 66 67 /* SQL storage modes */ 68 typedef enum 69 { 70 UDM_SQLDBMODE_SINGLE= 0, 71 UDM_SQLDBMODE_MULTI= 1, 72 UDM_SQLDBMODE_BLOB= 2, 73 UDM_SQLDBMODE_RAWBLOB= 3 74 } udm_sqldbmode_t; 75 76 77 struct udm_dbmode_handler_st; 78 79 typedef struct udm_sqldb_st 80 { 81 UDM_SQL sql; 82 const struct udm_dbmode_handler_st *dbmode_handler; 83 udm_sqldbmode_t DBMode; 84 time_t last_notargets_time; 85 UDM_WORD_CACHE WordCache; 86 UDM_URLID_CACHE URLIdCache; 87 } UDM_SQLDB; 88 89 90 udm_sqldbtype_t UdmSQLDBType(UDM_DB *db); 91 udm_sqldbapi_t UdmSQLDBDriver(UDM_DB *db); 92 int UdmSQLDBFlags(UDM_DB *db); 93 udm_sqldbmode_t UdmSQLDBMode(UDM_DB *db); 94 int UdmSQLDBVersion(UDM_DB *db); 95 UDM_SQLDB_HANDLER *UdmSQLDBHandler(UDM_DB *db); 96 const char *UdmSQLDBQueryFrom(UDM_QUERY *query); 97 UDM_SQLDB *UdmSQLDB(UDM_DB *db); 98 int UdmSQLDBConnected(UDM_DB *db); 99 UDM_VARLIST *UdmSQLDBVars(UDM_DB *db); 100 101 udm_rc_t UdmDBSQLQuery(UDM_AGENT *A, UDM_DB *db, UDM_SQLRES *Res, const char *buf); 102 udm_rc_t UdmDBSQLDropTableIfExists(UDM_AGENT *A, UDM_DB *db, const char *name); 103 udm_rc_t UdmDBSQLTableTruncateOrDelete(UDM_AGENT *A, UDM_DB *db, const char *name); 104 udm_rc_t UdmDBSQLCopyStructure(UDM_AGENT *A, UDM_DB *db, const char *from, const char *to); 105 udm_rc_t UdmDBSQLRenameTable(UDM_AGENT *A, UDM_DB *db, const char *from, const char *to); 106 udm_rc_t UdmDBSQLBegin(UDM_AGENT *A, UDM_DB *db); 107 udm_rc_t UdmDBSQLCommit(UDM_AGENT *A, UDM_DB *db); 108 udm_rc_t UdmDBSQLExecDirect(UDM_AGENT *A, UDM_DB *db, UDM_SQLRES *R, const char *query); 109 udm_rc_t UdmDBSQLPrepare(UDM_AGENT *A, UDM_DB *db, const char *query); 110 udm_rc_t UdmDBSQLExecute(UDM_AGENT *A, UDM_DB *db); 111 udm_rc_t UdmDBSQLFetchRow(UDM_AGENT *A, UDM_DB *db, UDM_SQLRES *R, UDM_STR *); 112 udm_rc_t UdmDBSQLStmtFree(UDM_AGENT *A, UDM_DB *db); 113 udm_rc_t UdmDBSQLBindParameter(UDM_AGENT *A, UDM_DB *db, int pos, const void *data, int size, udm_sqltype_t type); 114 udm_rc_t UdmDBSQLLockOrBegin(UDM_AGENT *A, UDM_DB *db, const char *param); 115 udm_rc_t UdmDBSQLUnlockOrCommit(UDM_AGENT *A, UDM_DB *db); 116 char* UdmDBSQLEscStrAlloc(UDM_AGENT *A, UDM_DB *db, const char *src, size_t srclen); 117 char* UdmDBSQLEscStrSimple(UDM_AGENT *A, UDM_DB *db, char *to, const char *from, size_t l); 118 size_t UdmDBSQLBinEscStr(UDM_AGENT *A, UDM_DB *db, char *dst, size_t dstlen, const char *src, size_t srclen); 119 udm_rc_t UdmDBSQLQueryOneRowInt(UDM_AGENT *A, UDM_DB *db, int *res, const char *qbuf); 120 size_t UdmDBSQLEscStr(UDM_AGENT *A, UDM_DB *db, char *to, const char *from, size_t l); 121 void UdmDBSQLTopClause(UDM_AGENT *A, UDM_DB *db, size_t top_num, UDM_SQL_TOP_CLAUSE *Top); 122 123 const char *UdmDBSQLParamPlaceHolder(UDM_DB *db, size_t i); 124 char *UdmDBSQLError(UDM_DB *db); 125 size_t UdmDBSQLErrorSize(UDM_DB *db); 126 127 /******************/ 128 129 typedef struct st_udm_findword_args 130 { 131 UDM_URLID_LIST urls; /* Combination of WHERE limit, fl, and live_deleted */ 132 UDM_URLID_LIST live_update_active_urls; /* Combination of WHERE and fl */ 133 UDM_URLID_LIST live_update_deleted_urls; /* The list of deleted */ 134 UDM_WIDEWORDLIST CollationMatches; 135 UDM_SEARCHSECTIONLISTLIST SearchSectionListList; 136 const char *cmparg; 137 const char *where; 138 UDM_WIDEWORD Word; 139 int live_updates; 140 int need_coords; /* 0 for regular word, 1 for multi-word */ 141 time_t live_updates_ts; 142 UDM_SQLRESLIST SQLResults; 143 UDM_INVERTED_INDEX_CACHE IndexCache; 144 UDM_QUERY_PARAM query_param; 145 } UDM_FINDWORD_ARGS; 146 147 148 typedef struct udm_dbmode_handler_st 149 { 150 const char *name; 151 udm_rc_t (*StoreWords)(UDM_AGENT *Indexer, UDM_DB *db, UDM_DOCUMENT *Doc); 152 udm_rc_t (*QueryAction)(UDM_AGENT *Indexer, UDM_DB *db, UDM_QUERY *Query, udm_querycmd_t cmd); 153 udm_rc_t (*DeleteWordsFromURL)(UDM_AGENT *Indexer, UDM_DB *db, urlid_t url_id); 154 udm_rc_t (*FindWord)(UDM_AGENT *A, UDM_DB *db, UDM_QUERY *Query, UDM_FINDWORD_ARGS *args); 155 udm_rc_t (*DumpWordInfo)(UDM_AGENT *A, UDM_DB *db, UDM_DOCUMENT *Doc); 156 udm_rc_t (*InitSearch)(UDM_AGENT *A, UDM_DB *db, UDM_QUERY *Query, UDM_FINDWORD_ARGS *args); 157 } UDM_DBMODE_HANDLER; 158 159 extern const UDM_DBMODE_HANDLER udm_dbmode_handler_single; 160 extern const UDM_DBMODE_HANDLER udm_dbmode_handler_multi; 161 extern const UDM_DBMODE_HANDLER udm_dbmode_handler_blob; 162 extern const UDM_DBMODE_HANDLER udm_dbmode_handler_rawblob; 163 164 /* Functions from sql.c */ 165 udm_rc_t UdmLoadSlowLimit(UDM_AGENT *A, UDM_DB *db, UDM_URLID_LIST *list, const char *q); 166 udm_rc_t UdmUserScoreListLoad(UDM_AGENT *A, UDM_DB *db, 167 UDM_URL_INT4_LIST *List, const char *q); 168 169 /* Functions from dbmode-blob.c */ 170 udm_rc_t UdmBlobLoadFastURLLimit(UDM_AGENT *A, UDM_DB *db, const char *name, 171 UDM_URLID_LIST *buf); 172 udm_rc_t UdmBlobLoadFastOrder(UDM_AGENT *A, UDM_DB *db, UDM_URL_INT4_LIST *,const char *name); 173 udm_rc_t UdmBlobLoadFastScore(UDM_AGENT *A, UDM_DB *db, UDM_URL_INT4_LIST *,const char *name); 174 udm_rc_t UdmLoadURLDataFromBdict(UDM_AGENT *A, UDM_DB *db, 175 UDM_URLDATALIST *DataList, int flags); 176 udm_rc_t UdmBlobReadTimestamp(UDM_AGENT *A, UDM_DB *db, int *ts, int def); 177 178 /* Functions from dbmode-rawblob.c */ 179 #define UDM_RAWBLOB_SEARCH 0 180 #define UDM_RAWBLOB_DELTA 1 181 udm_rc_t UdmFindWordRawBlobDelta(UDM_AGENT *A, UDM_DB *db, UDM_QUERY *Query, 182 UDM_FINDWORD_ARGS *args); 183 184 /* Functions from wordcache.c */ 185 udm_rc_t UdmWordCacheWrite(UDM_AGENT *Indexer, UDM_DB *db, size_t limit); 186 udm_rc_t UdmWordCacheAddWordList(UDM_WORD_CACHE *Cache, 187 UDM_WORDLIST *List, urlid_t id); 188 UDM_WORD_CACHE *UdmWordCacheInit (UDM_WORD_CACHE *cache); 189 void UdmWordCacheFree(UDM_WORD_CACHE *cache); 190 void UdmWordCacheSort(UDM_WORD_CACHE *cache); 191 udm_rc_t UdmWordCacheAdd(UDM_WORD_CACHE *cache, urlid_t url_id, UDM_WORD *W); 192 udm_rc_t UdmWordCacheAddURL (UDM_WORD_CACHE *cache, urlid_t url_id); 193 194 /* Functions from suggest.c */ 195 udm_rc_t UdmWordStatQuery(UDM_AGENT *A, UDM_DB *db, const char *src); 196 udm_rc_t UdmQuerySuggest(UDM_AGENT *A, UDM_DB *db, UDM_QUERY *Query); 197 198 /* Functions from qcache.c */ 199 udm_rc_t UdmQueryCachePutSQL(UDM_AGENT *Indexer, UDM_DB *db, UDM_QUERY *Query); 200 udm_rc_t UdmQueryCacheGetSQL(UDM_AGENT *Indexer, UDM_DB *db, UDM_QUERY *Query); 201 udm_rc_t UdmApplyCachedQueryLimit(UDM_AGENT *query, 202 UDM_URLSCORELIST *ScoreList, UDM_DB *db); 203 204 #endif 205