1 /* This file is part of the Zebra server. 2 Copyright (C) 2004-2013 Index Data 3 4 Zebra is free software; you can redistribute it and/or modify it under 5 the terms of the GNU General Public License as published by the Free 6 Software Foundation; either version 2, or (at your option) any later 7 version. 8 9 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY 10 WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 for more details. 13 14 You should have received a copy of the GNU General Public License 15 along with this program; if not, write to the Free Software 16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 17 18 */ 19 20 #ifndef ZEBRA_INDEX_H 21 #define ZEBRA_INDEX_H 22 23 #include <time.h> 24 #include <stdlib.h> 25 #include <idzebra/version.h> 26 #include <idzebra/util.h> 27 #include <idzebra/flock.h> 28 #include <sortidx.h> 29 #if HAVE_SYS_TIMES_H 30 #include <sys/times.h> 31 #endif 32 #include <sys/stat.h> 33 34 #include <yaz/timing.h> 35 #include <idzebra/dict.h> 36 #include <idzebra/isams.h> 37 #include <idzebra/isamc.h> 38 #include <idzebra/isamb.h> 39 #include <d1_absyn.h> 40 #include <idzebra/recgrs.h> 41 #include "recindex.h" 42 #include <idzebra/api.h> 43 #include "zinfo.h" 44 #include <passwddb.h> 45 #include <rset.h> 46 #include <zebramap.h> 47 48 #include <it_key.h> 49 #include <su_codec.h> 50 51 YAZ_BEGIN_CDECL 52 53 #define ISAM_DEFAULT "b" 54 55 enum dirsKind { dirs_dir, dirs_file }; 56 57 struct dir_entry { 58 enum dirsKind kind; 59 char *name; 60 time_t mtime; 61 }; 62 63 struct dirs_entry { 64 enum dirsKind kind; 65 char path[256]; 66 zint sysno; 67 time_t mtime; 68 }; 69 70 void getFnameTmp(Res res, char *fname, int no); 71 72 struct dirs_info *dirs_open(Dict dict, const char *rep, int rw); 73 struct dirs_info *dirs_fopen(Dict dict, const char *path, int rw); 74 struct dirs_entry *dirs_read(struct dirs_info *p); 75 struct dirs_entry *dirs_last(struct dirs_info *p); 76 void dirs_mkdir(struct dirs_info *p, const char *src, time_t mtime); 77 void dirs_rmdir(struct dirs_info *p, const char *src); 78 void dirs_add(struct dirs_info *p, const char *src, zint sysno, time_t mtime); 79 void dirs_del(struct dirs_info *p, const char *src); 80 void dirs_free(struct dirs_info **pp); 81 82 struct dir_entry *dir_open(const char *rep, const char *base, 83 int follow_links); 84 void dir_sort(struct dir_entry *e); 85 void dir_free(struct dir_entry **e_p); 86 87 void repositoryUpdate(ZebraHandle zh, const char *path); 88 void repositoryAdd(ZebraHandle zh, const char *path); 89 void repositoryDelete(ZebraHandle zh, const char *path); 90 void repositoryShow(ZebraHandle zh, const char *path); 91 92 void inv_prstat(ZebraHandle zh); 93 void inv_compact(BFiles bfs); 94 void key_input(ZebraHandle zh, int nkeys, int cache, Res res); 95 ISAMS_M *key_isams_m(Res res, ISAMS_M *me); 96 ISAMC_M *key_isamc_m(Res res, ISAMC_M *me); 97 98 #define FNAME_DICT "dict" 99 #define FNAME_ISAM "isam" 100 #define FNAME_ISAMC "isamc" 101 #define FNAME_ISAMS "isams" 102 #define FNAME_CONFIG "zebra.cfg" 103 104 #define GMATCH_DICT "gmatch" 105 #define FMATCH_DICT "fmatch%d" 106 107 void zebra_lock_prefix(Res res, char *dst); 108 109 #define FNAME_MAIN_LOCK "zebraidx.LCK" 110 #define FNAME_COMMIT_LOCK "zebracmt.LCK" 111 #define FNAME_ORG_LOCK "zebraorg.LCK" 112 #define FNAME_TOUCH_TIME "zebraidx.time" 113 114 typedef struct zebra_set *ZebraSet; 115 116 typedef struct zebra_rank_class { 117 struct rank_control *control; 118 int init_flag; 119 void *class_handle; 120 struct zebra_rank_class *next; 121 } *ZebraRankClass; 122 123 #include "reckeys.h" 124 #include "key_block.h" 125 126 struct zebra_register { 127 char *name; 128 129 ISAMS isams; 130 ISAMC isamc; 131 ISAMB isamb; 132 Dict dict; 133 Dict matchDict; 134 zebra_sort_index_t sort_index; 135 int registerState; /* 0 (no commit pages), 1 (use commit pages) */ 136 time_t registerChange; 137 BFiles bfs; 138 Records records; 139 ZebraExplainInfo zei; 140 141 char *server_path_prefix; 142 data1_handle dh; 143 zebra_maps_t zebra_maps; 144 ZebraRankClass rank_classes; 145 RecTypes recTypes; 146 int seqno; 147 int last_val; 148 int stop_flag; 149 150 zebra_rec_keys_t keys; 151 zebra_rec_keys_t sortKeys; 152 153 zebra_key_block_t key_block; 154 }; 155 156 struct zebra_service { 157 int stop_flag; 158 Res global_res; 159 struct zebra_session *sessions; 160 struct zebra_register *regs; 161 Zebra_mutex_cond session_lock; 162 Passwd_db passwd_db; 163 Res dbaccess; 164 const char *path_root; 165 RecTypeClass record_classes; 166 NMEM nmem; 167 yaz_timing_t timing; 168 }; 169 170 171 struct zebra_session { 172 struct zebra_session *next; 173 struct zebra_service *service; 174 struct zebra_register *reg; 175 176 char *xadmin_databaseName; 177 178 char **basenames; 179 int num_basenames; 180 zint approx_limit; 181 char *reg_name; 182 char *path_reg; 183 184 ZebraLockHandle lock_normal; 185 ZebraLockHandle lock_shadow; 186 187 int trans_no; 188 int trans_w_no; 189 190 int destroyed; 191 ZebraSet sets; 192 Res res; 193 Res session_res; 194 char *user_perm; 195 char *dbaccesslist; 196 int errCode; 197 char *errString; 198 int partial_result; 199 #if HAVE_SYS_TIMES_H 200 struct tms tms1; 201 struct tms tms2; 202 #endif 203 int shadow_enable; 204 205 int m_staticrank; 206 int m_segment_indexing; 207 208 zint records_inserted; 209 zint records_updated; 210 zint records_deleted; 211 zint records_processed; 212 zint records_skipped; 213 char *record_encoding; 214 215 yaz_iconv_t iconv_to_utf8; 216 yaz_iconv_t iconv_from_utf8; 217 218 int m_follow_links; 219 const char *m_group; 220 const char *m_record_id; 221 const char *m_record_type; 222 int m_store_data; 223 int m_store_keys; 224 int m_explain_database; 225 int m_flag_rw; 226 int m_file_verbose_limit; 227 228 void *store_data_buf; 229 size_t store_data_size; 230 NMEM nmem_error; 231 232 struct zebra_limit *m_limit; 233 234 int (*break_handler_func)(void *client_data); 235 void *break_handler_data; 236 }; 237 238 239 struct term_set_entry { 240 char *term; 241 struct term_set_entry *next; 242 }; 243 244 struct term_set_list { 245 struct term_set_entry *first; 246 struct term_set_entry *last; 247 }; 248 249 250 void zebra_limit_destroy(struct zebra_limit *zl); 251 struct zebra_limit *zebra_limit_create(int exclude_flag, zint *ids); 252 void zebra_limit_for_rset(struct zebra_limit *zl, 253 int (**filter_func)(const void *buf, void *data), 254 void (**filter_destroy)(void *data), 255 void **filter_data); 256 257 struct rset_key_control *zebra_key_control_create(ZebraHandle zh); 258 259 ZEBRA_RES rpn_search_top(ZebraHandle zh, Z_RPNStructure *zs, 260 const Odr_oid *attributeSet, zint hits_limit, 261 NMEM stream, NMEM rset_nmem, 262 Z_SortKeySpecList *sort_sequence, 263 int num_bases, const char **basenames, 264 RSET *result_set); 265 266 ZEBRA_RES rpn_get_top_approx_limit(ZebraHandle zh, Z_RPNStructure *zs, 267 zint *approx_limit); 268 269 ZEBRA_RES rpn_scan(ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt, 270 const Odr_oid *attributeset, 271 int num_bases, char **basenames, 272 int *position, int *num_entries, ZebraScanEntry **list, 273 int *is_partial, const char *set_name); 274 275 RSET rset_trunc(ZebraHandle zh, ISAM_P *isam_p, int no, 276 const char *term, int length_term, const char *flags, 277 int preserve_position, int term_type, NMEM rset_nmem, 278 struct rset_key_control *kctrl, int scope, 279 struct ord_list *ol, const char *index_type, 280 zint hits_limit, const char *term_ref_id); 281 282 ZEBRA_RES resultSetGetBaseNames(ZebraHandle zh, const char *setname, 283 const char ***basenames, int *num_bases); 284 285 void resultSetAddTerm(ZebraHandle zh, ZebraSet s, int reg_type, 286 const char *db, const char *index_name, 287 const char *term); 288 ZebraSet resultSetAdd(ZebraHandle zh, const char *name, int ov); 289 ZebraSet resultSetGet(ZebraHandle zh, const char *name); 290 ZEBRA_RES resultSetAddRPN(ZebraHandle zh, NMEM m, Z_RPNQuery *rpn, 291 int num_bases, char **basenames, 292 const char *setname, 293 zint *hits, int *estimated_hit_count); 294 RSET resultSetRef(ZebraHandle zh, const char *resultSetId); 295 void resultSetDestroy(ZebraHandle zh, int num_names, char **names, 296 int *statuses); 297 ZEBRA_RES resultSetSort(ZebraHandle zh, NMEM nmem, 298 int num_input_setnames, const char **input_setnames, 299 const char *output_setname, 300 Z_SortKeySpecList *sort_sequence, int *sort_status); 301 ZEBRA_RES resultSetSortSingle(ZebraHandle zh, NMEM nmem, 302 ZebraSet sset, RSET rset, 303 Z_SortKeySpecList *sort_sequence, 304 int *sort_status); 305 ZEBRA_RES resultSetRank(ZebraHandle zh, ZebraSet zebraSet, RSET rset, 306 NMEM nmem); 307 void resultSetInvalidate(ZebraHandle zh); 308 309 int zebra_record_fetch(ZebraHandle zh, const char *setname, 310 zint sysno, int score, 311 ODR stream, 312 const Odr_oid *input_format, Z_RecordComposition *comp, 313 const Odr_oid **output_format, char **rec_bufp, 314 int *rec_lenp, char **basenamep, 315 WRBUF addinfo_w); 316 317 void extract_get_fname_tmp(ZebraHandle zh, char *fname, int no); 318 319 void extract_snippet(ZebraHandle zh, zebra_snippets *sn, 320 struct ZebraRecStream *stream, RecType rt, 321 void *recTypeClientData); 322 323 int zebra_get_rec_snippets(ZebraHandle zh, zint sysno, 324 zebra_snippets *snippets); 325 326 void zebra_index_merge(ZebraHandle zh); 327 328 ZEBRA_RES zebra_buffer_extract_record(ZebraHandle zh, 329 const char *buf, size_t buf_size, 330 enum zebra_recctrl_action_t action, 331 const char *recordType, 332 zint *sysno, 333 const char *match_criteria, 334 const char *fname); 335 336 337 YAZ_EXPORT void zebra_create_stream_mem(struct ZebraRecStream *stream, 338 const char *buf, size_t sz); 339 YAZ_EXPORT void zebra_create_stream_fd(struct ZebraRecStream *stream, 340 int fd, off_t start_offset); 341 void print_rec_keys(ZebraHandle zh, zebra_rec_keys_t reckeys); 342 343 ZEBRA_RES zebra_rec_keys_to_snippets(ZebraHandle zh, zebra_rec_keys_t reckeys, 344 zebra_snippets *snippets); 345 ZEBRA_RES zebra_snippets_hit_vector(ZebraHandle zh, const char *setname, 346 zint sysno, zebra_snippets *snippets); 347 348 ZEBRA_RES zebra_extract_explain(void *handle, Record rec, data1_node *n); 349 350 ZEBRA_RES zebra_extract_file(ZebraHandle zh, zint *sysno, const char *fname, 351 enum zebra_recctrl_action_t action); 352 353 ZEBRA_RES zebra_begin_read(ZebraHandle zh); 354 ZEBRA_RES zebra_end_read(ZebraHandle zh); 355 356 int zebra_file_stat(const char *file_name, struct stat *buf, 357 int follow_links); 358 359 Dict dict_open_res(BFiles bfs, const char *name, int cache, int rw, 360 int compact_flag, Res res); 361 362 void zebra_setError(ZebraHandle zh, int code, const char *addinfo); 363 void zebra_setError_zint(ZebraHandle zh, int code, zint i); 364 365 int zebra_term_untrans_iconv(ZebraHandle zh, NMEM stream, 366 const char *index_type, 367 char **dst, const char *src); 368 369 ZEBRA_RES zebra_get_hit_vector(ZebraHandle zh, const char *setname, zint sysno); 370 371 int zebra_term_untrans(ZebraHandle zh, const char *index_type, 372 char *dst, const char *src); 373 374 ZEBRA_RES zebra_apt_get_ord(ZebraHandle zh, 375 Z_AttributesPlusTerm *zapt, 376 const char *index_type, 377 const char *xpath_use, 378 const Odr_oid *curAttributeSet, 379 int *ord); 380 381 ZEBRA_RES zebra_attr_list_get_ord(ZebraHandle zh, 382 Z_AttributeList *attr_list, 383 zinfo_index_category_t cat, 384 const char *index_type, 385 const Odr_oid *curAttributeSet, 386 int *ord); 387 388 ZEBRA_RES zebra_sort_get_ord(ZebraHandle zh, 389 Z_SortAttributes *sortAttributes, 390 int *ord, 391 int *numerical); 392 393 ZEBRA_RES zebra_update_file_match(ZebraHandle zh, const char *path); 394 ZEBRA_RES zebra_update_from_path(ZebraHandle zh, const char *path, 395 enum zebra_recctrl_action_t action); 396 ZEBRA_RES zebra_remove_file_match(ZebraHandle zh); 397 398 struct rpn_char_map_info 399 { 400 zebra_map_t zm; 401 int reg_type; 402 }; 403 404 void rpn_char_map_prepare(struct zebra_register *reg, zebra_map_t zm, 405 struct rpn_char_map_info *map_info); 406 407 ZEBRA_RES zapt_term_to_utf8(ZebraHandle zh, Z_AttributesPlusTerm *zapt, 408 char *termz); 409 410 411 void zebra_set_partial_result(ZebraHandle zh); 412 413 int zebra_check_res(Res res); 414 415 #define FIRST_IN_FIELD_STR "\001^" 416 #define FIRST_IN_FIELD_CHAR 1 417 #define FIRST_IN_FIELD_LEN 2 418 419 ZEBRA_RES zebra_term_limits_APT(ZebraHandle zh, 420 Z_AttributesPlusTerm *zapt, 421 zint *hits_limit_value, 422 const char **term_ref_id_str, 423 NMEM nmem); 424 425 ZEBRA_RES zebra_result_recid_to_sysno(ZebraHandle zh, 426 const char *setname, 427 zint recid, 428 zint *sysnos, int *no_sysnos); 429 430 void zebra_count_set(ZebraHandle zh, RSET rset, zint *count, 431 zint approx_limit); 432 433 RSET zebra_create_rset_isam(ZebraHandle zh, 434 NMEM rset_nmem, struct rset_key_control *kctl, 435 int scope, ISAM_P pos, TERMID termid); 436 437 void zebra_it_key_str_dump(ZebraHandle zh, struct it_key *key, 438 const char *str, size_t slen, NMEM nmem, int level); 439 440 YAZ_END_CDECL 441 442 #endif 443 /* 444 * Local variables: 445 * c-basic-offset: 4 446 * c-file-style: "Stroustrup" 447 * indent-tabs-mode: nil 448 * End: 449 * vim: shiftwidth=4 tabstop=8 expandtab 450 */ 451 452