1 #ifndef TOKENS_H 2 #define TOKENS_H 3 4 5 6 #include <stdlib.h> 7 #include <stdint.h> 8 9 #include "klib/khash.h" 10 #include "collections.h" 11 #include "string_utils.h" 12 #include "token_types.h" 13 #include "vector.h" 14 15 typedef libpostal_token_t token_t; 16 17 #define NULL_TOKEN (token_t){0, 0, END} 18 19 VECTOR_INIT(token_array, token_t) 20 21 typedef struct tokenized_string { 22 char *str; 23 cstring_array *strings; 24 token_array *tokens; 25 } tokenized_string_t; 26 27 tokenized_string_t *tokenized_string_new(void); 28 tokenized_string_t *tokenized_string_new_size(size_t len, size_t num_tokens); 29 tokenized_string_t *tokenized_string_new_from_str_size(char *src, size_t len, size_t num_tokens); 30 tokenized_string_t *tokenized_string_from_tokens(char *src, token_array *tokens, bool copy_tokens); 31 void tokenized_string_add_token(tokenized_string_t *self, const char *src, size_t len, uint16_t token_type, size_t position); 32 char *tokenized_string_get_token(tokenized_string_t *self, uint32_t index); 33 void tokenized_string_destroy(tokenized_string_t *self); 34 35 36 37 38 #endif 39