1 #ifndef TOKENS_H
2 #define TOKENS_H
3 
4 
5 
6 #include <stdlib.h>
7 #include <stdint.h>
8 
9 #include "klib/khash.h"
10 #include "collections.h"
11 #include "string_utils.h"
12 #include "token_types.h"
13 #include "vector.h"
14 
15 typedef libpostal_token_t token_t;
16 
17 #define NULL_TOKEN (token_t){0, 0, END}
18 
19 VECTOR_INIT(token_array, token_t)
20 
21 typedef struct tokenized_string {
22     char *str;
23     cstring_array *strings;
24     token_array *tokens;
25 } tokenized_string_t;
26 
27 tokenized_string_t *tokenized_string_new(void);
28 tokenized_string_t *tokenized_string_new_size(size_t len, size_t num_tokens);
29 tokenized_string_t *tokenized_string_new_from_str_size(char *src, size_t len, size_t num_tokens);
30 tokenized_string_t *tokenized_string_from_tokens(char *src, token_array *tokens, bool copy_tokens);
31 void tokenized_string_add_token(tokenized_string_t *self, const char *src, size_t len, uint16_t token_type, size_t position);
32 char *tokenized_string_get_token(tokenized_string_t *self, uint32_t index);
33 void tokenized_string_destroy(tokenized_string_t *self);
34 
35 
36 
37 
38 #endif
39