Searched refs:phrase_tokens (Results 1 – 5 of 5) sorted by relevance
| /dports/misc/libpostal/libpostal-1.1-alpha/src/ |
| H A D | trie_search.c | 837 inline char *cstring_array_get_phrase(cstring_array *str, char_array *phrase_tokens, phrase_t phras… in cstring_array_get_phrase() argument
838 char_array_clear(phrase_tokens); in cstring_array_get_phrase()
844 char_array_append(phrase_tokens, w); in cstring_array_get_phrase()
846 char_array_append(phrase_tokens, " "); in cstring_array_get_phrase()
849 char_array_terminate(phrase_tokens); in cstring_array_get_phrase()
851 return char_array_get_string(phrase_tokens); in cstring_array_get_phrase()
|
| H A D | trie_search.h | 46 char *cstring_array_get_phrase(cstring_array *str, char_array *phrase_tokens, phrase_t phrase);
|
| H A D | address_parser.c | 1151 char_array *phrase_tokens = context->phrase; in address_parser_features() local
1178 phrase_string = cstring_array_get_phrase(context->normalized, phrase_tokens, phrase); in address_parser_features()
1352 char_array_clear(phrase_tokens); in address_parser_features()
1354 char_array_add_len(phrase_tokens, word_pre_norm, prefix_len); in address_parser_features()
1355 prefix = char_array_get_string(phrase_tokens); in address_parser_features()
|
| /dports/textproc/py-gensim/gensim-4.0.1/gensim/models/ |
| H A D | phrases.py | 597 phrase_tokens = itertools.chain([start_token], in_between, [word])
598 joined_phrase_token = delimiter.join(phrase_tokens)
|
| /dports/misc/libpostal/libpostal-1.1-alpha/scripts/geodata/addresses/ |
| H A D | components.py | 478 for is_phrase, phrase_tokens, value in phrases:
480 …ny((c in (token_types.IDEOGRAPHIC_CHAR, token_types.IDEOGRAPHIC_NUMBER) for t, c in phrase_tokens))
497 current_phrase_len = len(phrase_tokens)
507 total_tokens += len(phrase_tokens)
|