Searched refs:normalized_tokens (Results 1 – 13 of 13) sorted by relevance
/dports/misc/libpostal/libpostal-1.1-alpha/scripts/geodata/text/ |
H A D | normalize.py | 56 def normalized_tokens(s, string_options=DEFAULT_STRING_OPTIONS, function 71 normalized_tokens = _normalize.normalized_tokens(s, string_options, token_options, whitespace) 74 normalized_tokens = remove_parens(normalized_tokens) 76 return [(s, token_types.from_id(token_type)) for s, token_type in normalized_tokens]
|
H A D | pynormalize.c | 136 …libpostal_normalized_token_t *normalized_tokens = libpostal_normalized_tokens(input, string_option… in py_normalized_tokens() local 138 if (normalized_tokens == NULL) { in py_normalized_tokens() 148 libpostal_normalized_token_t normalized_token = normalized_tokens[i]; in py_normalized_tokens() 167 free(normalized_tokens[i].str); in py_normalized_tokens() 169 free(normalized_tokens); in py_normalized_tokens() 179 free(normalized_tokens[i].str); in py_normalized_tokens() 181 free(normalized_tokens); in py_normalized_tokens()
|
/dports/misc/libpostal/libpostal-1.1-alpha/scripts/geodata/address_expansions/ |
H A D | equivalence.py | 9 from geodata.text.normalize import normalized_tokens 34 tokens_s1 = normalized_tokens(s1) 35 tokens_s2 = normalized_tokens(s2)
|
H A D | gazetteers.py | 9 from geodata.text.normalize import normalized_tokens, normalize_string
|
/dports/misc/libpostal/libpostal-1.1-alpha/scripts/geodata/chains/ |
H A D | query.py | 11 from geodata.text.normalize import normalized_tokens 25 tokens = normalized_tokens(name)
|
/dports/misc/libpostal/libpostal-1.1-alpha/scripts/geodata/language_id/ |
H A D | disambiguation.py | 16 from geodata.text.normalize import normalized_tokens, normalize_string 116 tokens = normalized_tokens(text)
|
/dports/misc/libpostal/libpostal-1.1-alpha/src/ |
H A D | address_parser.c | 437 if (self->normalized_tokens != NULL) { in address_parser_context_destroy() 438 token_array_destroy(self->normalized_tokens); in address_parser_context_destroy() 593 context->normalized_tokens = token_array_new(); in address_parser_context_new() 594 if (context->normalized_tokens == NULL) { in address_parser_context_new() 696 token_array *normalized_tokens = context->normalized_tokens; in address_parser_context_fill() local 698 token_array_clear(normalized_tokens); in address_parser_context_fill() 723 token_array_push(normalized_tokens, normalized_token); in address_parser_context_fill() 762 …s = search_address_dictionaries_tokens_with_phrases(normalized_str, normalized_tokens, NULL, &addr… in address_parser_context_fill()
|
H A D | address_parser.h | 167 token_array *normalized_tokens; member
|
/dports/misc/libpostal/libpostal-1.1-alpha/scripts/geodata/names/ |
H A D | deduping.py | 42 return normalized_tokens(s)
|
/dports/devel/py-maturin/maturin-0.11.3/cargo-crates/mailparse-0.13.5/src/ |
H A D | header.rs | 217 pub fn normalized_tokens(raw_value: &str) -> Vec<HeaderToken> { in normalized_tokens() function
|
H A D | addrparse.rs | 290 let v = crate::header::normalized_tokens(&chars); in addrparse_header()
|
H A D | lib.rs | 187 for tok in header::normalized_tokens(&chars) { in get_value()
|
/dports/misc/libpostal/libpostal-1.1-alpha/scripts/geodata/addresses/ |
H A D | components.py | 375 …name_norm = six.u('').join([t for t, c in normalized_tokens(name, string_options=NORMALIZE_STRING_… 386 …norm = six.u('').join([t for t, c in normalized_tokens(n, string_options=NORMALIZE_STRING_LOWERCAS… 424 tokens_lower = normalized_tokens(name, string_options=NORMALIZE_STRING_LOWERCASE, 463 tokens_lower = normalized_tokens(name, string_options=NORMALIZE_STRING_LOWERCASE,
|