Home
last modified time | relevance | path

Searched refs:normalized_tokens (Results 1 – 13 of 13) sorted by relevance

/dports/misc/libpostal/libpostal-1.1-alpha/scripts/geodata/text/
H A Dnormalize.py56 def normalized_tokens(s, string_options=DEFAULT_STRING_OPTIONS, function
71 normalized_tokens = _normalize.normalized_tokens(s, string_options, token_options, whitespace)
74 normalized_tokens = remove_parens(normalized_tokens)
76 return [(s, token_types.from_id(token_type)) for s, token_type in normalized_tokens]
H A Dpynormalize.c136 …libpostal_normalized_token_t *normalized_tokens = libpostal_normalized_tokens(input, string_option… in py_normalized_tokens() local
138 if (normalized_tokens == NULL) { in py_normalized_tokens()
148 libpostal_normalized_token_t normalized_token = normalized_tokens[i]; in py_normalized_tokens()
167 free(normalized_tokens[i].str); in py_normalized_tokens()
169 free(normalized_tokens); in py_normalized_tokens()
179 free(normalized_tokens[i].str); in py_normalized_tokens()
181 free(normalized_tokens); in py_normalized_tokens()
/dports/misc/libpostal/libpostal-1.1-alpha/scripts/geodata/address_expansions/
H A Dequivalence.py9 from geodata.text.normalize import normalized_tokens
34 tokens_s1 = normalized_tokens(s1)
35 tokens_s2 = normalized_tokens(s2)
H A Dgazetteers.py9 from geodata.text.normalize import normalized_tokens, normalize_string
/dports/misc/libpostal/libpostal-1.1-alpha/scripts/geodata/chains/
H A Dquery.py11 from geodata.text.normalize import normalized_tokens
25 tokens = normalized_tokens(name)
/dports/misc/libpostal/libpostal-1.1-alpha/scripts/geodata/language_id/
H A Ddisambiguation.py16 from geodata.text.normalize import normalized_tokens, normalize_string
116 tokens = normalized_tokens(text)
/dports/misc/libpostal/libpostal-1.1-alpha/src/
H A Daddress_parser.c437 if (self->normalized_tokens != NULL) { in address_parser_context_destroy()
438 token_array_destroy(self->normalized_tokens); in address_parser_context_destroy()
593 context->normalized_tokens = token_array_new(); in address_parser_context_new()
594 if (context->normalized_tokens == NULL) { in address_parser_context_new()
696 token_array *normalized_tokens = context->normalized_tokens; in address_parser_context_fill() local
698 token_array_clear(normalized_tokens); in address_parser_context_fill()
723 token_array_push(normalized_tokens, normalized_token); in address_parser_context_fill()
762 …s = search_address_dictionaries_tokens_with_phrases(normalized_str, normalized_tokens, NULL, &addr… in address_parser_context_fill()
H A Daddress_parser.h167 token_array *normalized_tokens; member
/dports/misc/libpostal/libpostal-1.1-alpha/scripts/geodata/names/
H A Ddeduping.py42 return normalized_tokens(s)
/dports/devel/py-maturin/maturin-0.11.3/cargo-crates/mailparse-0.13.5/src/
H A Dheader.rs217 pub fn normalized_tokens(raw_value: &str) -> Vec<HeaderToken> { in normalized_tokens() function
H A Daddrparse.rs290 let v = crate::header::normalized_tokens(&chars); in addrparse_header()
H A Dlib.rs187 for tok in header::normalized_tokens(&chars) { in get_value()
/dports/misc/libpostal/libpostal-1.1-alpha/scripts/geodata/addresses/
H A Dcomponents.py375 …name_norm = six.u('').join([t for t, c in normalized_tokens(name, string_options=NORMALIZE_STRING_…
386 …norm = six.u('').join([t for t, c in normalized_tokens(n, string_options=NORMALIZE_STRING_LOWERCAS…
424 tokens_lower = normalized_tokens(name, string_options=NORMALIZE_STRING_LOWERCASE,
463 tokens_lower = normalized_tokens(name, string_options=NORMALIZE_STRING_LOWERCASE,