Searched refs:END_OF_TOKEN (Results 1 – 7 of 7) sorted by relevance
222 END_OF_TOKEN (s); in end_of_token()
448 #define END_OF_TOKEN(s) while (! STOP_SET (*(s), MAP_SPACE|MAP_NUL)) ++(s) macro
311 END_OF_TOKEN (s); in end_of_token()
451 #define END_OF_TOKEN(s) while (! STOP_SET (*(s), MAP_SPACE|MAP_NUL)) ++(s) macro
452 #define END_OF_TOKEN(s) while (! STOP_SET (*(s), MAP_SPACE|MAP_NUL)) ++(s)
1787 NEW_TOKEN(), END_OF_TOKEN(), ISBLANK(), ISSPACE() macros.
12428 enum { NO_SPLIT, END_OF_TOKEN, END_OF_SENTENCE, OUTCOMES }; enumerator12737 if (!allow_spaces || network_outcomes[i+1].outcome == gru_tokenizer_network::END_OF_TOKEN) in next_outcome()12738 network_outcomes[i].outcome = gru_tokenizer_network::END_OF_TOKEN; in next_outcome()13028 …i+1 < sentence.tokens.size() ? gru_tokenizer_network::END_OF_TOKEN : gru_tokenizer_network::END_OF… in train()