1version=0.2 2[RULE-ORDER] 3PREFIX SUFFIX WORD-TOKEN URL URL-WWW URL-DOMAIN 4E-MAIL WORD-PARPREFIX WORD-PARSUFFIX WORD-COMPOUND 5ABBREVIATION INITIAL SMILEY REVERSE-SMILEY PUNCTUATION-MULTI DATE-REVERSE DATE 6NUMBER-YEAR FRACNUMBER TIME NUMBER CURRENCY WORD PUNCTUATION UNKNOWN 7 8[META-RULES] 9SPLITTER=% 10#ABBREVIATION-KNOWN = (?:\p{P}*)?(?:\A|[^\p{L}\.])(?:\A)((?:%ABBREVIATIONS%)(?:\.{0,1}))(?:\Z|\P{L}) 11WORD-TOKEN =(%TOKENS%)(?:\p{P}*)?$ 12PREFIX = (?:\A|[^\p{Lu}\.]|[^\p{Ll}\.])(%PREFIXES% )\p{L}+ 13SUFFIX = (?:\A|\p{L})+( %SUFFIXES% )(?:\Z|\P{L}) 14 15[RULES] 16%include url 17%include e-mail 18%include smiley 19 20#Ex: (dis)information 21WORD-PARPREFIX=(?:\p{Ps}\p{L}+[\p{Pc}\p{Pd}]?\p{Pe}[\p{Pc}\p{Pd}]?)\p{L}+(?:[\p{Pc}\p{Pd}]\p{L}+)* 22 23#Ex: understand(s) 24WORD-PARSUFFIX=\p{L}+(?:[\p{Pc}\p{Pd}]\p{L}+)*(?:[\p{Pc}\p{Pd}]?\p{Ps}[\p{Pc}\p{Pd}]?\p{L}+\p{Pe}) 25 26#Keep dash/underscore connected parts (even if they are in parenthesis) 27WORD-COMPOUND=\p{L}+(?:[\p{Pc}\p{Pd}]\p{L}+)+ 28 29#Abbreviations with multiple periods 30ABBREVIATION=^(?:[\{\(\[\<]?)(\p{L}{1,3}(?:\.\p{L}{1,3})+\.?)(?:\Z|[,:;\}\)\]\>]) 31 32#retain initials 33INITIAL=^(?:\p{Lt}|\p{Lu})\.$ 34 35#Homogeneous punctuation (ellipsis etc) 36PUNCTUATION-MULTI=(?:\.|\-|[!\?]){2,} 37 38#Date 39DATE=\p{N}{1,2}[/\-]\p{N}{1,2}[/-]\p{N}{2,4} 40DATE-REVERSE=\p{N}{4}[/\-]\p{N}{1,2}[/\-]\p{N}{1,2} 41 42FRACNUMBER=\p{N}+(?:/\p{N}+)+ 43 44NUMBER-YEAR=(['`’‘´]\p{N}{2})(?:\P{N}|\z) 45 46#Times 47TIME=\p{N}{1,2}:\p{N}{1,2}(?::\p{N})?(?i:am|pm)? 48 49#retain digits, including those starting with initial period (.22), and negative numbers 50NUMBER=-?(?:[\.,]?\p{N}+)+ 51 52CURRENCY=\p{Sc} 53 54WORD=[\p{L}\p{Mn}]+ 55 56PUNCTUATION=\p{P} 57 58UNKNOWN=. 59 60[PREFIXES] 61(?:qu|Qu|QU)['`’‘´] 62\[lL]['`’‘´] 63\[dD]['`’‘´] 64\[mM]['`’‘´] 65\[nN]['`’‘´] 66\[tT]['`’‘´] 67\[sS]['`’‘´] 68\[cC]['`’‘´] 69\[jJ]['`’‘´] 70 71[SUFFIXES] 72-je|-JE 73-me|-ME 74-tu|-TU 75-te|-TE 76-[tT] 77-il|-IL 78-se|-SE 79-lui|-LUI 80-elle|-ELLE 81-nous|-NOUS 82-vous|-VOUS 83-ils|-ILS 84-eux|-EUX 85-elles|-ELLES 86-ce|-CE 87-ci|-CI 88-là|-LÀ 89-la|-LA 90-le|-LE 91-les|-LES 92 93[TOKENS] 94\[lL]['`’‘´] 95\[dD]['`’‘´] 96\[mM]['`’‘´] 97\[nN]['`’‘´] 98\[tT]['`’‘´] 99\[sS]['`’‘´] 100\[cC]['`’‘´] 101\[jJ]['`’‘´] 102\[aA]ujourd['`’‘´]hui 103AUJOURD['`’‘´]HUI 104(?:qu|Qu|QU)['`’‘´] 105 106[UNITS] 107km 108m 109cm 110mm 111g 112kg 113C 114l 115s 116sec 117min 118gb 119mb 120kb 121 122 123[CURRENCY] 124EUR 125CAD 126 127[ABBREVIATIONS] 128%include fra 129 130[FILTER] 131fl fl 132ff ff 133ffi ffi 134ffl ffl 135# also filter soft hyphen 136\u00AD 137 138 139[EOSMARKERS] 140%include standard-eos 141 142[QUOTES] 143%include standard-quotes 144