1version=0.2
2[RULE-ORDER]
3PREFIX SUFFIX WORD-TOKEN URL URL-WWW URL-DOMAIN
4E-MAIL WORD-PARPREFIX WORD-PARSUFFIX WORD-COMPOUND
5ABBREVIATION INITIAL SMILEY REVERSE-SMILEY PUNCTUATION-MULTI DATE-REVERSE DATE
6NUMBER-YEAR FRACNUMBER TIME NUMBER CURRENCY WORD PUNCTUATION UNKNOWN
7
8[META-RULES]
9SPLITTER=%
10#ABBREVIATION-KNOWN = (?:\p{P}*)?(?:\A|[^\p{L}\.])(?:\A)((?:%ABBREVIATIONS%)(?:\.{0,1}))(?:\Z|\P{L})
11WORD-TOKEN =(%TOKENS%)(?:\p{P}*)?$
12PREFIX = (?:\A|[^\p{Lu}\.]|[^\p{Ll}\.])(%PREFIXES% )\p{L}+
13SUFFIX = (?:\A|\p{L})+( %SUFFIXES% )(?:\Z|\P{L})
14
15[RULES]
16%include url
17%include e-mail
18%include smiley
19
20#Ex: (dis)information
21WORD-PARPREFIX=(?:\p{Ps}\p{L}+[\p{Pc}\p{Pd}]?\p{Pe}[\p{Pc}\p{Pd}]?)\p{L}+(?:[\p{Pc}\p{Pd}]\p{L}+)*
22
23#Ex: understand(s)
24WORD-PARSUFFIX=\p{L}+(?:[\p{Pc}\p{Pd}]\p{L}+)*(?:[\p{Pc}\p{Pd}]?\p{Ps}[\p{Pc}\p{Pd}]?\p{L}+\p{Pe})
25
26#Keep dash/underscore connected parts (even if they are in parenthesis)
27WORD-COMPOUND=\p{L}+(?:[\p{Pc}\p{Pd}]\p{L}+)+
28
29#Abbreviations with multiple periods
30ABBREVIATION=^(?:[\{\(\[\<]?)(\p{L}{1,3}(?:\.\p{L}{1,3})+\.?)(?:\Z|[,:;\}\)\]\>])
31
32#retain initials
33INITIAL=^(?:\p{Lt}|\p{Lu})\.$
34
35#Homogeneous punctuation (ellipsis etc)
36PUNCTUATION-MULTI=(?:\.|\-|[!\?]){2,}
37
38#Date
39DATE=\p{N}{1,2}[/\-]\p{N}{1,2}[/-]\p{N}{2,4}
40DATE-REVERSE=\p{N}{4}[/\-]\p{N}{1,2}[/\-]\p{N}{1,2}
41
42FRACNUMBER=\p{N}+(?:/\p{N}+)+
43
44NUMBER-YEAR=(['`’‘´]\p{N}{2})(?:\P{N}|\z)
45
46#Times
47TIME=\p{N}{1,2}:\p{N}{1,2}(?::\p{N})?(?i:am|pm)?
48
49#retain digits, including those starting with initial period (.22), and negative numbers
50NUMBER=-?(?:[\.,]?\p{N}+)+
51
52CURRENCY=\p{Sc}
53
54WORD=[\p{L}\p{Mn}]+
55
56PUNCTUATION=\p{P}
57
58UNKNOWN=.
59
60[PREFIXES]
61(?:qu|Qu|QU)['`’‘´]
62\[lL]['`’‘´]
63\[dD]['`’‘´]
64\[mM]['`’‘´]
65\[nN]['`’‘´]
66\[tT]['`’‘´]
67\[sS]['`’‘´]
68\[cC]['`’‘´]
69\[jJ]['`’‘´]
70
71[SUFFIXES]
72-je|-JE
73-me|-ME
74-tu|-TU
75-te|-TE
76-[tT]
77-il|-IL
78-se|-SE
79-lui|-LUI
80-elle|-ELLE
81-nous|-NOUS
82-vous|-VOUS
83-ils|-ILS
84-eux|-EUX
85-elles|-ELLES
86-ce|-CE
87-ci|-CI
88-là|-LÀ
89-la|-LA
90-le|-LE
91-les|-LES
92
93[TOKENS]
94\[lL]['`’‘´]
95\[dD]['`’‘´]
96\[mM]['`’‘´]
97\[nN]['`’‘´]
98\[tT]['`’‘´]
99\[sS]['`’‘´]
100\[cC]['`’‘´]
101\[jJ]['`’‘´]
102\[aA]ujourd['`’‘´]hui
103AUJOURD['`’‘´]HUI
104(?:qu|Qu|QU)['`’‘´]
105
106[UNITS]
107km
108m
109cm
110mm
111g
112kg
113C
114l
115s
116sec
117min
118gb
119mb
120kb
121
122
123[CURRENCY]
124EUR
125CAD
126
127[ABBREVIATIONS]
128%include fra
129
130[FILTER]
131fl fl
132ff ff
133ffi ffi
134ffl ffl
135# also filter soft hyphen
136\u00AD
137
138
139[EOSMARKERS]
140%include standard-eos
141
142[QUOTES]
143%include standard-quotes
144