1 // © 2016 and later: Unicode, Inc. and others.
2 // License & terms of use: http://www.unicode.org/copyright.html
3 /*
4 **********************************************************************
5 * Copyright (C) 1999-2016, International Business Machines
6 * Corporation and others. All Rights Reserved.
7 **********************************************************************
8 * Date Name Description
9 * 11/17/99 aliu Creation.
10 **********************************************************************
11 */
12
13 #include "unicode/utypes.h"
14
15 #if !UCONFIG_NO_TRANSLITERATION
16
17 #include "unicode/uobject.h"
18 #include "unicode/parseerr.h"
19 #include "unicode/parsepos.h"
20 #include "unicode/putil.h"
21 #include "unicode/uchar.h"
22 #include "unicode/ustring.h"
23 #include "unicode/uniset.h"
24 #include "unicode/utf16.h"
25 #include "cstring.h"
26 #include "funcrepl.h"
27 #include "hash.h"
28 #include "quant.h"
29 #include "rbt.h"
30 #include "rbt_data.h"
31 #include "rbt_pars.h"
32 #include "rbt_rule.h"
33 #include "strmatch.h"
34 #include "strrepl.h"
35 #include "unicode/symtable.h"
36 #include "tridpars.h"
37 #include "uvector.h"
38 #include "hash.h"
39 #include "patternprops.h"
40 #include "util.h"
41 #include "cmemory.h"
42 #include "uprops.h"
43 #include "putilimp.h"
44
45 // Operators
46 #define VARIABLE_DEF_OP ((UChar)0x003D) /*=*/
47 #define FORWARD_RULE_OP ((UChar)0x003E) /*>*/
48 #define REVERSE_RULE_OP ((UChar)0x003C) /*<*/
49 #define FWDREV_RULE_OP ((UChar)0x007E) /*~*/ // internal rep of <> op
50
51 // Other special characters
52 #define QUOTE ((UChar)0x0027) /*'*/
53 #define ESCAPE ((UChar)0x005C) /*\*/
54 #define END_OF_RULE ((UChar)0x003B) /*;*/
55 #define RULE_COMMENT_CHAR ((UChar)0x0023) /*#*/
56
57 #define SEGMENT_OPEN ((UChar)0x0028) /*(*/
58 #define SEGMENT_CLOSE ((UChar)0x0029) /*)*/
59 #define CONTEXT_ANTE ((UChar)0x007B) /*{*/
60 #define CONTEXT_POST ((UChar)0x007D) /*}*/
61 #define CURSOR_POS ((UChar)0x007C) /*|*/
62 #define CURSOR_OFFSET ((UChar)0x0040) /*@*/
63 #define ANCHOR_START ((UChar)0x005E) /*^*/
64 #define KLEENE_STAR ((UChar)0x002A) /***/
65 #define ONE_OR_MORE ((UChar)0x002B) /*+*/
66 #define ZERO_OR_ONE ((UChar)0x003F) /*?*/
67
68 #define DOT ((UChar)46) /*.*/
69
70 static const UChar DOT_SET[] = { // "[^[:Zp:][:Zl:]\r\n$]";
71 91, 94, 91, 58, 90, 112, 58, 93, 91, 58, 90,
72 108, 58, 93, 92, 114, 92, 110, 36, 93, 0
73 };
74
75 // A function is denoted &Source-Target/Variant(text)
76 #define FUNCTION ((UChar)38) /*&*/
77
78 // Aliases for some of the syntax characters. These are provided so
79 // transliteration rules can be expressed in XML without clashing with
80 // XML syntax characters '<', '>', and '&'.
81 #define ALT_REVERSE_RULE_OP ((UChar)0x2190) // Left Arrow
82 #define ALT_FORWARD_RULE_OP ((UChar)0x2192) // Right Arrow
83 #define ALT_FWDREV_RULE_OP ((UChar)0x2194) // Left Right Arrow
84 #define ALT_FUNCTION ((UChar)0x2206) // Increment (~Greek Capital Delta)
85
86 // Special characters disallowed at the top level
87 static const UChar ILLEGAL_TOP[] = {41,0}; // ")"
88
89 // Special characters disallowed within a segment
90 static const UChar ILLEGAL_SEG[] = {123,125,124,64,0}; // "{}|@"
91
92 // Special characters disallowed within a function argument
93 static const UChar ILLEGAL_FUNC[] = {94,40,46,42,43,63,123,125,124,64,0}; // "^(.*+?{}|@"
94
95 // By definition, the ANCHOR_END special character is a
96 // trailing SymbolTable.SYMBOL_REF character.
97 // private static final char ANCHOR_END = '$';
98
99 static const UChar gOPERATORS[] = { // "=><"
100 VARIABLE_DEF_OP, FORWARD_RULE_OP, REVERSE_RULE_OP,
101 ALT_FORWARD_RULE_OP, ALT_REVERSE_RULE_OP, ALT_FWDREV_RULE_OP,
102 0
103 };
104
105 static const UChar HALF_ENDERS[] = { // "=><;"
106 VARIABLE_DEF_OP, FORWARD_RULE_OP, REVERSE_RULE_OP,
107 ALT_FORWARD_RULE_OP, ALT_REVERSE_RULE_OP, ALT_FWDREV_RULE_OP,
108 END_OF_RULE,
109 0
110 };
111
112 // These are also used in Transliterator::toRules()
113 static const int32_t ID_TOKEN_LEN = 2;
114 static const UChar ID_TOKEN[] = { 0x3A, 0x3A }; // ':', ':'
115
116 /*
117 commented out until we do real ::BEGIN/::END functionality
118 static const int32_t BEGIN_TOKEN_LEN = 5;
119 static const UChar BEGIN_TOKEN[] = { 0x42, 0x45, 0x47, 0x49, 0x4e }; // 'BEGIN'
120
121 static const int32_t END_TOKEN_LEN = 3;
122 static const UChar END_TOKEN[] = { 0x45, 0x4e, 0x44 }; // 'END'
123 */
124
125 U_NAMESPACE_BEGIN
126
127 //----------------------------------------------------------------------
128 // BEGIN ParseData
129 //----------------------------------------------------------------------
130
131 /**
132 * This class implements the SymbolTable interface. It is used
133 * during parsing to give UnicodeSet access to variables that
134 * have been defined so far. Note that it uses variablesVector,
135 * _not_ data.setVariables.
136 */
137 class ParseData : public UMemory, public SymbolTable {
138 public:
139 const TransliterationRuleData* data; // alias
140
141 const UVector* variablesVector; // alias
142
143 const Hashtable* variableNames; // alias
144
145 ParseData(const TransliterationRuleData* data = 0,
146 const UVector* variablesVector = 0,
147 const Hashtable* variableNames = 0);
148
149 virtual ~ParseData();
150
151 virtual const UnicodeString* lookup(const UnicodeString& s) const override;
152
153 virtual const UnicodeFunctor* lookupMatcher(UChar32 ch) const override;
154
155 virtual UnicodeString parseReference(const UnicodeString& text,
156 ParsePosition& pos, int32_t limit) const override;
157 /**
158 * Return true if the given character is a matcher standin or a plain
159 * character (non standin).
160 */
161 UBool isMatcher(UChar32 ch);
162
163 /**
164 * Return true if the given character is a replacer standin or a plain
165 * character (non standin).
166 */
167 UBool isReplacer(UChar32 ch);
168
169 private:
170 ParseData(const ParseData &other); // forbid copying of this class
171 ParseData &operator=(const ParseData &other); // forbid copying of this class
172 };
173
ParseData(const TransliterationRuleData * d,const UVector * sets,const Hashtable * vNames)174 ParseData::ParseData(const TransliterationRuleData* d,
175 const UVector* sets,
176 const Hashtable* vNames) :
177 data(d), variablesVector(sets), variableNames(vNames) {}
178
~ParseData()179 ParseData::~ParseData() {}
180
181 /**
182 * Implement SymbolTable API.
183 */
lookup(const UnicodeString & name) const184 const UnicodeString* ParseData::lookup(const UnicodeString& name) const {
185 return (const UnicodeString*) variableNames->get(name);
186 }
187
188 /**
189 * Implement SymbolTable API.
190 */
lookupMatcher(UChar32 ch) const191 const UnicodeFunctor* ParseData::lookupMatcher(UChar32 ch) const {
192 // Note that we cannot use data.lookupSet() because the
193 // set array has not been constructed yet.
194 const UnicodeFunctor* set = NULL;
195 int32_t i = ch - data->variablesBase;
196 if (i >= 0 && i < variablesVector->size()) {
197 int32_t j = ch - data->variablesBase;
198 set = (j < variablesVector->size()) ?
199 (UnicodeFunctor*) variablesVector->elementAt(j) : 0;
200 }
201 return set;
202 }
203
204 /**
205 * Implement SymbolTable API. Parse out a symbol reference
206 * name.
207 */
parseReference(const UnicodeString & text,ParsePosition & pos,int32_t limit) const208 UnicodeString ParseData::parseReference(const UnicodeString& text,
209 ParsePosition& pos, int32_t limit) const {
210 int32_t start = pos.getIndex();
211 int32_t i = start;
212 UnicodeString result;
213 while (i < limit) {
214 UChar c = text.charAt(i);
215 if ((i==start && !u_isIDStart(c)) || !u_isIDPart(c)) {
216 break;
217 }
218 ++i;
219 }
220 if (i == start) { // No valid name chars
221 return result; // Indicate failure with empty string
222 }
223 pos.setIndex(i);
224 text.extractBetween(start, i, result);
225 return result;
226 }
227
isMatcher(UChar32 ch)228 UBool ParseData::isMatcher(UChar32 ch) {
229 // Note that we cannot use data.lookup() because the
230 // set array has not been constructed yet.
231 int32_t i = ch - data->variablesBase;
232 if (i >= 0 && i < variablesVector->size()) {
233 UnicodeFunctor *f = (UnicodeFunctor*) variablesVector->elementAt(i);
234 return f != NULL && f->toMatcher() != NULL;
235 }
236 return TRUE;
237 }
238
239 /**
240 * Return true if the given character is a replacer standin or a plain
241 * character (non standin).
242 */
isReplacer(UChar32 ch)243 UBool ParseData::isReplacer(UChar32 ch) {
244 // Note that we cannot use data.lookup() because the
245 // set array has not been constructed yet.
246 int i = ch - data->variablesBase;
247 if (i >= 0 && i < variablesVector->size()) {
248 UnicodeFunctor *f = (UnicodeFunctor*) variablesVector->elementAt(i);
249 return f != NULL && f->toReplacer() != NULL;
250 }
251 return TRUE;
252 }
253
254 //----------------------------------------------------------------------
255 // BEGIN RuleHalf
256 //----------------------------------------------------------------------
257
258 /**
259 * A class representing one side of a rule. This class knows how to
260 * parse half of a rule. It is tightly coupled to the method
261 * RuleBasedTransliterator.Parser.parseRule().
262 */
263 class RuleHalf : public UMemory {
264
265 public:
266
267 UnicodeString text;
268
269 int32_t cursor; // position of cursor in text
270 int32_t ante; // position of ante context marker '{' in text
271 int32_t post; // position of post context marker '}' in text
272
273 // Record the offset to the cursor either to the left or to the
274 // right of the key. This is indicated by characters on the output
275 // side that allow the cursor to be positioned arbitrarily within
276 // the matching text. For example, abc{def} > | @@@ xyz; changes
277 // def to xyz and moves the cursor to before abc. Offset characters
278 // must be at the start or end, and they cannot move the cursor past
279 // the ante- or postcontext text. Placeholders are only valid in
280 // output text. The length of the ante and post context is
281 // determined at runtime, because of supplementals and quantifiers.
282 int32_t cursorOffset; // only nonzero on output side
283
284 // Position of first CURSOR_OFFSET on _right_. This will be -1
285 // for |@, -2 for |@@, etc., and 1 for @|, 2 for @@|, etc.
286 int32_t cursorOffsetPos;
287
288 UBool anchorStart;
289 UBool anchorEnd;
290
291 /**
292 * The segment number from 1..n of the next '(' we see
293 * during parsing; 1-based.
294 */
295 int32_t nextSegmentNumber;
296
297 TransliteratorParser& parser;
298
299 //--------------------------------------------------
300 // Methods
301
302 RuleHalf(TransliteratorParser& parser);
303 ~RuleHalf();
304
305 int32_t parse(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status);
306
307 int32_t parseSection(const UnicodeString& rule, int32_t pos, int32_t limit,
308 UnicodeString& buf,
309 const UnicodeString& illegal,
310 UBool isSegment,
311 UErrorCode& status);
312
313 /**
314 * Remove context.
315 */
316 void removeContext();
317
318 /**
319 * Return true if this half looks like valid output, that is, does not
320 * contain quantifiers or other special input-only elements.
321 */
322 UBool isValidOutput(TransliteratorParser& parser);
323
324 /**
325 * Return true if this half looks like valid input, that is, does not
326 * contain functions or other special output-only elements.
327 */
328 UBool isValidInput(TransliteratorParser& parser);
329
syntaxError(UErrorCode code,const UnicodeString & rule,int32_t start,UErrorCode & status)330 int syntaxError(UErrorCode code,
331 const UnicodeString& rule,
332 int32_t start,
333 UErrorCode& status) {
334 return parser.syntaxError(code, rule, start, status);
335 }
336
337 private:
338 // Disallowed methods; no impl.
339 RuleHalf(const RuleHalf&);
340 RuleHalf& operator=(const RuleHalf&);
341 };
342
RuleHalf(TransliteratorParser & p)343 RuleHalf::RuleHalf(TransliteratorParser& p) :
344 parser(p)
345 {
346 cursor = -1;
347 ante = -1;
348 post = -1;
349 cursorOffset = 0;
350 cursorOffsetPos = 0;
351 anchorStart = anchorEnd = FALSE;
352 nextSegmentNumber = 1;
353 }
354
~RuleHalf()355 RuleHalf::~RuleHalf() {
356 }
357
358 /**
359 * Parse one side of a rule, stopping at either the limit,
360 * the END_OF_RULE character, or an operator.
361 * @return the index after the terminating character, or
362 * if limit was reached, limit
363 */
parse(const UnicodeString & rule,int32_t pos,int32_t limit,UErrorCode & status)364 int32_t RuleHalf::parse(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status) {
365 int32_t start = pos;
366 text.truncate(0);
367 pos = parseSection(rule, pos, limit, text, UnicodeString(TRUE, ILLEGAL_TOP, -1), FALSE, status);
368
369 if (cursorOffset > 0 && cursor != cursorOffsetPos) {
370 return syntaxError(U_MISPLACED_CURSOR_OFFSET, rule, start, status);
371 }
372
373 return pos;
374 }
375
376 /**
377 * Parse a section of one side of a rule, stopping at either
378 * the limit, the END_OF_RULE character, an operator, or a
379 * segment close character. This method parses both a
380 * top-level rule half and a segment within such a rule half.
381 * It calls itself recursively to parse segments and nested
382 * segments.
383 * @param buf buffer into which to accumulate the rule pattern
384 * characters, either literal characters from the rule or
385 * standins for UnicodeMatcher objects including segments.
386 * @param illegal the set of special characters that is illegal during
387 * this parse.
388 * @param isSegment if true, then we've already seen a '(' and
389 * pos on entry points right after it. Accumulate everything
390 * up to the closing ')', put it in a segment matcher object,
391 * generate a standin for it, and add the standin to buf. As
392 * a side effect, update the segments vector with a reference
393 * to the segment matcher. This works recursively for nested
394 * segments. If isSegment is false, just accumulate
395 * characters into buf.
396 * @return the index after the terminating character, or
397 * if limit was reached, limit
398 */
parseSection(const UnicodeString & rule,int32_t pos,int32_t limit,UnicodeString & buf,const UnicodeString & illegal,UBool isSegment,UErrorCode & status)399 int32_t RuleHalf::parseSection(const UnicodeString& rule, int32_t pos, int32_t limit,
400 UnicodeString& buf,
401 const UnicodeString& illegal,
402 UBool isSegment, UErrorCode& status) {
403 int32_t start = pos;
404 ParsePosition pp;
405 UnicodeString scratch;
406 UBool done = FALSE;
407 int32_t quoteStart = -1; // Most recent 'single quoted string'
408 int32_t quoteLimit = -1;
409 int32_t varStart = -1; // Most recent $variableReference
410 int32_t varLimit = -1;
411 int32_t bufStart = buf.length();
412
413 while (pos < limit && !done) {
414 // Since all syntax characters are in the BMP, fetching
415 // 16-bit code units suffices here.
416 UChar c = rule.charAt(pos++);
417 if (PatternProps::isWhiteSpace(c)) {
418 // Ignore whitespace. Note that this is not Unicode
419 // spaces, but Java spaces -- a subset, representing
420 // whitespace likely to be seen in code.
421 continue;
422 }
423 if (u_strchr(HALF_ENDERS, c) != NULL) {
424 if (isSegment) {
425 // Unclosed segment
426 return syntaxError(U_UNCLOSED_SEGMENT, rule, start, status);
427 }
428 break;
429 }
430 if (anchorEnd) {
431 // Text after a presumed end anchor is a syntax err
432 return syntaxError(U_MALFORMED_VARIABLE_REFERENCE, rule, start, status);
433 }
434 if (UnicodeSet::resemblesPattern(rule, pos-1)) {
435 pp.setIndex(pos-1); // Backup to opening '['
436 buf.append(parser.parseSet(rule, pp, status));
437 if (U_FAILURE(status)) {
438 return syntaxError(U_MALFORMED_SET, rule, start, status);
439 }
440 pos = pp.getIndex();
441 continue;
442 }
443 // Handle escapes
444 if (c == ESCAPE) {
445 if (pos == limit) {
446 return syntaxError(U_TRAILING_BACKSLASH, rule, start, status);
447 }
448 UChar32 escaped = rule.unescapeAt(pos); // pos is already past '\\'
449 if (escaped == (UChar32) -1) {
450 return syntaxError(U_MALFORMED_UNICODE_ESCAPE, rule, start, status);
451 }
452 if (!parser.checkVariableRange(escaped)) {
453 return syntaxError(U_VARIABLE_RANGE_OVERLAP, rule, start, status);
454 }
455 buf.append(escaped);
456 continue;
457 }
458 // Handle quoted matter
459 if (c == QUOTE) {
460 int32_t iq = rule.indexOf(QUOTE, pos);
461 if (iq == pos) {
462 buf.append(c); // Parse [''] outside quotes as [']
463 ++pos;
464 } else {
465 /* This loop picks up a run of quoted text of the
466 * form 'aaaa' each time through. If this run
467 * hasn't really ended ('aaaa''bbbb') then it keeps
468 * looping, each time adding on a new run. When it
469 * reaches the final quote it breaks.
470 */
471 quoteStart = buf.length();
472 for (;;) {
473 if (iq < 0) {
474 return syntaxError(U_UNTERMINATED_QUOTE, rule, start, status);
475 }
476 scratch.truncate(0);
477 rule.extractBetween(pos, iq, scratch);
478 buf.append(scratch);
479 pos = iq+1;
480 if (pos < limit && rule.charAt(pos) == QUOTE) {
481 // Parse [''] inside quotes as [']
482 iq = rule.indexOf(QUOTE, pos+1);
483 // Continue looping
484 } else {
485 break;
486 }
487 }
488 quoteLimit = buf.length();
489
490 for (iq=quoteStart; iq<quoteLimit; ++iq) {
491 if (!parser.checkVariableRange(buf.charAt(iq))) {
492 return syntaxError(U_VARIABLE_RANGE_OVERLAP, rule, start, status);
493 }
494 }
495 }
496 continue;
497 }
498
499 if (!parser.checkVariableRange(c)) {
500 return syntaxError(U_VARIABLE_RANGE_OVERLAP, rule, start, status);
501 }
502
503 if (illegal.indexOf(c) >= 0) {
504 syntaxError(U_ILLEGAL_CHARACTER, rule, start, status);
505 }
506
507 switch (c) {
508
509 //------------------------------------------------------
510 // Elements allowed within and out of segments
511 //------------------------------------------------------
512 case ANCHOR_START:
513 if (buf.length() == 0 && !anchorStart) {
514 anchorStart = TRUE;
515 } else {
516 return syntaxError(U_MISPLACED_ANCHOR_START,
517 rule, start, status);
518 }
519 break;
520 case SEGMENT_OPEN:
521 {
522 // bufSegStart is the offset in buf to the first
523 // character of the segment we are parsing.
524 int32_t bufSegStart = buf.length();
525
526 // Record segment number now, since nextSegmentNumber
527 // will be incremented during the call to parseSection
528 // if there are nested segments.
529 int32_t segmentNumber = nextSegmentNumber++; // 1-based
530
531 // Parse the segment
532 pos = parseSection(rule, pos, limit, buf, UnicodeString(TRUE, ILLEGAL_SEG, -1), TRUE, status);
533
534 // After parsing a segment, the relevant characters are
535 // in buf, starting at offset bufSegStart. Extract them
536 // into a string matcher, and replace them with a
537 // standin for that matcher.
538 StringMatcher* m =
539 new StringMatcher(buf, bufSegStart, buf.length(),
540 segmentNumber, *parser.curData);
541 if (m == NULL) {
542 return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status);
543 }
544
545 // Record and associate object and segment number
546 parser.setSegmentObject(segmentNumber, m, status);
547 buf.truncate(bufSegStart);
548 buf.append(parser.getSegmentStandin(segmentNumber, status));
549 }
550 break;
551 case FUNCTION:
552 case ALT_FUNCTION:
553 {
554 int32_t iref = pos;
555 TransliteratorIDParser::SingleID* single =
556 TransliteratorIDParser::parseFilterID(rule, iref);
557 // The next character MUST be a segment open
558 if (single == NULL ||
559 !ICU_Utility::parseChar(rule, iref, SEGMENT_OPEN)) {
560 return syntaxError(U_INVALID_FUNCTION, rule, start, status);
561 }
562
563 Transliterator *t = single->createInstance();
564 delete single;
565 if (t == NULL) {
566 return syntaxError(U_INVALID_FUNCTION, rule, start, status);
567 }
568
569 // bufSegStart is the offset in buf to the first
570 // character of the segment we are parsing.
571 int32_t bufSegStart = buf.length();
572
573 // Parse the segment
574 pos = parseSection(rule, iref, limit, buf, UnicodeString(TRUE, ILLEGAL_FUNC, -1), TRUE, status);
575
576 // After parsing a segment, the relevant characters are
577 // in buf, starting at offset bufSegStart.
578 UnicodeString output;
579 buf.extractBetween(bufSegStart, buf.length(), output);
580 FunctionReplacer *r =
581 new FunctionReplacer(t, new StringReplacer(output, parser.curData));
582 if (r == NULL) {
583 return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status);
584 }
585
586 // Replace the buffer contents with a stand-in
587 buf.truncate(bufSegStart);
588 buf.append(parser.generateStandInFor(r, status));
589 }
590 break;
591 case SymbolTable::SYMBOL_REF:
592 // Handle variable references and segment references "$1" .. "$9"
593 {
594 // A variable reference must be followed immediately
595 // by a Unicode identifier start and zero or more
596 // Unicode identifier part characters, or by a digit
597 // 1..9 if it is a segment reference.
598 if (pos == limit) {
599 // A variable ref character at the end acts as
600 // an anchor to the context limit, as in perl.
601 anchorEnd = TRUE;
602 break;
603 }
604 // Parse "$1" "$2" .. "$9" .. (no upper limit)
605 c = rule.charAt(pos);
606 int32_t r = u_digit(c, 10);
607 if (r >= 1 && r <= 9) {
608 r = ICU_Utility::parseNumber(rule, pos, 10);
609 if (r < 0) {
610 return syntaxError(U_UNDEFINED_SEGMENT_REFERENCE,
611 rule, start, status);
612 }
613 buf.append(parser.getSegmentStandin(r, status));
614 } else {
615 pp.setIndex(pos);
616 UnicodeString name = parser.parseData->
617 parseReference(rule, pp, limit);
618 if (name.length() == 0) {
619 // This means the '$' was not followed by a
620 // valid name. Try to interpret it as an
621 // end anchor then. If this also doesn't work
622 // (if we see a following character) then signal
623 // an error.
624 anchorEnd = TRUE;
625 break;
626 }
627 pos = pp.getIndex();
628 // If this is a variable definition statement,
629 // then the LHS variable will be undefined. In
630 // that case appendVariableDef() will append the
631 // special placeholder char variableLimit-1.
632 varStart = buf.length();
633 parser.appendVariableDef(name, buf, status);
634 varLimit = buf.length();
635 }
636 }
637 break;
638 case DOT:
639 buf.append(parser.getDotStandIn(status));
640 break;
641 case KLEENE_STAR:
642 case ONE_OR_MORE:
643 case ZERO_OR_ONE:
644 // Quantifiers. We handle single characters, quoted strings,
645 // variable references, and segments.
646 // a+ matches aaa
647 // 'foo'+ matches foofoofoo
648 // $v+ matches xyxyxy if $v == xy
649 // (seg)+ matches segsegseg
650 {
651 if (isSegment && buf.length() == bufStart) {
652 // The */+ immediately follows '('
653 return syntaxError(U_MISPLACED_QUANTIFIER, rule, start, status);
654 }
655
656 int32_t qstart, qlimit;
657 // The */+ follows an isolated character or quote
658 // or variable reference
659 if (buf.length() == quoteLimit) {
660 // The */+ follows a 'quoted string'
661 qstart = quoteStart;
662 qlimit = quoteLimit;
663 } else if (buf.length() == varLimit) {
664 // The */+ follows a $variableReference
665 qstart = varStart;
666 qlimit = varLimit;
667 } else {
668 // The */+ follows a single character, possibly
669 // a segment standin
670 qstart = buf.length() - 1;
671 qlimit = qstart + 1;
672 }
673
674 UnicodeFunctor *m =
675 new StringMatcher(buf, qstart, qlimit, 0, *parser.curData);
676 if (m == NULL) {
677 return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status);
678 }
679 int32_t min = 0;
680 int32_t max = Quantifier::MAX;
681 switch (c) {
682 case ONE_OR_MORE:
683 min = 1;
684 break;
685 case ZERO_OR_ONE:
686 min = 0;
687 max = 1;
688 break;
689 // case KLEENE_STAR:
690 // do nothing -- min, max already set
691 }
692 m = new Quantifier(m, min, max);
693 if (m == NULL) {
694 return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status);
695 }
696 buf.truncate(qstart);
697 buf.append(parser.generateStandInFor(m, status));
698 }
699 break;
700
701 //------------------------------------------------------
702 // Elements allowed ONLY WITHIN segments
703 //------------------------------------------------------
704 case SEGMENT_CLOSE:
705 // assert(isSegment);
706 // We're done parsing a segment.
707 done = TRUE;
708 break;
709
710 //------------------------------------------------------
711 // Elements allowed ONLY OUTSIDE segments
712 //------------------------------------------------------
713 case CONTEXT_ANTE:
714 if (ante >= 0) {
715 return syntaxError(U_MULTIPLE_ANTE_CONTEXTS, rule, start, status);
716 }
717 ante = buf.length();
718 break;
719 case CONTEXT_POST:
720 if (post >= 0) {
721 return syntaxError(U_MULTIPLE_POST_CONTEXTS, rule, start, status);
722 }
723 post = buf.length();
724 break;
725 case CURSOR_POS:
726 if (cursor >= 0) {
727 return syntaxError(U_MULTIPLE_CURSORS, rule, start, status);
728 }
729 cursor = buf.length();
730 break;
731 case CURSOR_OFFSET:
732 if (cursorOffset < 0) {
733 if (buf.length() > 0) {
734 return syntaxError(U_MISPLACED_CURSOR_OFFSET, rule, start, status);
735 }
736 --cursorOffset;
737 } else if (cursorOffset > 0) {
738 if (buf.length() != cursorOffsetPos || cursor >= 0) {
739 return syntaxError(U_MISPLACED_CURSOR_OFFSET, rule, start, status);
740 }
741 ++cursorOffset;
742 } else {
743 if (cursor == 0 && buf.length() == 0) {
744 cursorOffset = -1;
745 } else if (cursor < 0) {
746 cursorOffsetPos = buf.length();
747 cursorOffset = 1;
748 } else {
749 return syntaxError(U_MISPLACED_CURSOR_OFFSET, rule, start, status);
750 }
751 }
752 break;
753
754
755 //------------------------------------------------------
756 // Non-special characters
757 //------------------------------------------------------
758 default:
759 // Disallow unquoted characters other than [0-9A-Za-z]
760 // in the printable ASCII range. These characters are
761 // reserved for possible future use.
762 if (c >= 0x0021 && c <= 0x007E &&
763 !((c >= 0x0030/*'0'*/ && c <= 0x0039/*'9'*/) ||
764 (c >= 0x0041/*'A'*/ && c <= 0x005A/*'Z'*/) ||
765 (c >= 0x0061/*'a'*/ && c <= 0x007A/*'z'*/))) {
766 return syntaxError(U_UNQUOTED_SPECIAL, rule, start, status);
767 }
768 buf.append(c);
769 break;
770 }
771 }
772
773 return pos;
774 }
775
776 /**
777 * Remove context.
778 */
removeContext()779 void RuleHalf::removeContext() {
780 //text = text.substring(ante < 0 ? 0 : ante,
781 // post < 0 ? text.length() : post);
782 if (post >= 0) {
783 text.remove(post);
784 }
785 if (ante >= 0) {
786 text.removeBetween(0, ante);
787 }
788 ante = post = -1;
789 anchorStart = anchorEnd = FALSE;
790 }
791
792 /**
793 * Return true if this half looks like valid output, that is, does not
794 * contain quantifiers or other special input-only elements.
795 */
isValidOutput(TransliteratorParser & transParser)796 UBool RuleHalf::isValidOutput(TransliteratorParser& transParser) {
797 for (int32_t i=0; i<text.length(); ) {
798 UChar32 c = text.char32At(i);
799 i += U16_LENGTH(c);
800 if (!transParser.parseData->isReplacer(c)) {
801 return FALSE;
802 }
803 }
804 return TRUE;
805 }
806
807 /**
808 * Return true if this half looks like valid input, that is, does not
809 * contain functions or other special output-only elements.
810 */
isValidInput(TransliteratorParser & transParser)811 UBool RuleHalf::isValidInput(TransliteratorParser& transParser) {
812 for (int32_t i=0; i<text.length(); ) {
813 UChar32 c = text.char32At(i);
814 i += U16_LENGTH(c);
815 if (!transParser.parseData->isMatcher(c)) {
816 return FALSE;
817 }
818 }
819 return TRUE;
820 }
821
822 //----------------------------------------------------------------------
823 // PUBLIC API
824 //----------------------------------------------------------------------
825
826 /**
827 * Constructor.
828 */
TransliteratorParser(UErrorCode & statusReturn)829 TransliteratorParser::TransliteratorParser(UErrorCode &statusReturn) :
830 dataVector(statusReturn),
831 idBlockVector(statusReturn),
832 variablesVector(statusReturn),
833 segmentObjects(statusReturn)
834 {
835 idBlockVector.setDeleter(uprv_deleteUObject);
836 curData = NULL;
837 compoundFilter = NULL;
838 parseData = NULL;
839 variableNames.setValueDeleter(uprv_deleteUObject);
840 }
841
842 /**
843 * Destructor.
844 */
~TransliteratorParser()845 TransliteratorParser::~TransliteratorParser() {
846 while (!dataVector.isEmpty())
847 delete (TransliterationRuleData*)(dataVector.orphanElementAt(0));
848 delete compoundFilter;
849 delete parseData;
850 while (!variablesVector.isEmpty())
851 delete (UnicodeFunctor*)variablesVector.orphanElementAt(0);
852 }
853
854 void
parse(const UnicodeString & rules,UTransDirection transDirection,UParseError & pe,UErrorCode & ec)855 TransliteratorParser::parse(const UnicodeString& rules,
856 UTransDirection transDirection,
857 UParseError& pe,
858 UErrorCode& ec) {
859 if (U_SUCCESS(ec)) {
860 parseRules(rules, transDirection, ec);
861 pe = parseError;
862 }
863 }
864
865 /**
866 * Return the compound filter parsed by parse(). Caller owns result.
867 */
orphanCompoundFilter()868 UnicodeSet* TransliteratorParser::orphanCompoundFilter() {
869 UnicodeSet* f = compoundFilter;
870 compoundFilter = NULL;
871 return f;
872 }
873
874 //----------------------------------------------------------------------
875 // Private implementation
876 //----------------------------------------------------------------------
877
878 /**
879 * Parse the given string as a sequence of rules, separated by newline
880 * characters ('\n'), and cause this object to implement those rules. Any
881 * previous rules are discarded. Typically this method is called exactly
882 * once, during construction.
883 * @exception IllegalArgumentException if there is a syntax error in the
884 * rules
885 */
parseRules(const UnicodeString & rule,UTransDirection theDirection,UErrorCode & status)886 void TransliteratorParser::parseRules(const UnicodeString& rule,
887 UTransDirection theDirection,
888 UErrorCode& status)
889 {
890 // Clear error struct
891 uprv_memset(&parseError, 0, sizeof(parseError));
892 parseError.line = parseError.offset = -1;
893
894 UBool parsingIDs = TRUE;
895 int32_t ruleCount = 0;
896
897 while (!dataVector.isEmpty()) {
898 delete (TransliterationRuleData*)(dataVector.orphanElementAt(0));
899 }
900 if (U_FAILURE(status)) {
901 return;
902 }
903
904 idBlockVector.removeAllElements();
905 curData = NULL;
906 direction = theDirection;
907 ruleCount = 0;
908
909 delete compoundFilter;
910 compoundFilter = NULL;
911
912 while (!variablesVector.isEmpty()) {
913 delete (UnicodeFunctor*)variablesVector.orphanElementAt(0);
914 }
915 variableNames.removeAll();
916 parseData = new ParseData(0, &variablesVector, &variableNames);
917 if (parseData == NULL) {
918 status = U_MEMORY_ALLOCATION_ERROR;
919 return;
920 }
921
922 dotStandIn = (UChar) -1;
923
924 UnicodeString *tempstr = NULL; // used for memory allocation error checking
925 UnicodeString str; // scratch
926 UnicodeString idBlockResult;
927 int32_t pos = 0;
928 int32_t limit = rule.length();
929
930 // The compound filter offset is an index into idBlockResult.
931 // If it is 0, then the compound filter occurred at the start,
932 // and it is the offset to the _start_ of the compound filter
933 // pattern. Otherwise it is the offset to the _limit_ of the
934 // compound filter pattern within idBlockResult.
935 compoundFilter = NULL;
936 int32_t compoundFilterOffset = -1;
937
938 while (pos < limit && U_SUCCESS(status)) {
939 UChar c = rule.charAt(pos++);
940 if (PatternProps::isWhiteSpace(c)) {
941 // Ignore leading whitespace.
942 continue;
943 }
944 // Skip lines starting with the comment character
945 if (c == RULE_COMMENT_CHAR) {
946 pos = rule.indexOf((UChar)0x000A /*\n*/, pos) + 1;
947 if (pos == 0) {
948 break; // No "\n" found; rest of rule is a comment
949 }
950 continue; // Either fall out or restart with next line
951 }
952
953 // skip empty rules
954 if (c == END_OF_RULE)
955 continue;
956
957 // keep track of how many rules we've seen
958 ++ruleCount;
959
960 // We've found the start of a rule or ID. c is its first
961 // character, and pos points past c.
962 --pos;
963 // Look for an ID token. Must have at least ID_TOKEN_LEN + 1
964 // chars left.
965 if ((pos + ID_TOKEN_LEN + 1) <= limit &&
966 rule.compare(pos, ID_TOKEN_LEN, ID_TOKEN) == 0) {
967 pos += ID_TOKEN_LEN;
968 c = rule.charAt(pos);
969 while (PatternProps::isWhiteSpace(c) && pos < limit) {
970 ++pos;
971 c = rule.charAt(pos);
972 }
973
974 int32_t p = pos;
975
976 if (!parsingIDs) {
977 if (curData != NULL) {
978 U_ASSERT(!dataVector.hasDeleter());
979 if (direction == UTRANS_FORWARD)
980 dataVector.addElement(curData, status);
981 else
982 dataVector.insertElementAt(curData, 0, status);
983 if (U_FAILURE(status)) {
984 delete curData;
985 }
986 curData = NULL;
987 }
988 parsingIDs = TRUE;
989 }
990
991 TransliteratorIDParser::SingleID* id =
992 TransliteratorIDParser::parseSingleID(rule, p, direction, status);
993 if (p != pos && ICU_Utility::parseChar(rule, p, END_OF_RULE)) {
994 // Successful ::ID parse.
995
996 if (direction == UTRANS_FORWARD) {
997 idBlockResult.append(id->canonID).append(END_OF_RULE);
998 } else {
999 idBlockResult.insert(0, END_OF_RULE);
1000 idBlockResult.insert(0, id->canonID);
1001 }
1002
1003 } else {
1004 // Couldn't parse an ID. Try to parse a global filter
1005 int32_t withParens = -1;
1006 UnicodeSet* f = TransliteratorIDParser::parseGlobalFilter(rule, p, direction, withParens, NULL);
1007 if (f != NULL) {
1008 if (ICU_Utility::parseChar(rule, p, END_OF_RULE)
1009 && (direction == UTRANS_FORWARD) == (withParens == 0))
1010 {
1011 if (compoundFilter != NULL) {
1012 // Multiple compound filters
1013 syntaxError(U_MULTIPLE_COMPOUND_FILTERS, rule, pos, status);
1014 delete f;
1015 } else {
1016 compoundFilter = f;
1017 compoundFilterOffset = ruleCount;
1018 }
1019 } else {
1020 delete f;
1021 }
1022 } else {
1023 // Invalid ::id
1024 // Can be parsed as neither an ID nor a global filter
1025 syntaxError(U_INVALID_ID, rule, pos, status);
1026 }
1027 }
1028 delete id;
1029 pos = p;
1030 } else {
1031 if (parsingIDs) {
1032 tempstr = new UnicodeString(idBlockResult);
1033 // NULL pointer check
1034 if (tempstr == NULL) {
1035 status = U_MEMORY_ALLOCATION_ERROR;
1036 return;
1037 }
1038 U_ASSERT(idBlockVector.hasDeleter());
1039 if (direction == UTRANS_FORWARD)
1040 idBlockVector.adoptElement(tempstr, status);
1041 else
1042 idBlockVector.insertElementAt(tempstr, 0, status);
1043 if (U_FAILURE(status)) {
1044 return;
1045 }
1046 idBlockResult.remove();
1047 parsingIDs = FALSE;
1048 curData = new TransliterationRuleData(status);
1049 // NULL pointer check
1050 if (curData == NULL) {
1051 status = U_MEMORY_ALLOCATION_ERROR;
1052 return;
1053 }
1054 parseData->data = curData;
1055
1056 // By default, rules use part of the private use area
1057 // E000..F8FF for variables and other stand-ins. Currently
1058 // the range F000..F8FF is typically sufficient. The 'use
1059 // variable range' pragma allows rule sets to modify this.
1060 setVariableRange(0xF000, 0xF8FF, status);
1061 }
1062
1063 if (resemblesPragma(rule, pos, limit)) {
1064 int32_t ppp = parsePragma(rule, pos, limit, status);
1065 if (ppp < 0) {
1066 syntaxError(U_MALFORMED_PRAGMA, rule, pos, status);
1067 }
1068 pos = ppp;
1069 // Parse a rule
1070 } else {
1071 pos = parseRule(rule, pos, limit, status);
1072 }
1073 }
1074 }
1075
1076 if (parsingIDs && idBlockResult.length() > 0) {
1077 tempstr = new UnicodeString(idBlockResult);
1078 // NULL pointer check
1079 if (tempstr == NULL) {
1080 // TODO: Testing, forcing this path, shows many memory leaks. ICU-21701
1081 // intltest translit/TransliteratorTest/TestInstantiation
1082 status = U_MEMORY_ALLOCATION_ERROR;
1083 return;
1084 }
1085 if (direction == UTRANS_FORWARD)
1086 idBlockVector.adoptElement(tempstr, status);
1087 else
1088 idBlockVector.insertElementAt(tempstr, 0, status);
1089 if (U_FAILURE(status)) {
1090 return;
1091 }
1092 }
1093 else if (!parsingIDs && curData != NULL) {
1094 if (direction == UTRANS_FORWARD) {
1095 dataVector.addElement(curData, status);
1096 } else {
1097 dataVector.insertElementAt(curData, 0, status);
1098 }
1099 if (U_FAILURE(status)) {
1100 delete curData;
1101 curData = nullptr;
1102 }
1103 }
1104
1105 if (U_SUCCESS(status)) {
1106 // Convert the set vector to an array
1107 int32_t i, dataVectorSize = dataVector.size();
1108 for (i = 0; i < dataVectorSize; i++) {
1109 TransliterationRuleData* data = (TransliterationRuleData*)dataVector.elementAt(i);
1110 data->variablesLength = variablesVector.size();
1111 if (data->variablesLength == 0) {
1112 data->variables = 0;
1113 } else {
1114 data->variables = (UnicodeFunctor**)uprv_malloc(data->variablesLength * sizeof(UnicodeFunctor*));
1115 // NULL pointer check
1116 if (data->variables == NULL) {
1117 status = U_MEMORY_ALLOCATION_ERROR;
1118 return;
1119 }
1120 data->variablesAreOwned = (i == 0);
1121 }
1122
1123 for (int32_t j = 0; j < data->variablesLength; j++) {
1124 data->variables[j] =
1125 static_cast<UnicodeFunctor *>(variablesVector.elementAt(j));
1126 }
1127
1128 data->variableNames.removeAll();
1129 int32_t p = UHASH_FIRST;
1130 const UHashElement* he = variableNames.nextElement(p);
1131 while (he != NULL) {
1132 UnicodeString* tempus = ((UnicodeString*)(he->value.pointer))->clone();
1133 if (tempus == NULL) {
1134 status = U_MEMORY_ALLOCATION_ERROR;
1135 return;
1136 }
1137 data->variableNames.put(*((UnicodeString*)(he->key.pointer)),
1138 tempus, status);
1139 he = variableNames.nextElement(p);
1140 }
1141 }
1142 variablesVector.removeAllElements(); // keeps them from getting deleted when we succeed
1143
1144 // Index the rules
1145 if (compoundFilter != NULL) {
1146 if ((direction == UTRANS_FORWARD && compoundFilterOffset != 1) ||
1147 (direction == UTRANS_REVERSE && compoundFilterOffset != ruleCount)) {
1148 status = U_MISPLACED_COMPOUND_FILTER;
1149 }
1150 }
1151
1152 for (i = 0; i < dataVectorSize; i++) {
1153 TransliterationRuleData* data = (TransliterationRuleData*)dataVector.elementAt(i);
1154 data->ruleSet.freeze(parseError, status);
1155 }
1156 if (idBlockVector.size() == 1 && ((UnicodeString*)idBlockVector.elementAt(0))->isEmpty()) {
1157 idBlockVector.removeElementAt(0);
1158 }
1159 }
1160 }
1161
1162 /**
1163 * Set the variable range to [start, end] (inclusive).
1164 */
setVariableRange(int32_t start,int32_t end,UErrorCode & status)1165 void TransliteratorParser::setVariableRange(int32_t start, int32_t end, UErrorCode& status) {
1166 if (start > end || start < 0 || end > 0xFFFF) {
1167 status = U_MALFORMED_PRAGMA;
1168 return;
1169 }
1170
1171 curData->variablesBase = (UChar) start;
1172 if (dataVector.size() == 0) {
1173 variableNext = (UChar) start;
1174 variableLimit = (UChar) (end + 1);
1175 }
1176 }
1177
1178 /**
1179 * Assert that the given character is NOT within the variable range.
1180 * If it is, return FALSE. This is necessary to ensure that the
1181 * variable range does not overlap characters used in a rule.
1182 */
checkVariableRange(UChar32 ch) const1183 UBool TransliteratorParser::checkVariableRange(UChar32 ch) const {
1184 return !(ch >= curData->variablesBase && ch < variableLimit);
1185 }
1186
1187 /**
1188 * Set the maximum backup to 'backup', in response to a pragma
1189 * statement.
1190 */
pragmaMaximumBackup(int32_t)1191 void TransliteratorParser::pragmaMaximumBackup(int32_t /*backup*/) {
1192 //TODO Finish
1193 }
1194
1195 /**
1196 * Begin normalizing all rules using the given mode, in response
1197 * to a pragma statement.
1198 */
pragmaNormalizeRules(UNormalizationMode)1199 void TransliteratorParser::pragmaNormalizeRules(UNormalizationMode /*mode*/) {
1200 //TODO Finish
1201 }
1202
1203 static const UChar PRAGMA_USE[] = {0x75,0x73,0x65,0x20,0}; // "use "
1204
1205 static const UChar PRAGMA_VARIABLE_RANGE[] = {0x7E,0x76,0x61,0x72,0x69,0x61,0x62,0x6C,0x65,0x20,0x72,0x61,0x6E,0x67,0x65,0x20,0x23,0x20,0x23,0x7E,0x3B,0}; // "~variable range # #~;"
1206
1207 static const UChar PRAGMA_MAXIMUM_BACKUP[] = {0x7E,0x6D,0x61,0x78,0x69,0x6D,0x75,0x6D,0x20,0x62,0x61,0x63,0x6B,0x75,0x70,0x20,0x23,0x7E,0x3B,0}; // "~maximum backup #~;"
1208
1209 static const UChar PRAGMA_NFD_RULES[] = {0x7E,0x6E,0x66,0x64,0x20,0x72,0x75,0x6C,0x65,0x73,0x7E,0x3B,0}; // "~nfd rules~;"
1210
1211 static const UChar PRAGMA_NFC_RULES[] = {0x7E,0x6E,0x66,0x63,0x20,0x72,0x75,0x6C,0x65,0x73,0x7E,0x3B,0}; // "~nfc rules~;"
1212
1213 /**
1214 * Return true if the given rule looks like a pragma.
1215 * @param pos offset to the first non-whitespace character
1216 * of the rule.
1217 * @param limit pointer past the last character of the rule.
1218 */
resemblesPragma(const UnicodeString & rule,int32_t pos,int32_t limit)1219 UBool TransliteratorParser::resemblesPragma(const UnicodeString& rule, int32_t pos, int32_t limit) {
1220 // Must start with /use\s/i
1221 return ICU_Utility::parsePattern(rule, pos, limit, UnicodeString(TRUE, PRAGMA_USE, 4), NULL) >= 0;
1222 }
1223
1224 /**
1225 * Parse a pragma. This method assumes resemblesPragma() has
1226 * already returned true.
1227 * @param pos offset to the first non-whitespace character
1228 * of the rule.
1229 * @param limit pointer past the last character of the rule.
1230 * @return the position index after the final ';' of the pragma,
1231 * or -1 on failure.
1232 */
parsePragma(const UnicodeString & rule,int32_t pos,int32_t limit,UErrorCode & status)1233 int32_t TransliteratorParser::parsePragma(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status) {
1234 int32_t array[2];
1235
1236 // resemblesPragma() has already returned true, so we
1237 // know that pos points to /use\s/i; we can skip 4 characters
1238 // immediately
1239 pos += 4;
1240
1241 // Here are the pragmas we recognize:
1242 // use variable range 0xE000 0xEFFF;
1243 // use maximum backup 16;
1244 // use nfd rules;
1245 // use nfc rules;
1246 int p = ICU_Utility::parsePattern(rule, pos, limit, UnicodeString(TRUE, PRAGMA_VARIABLE_RANGE, -1), array);
1247 if (p >= 0) {
1248 setVariableRange(array[0], array[1], status);
1249 return p;
1250 }
1251
1252 p = ICU_Utility::parsePattern(rule, pos, limit, UnicodeString(TRUE, PRAGMA_MAXIMUM_BACKUP, -1), array);
1253 if (p >= 0) {
1254 pragmaMaximumBackup(array[0]);
1255 return p;
1256 }
1257
1258 p = ICU_Utility::parsePattern(rule, pos, limit, UnicodeString(TRUE, PRAGMA_NFD_RULES, -1), NULL);
1259 if (p >= 0) {
1260 pragmaNormalizeRules(UNORM_NFD);
1261 return p;
1262 }
1263
1264 p = ICU_Utility::parsePattern(rule, pos, limit, UnicodeString(TRUE, PRAGMA_NFC_RULES, -1), NULL);
1265 if (p >= 0) {
1266 pragmaNormalizeRules(UNORM_NFC);
1267 return p;
1268 }
1269
1270 // Syntax error: unable to parse pragma
1271 return -1;
1272 }
1273
1274 /**
1275 * MAIN PARSER. Parse the next rule in the given rule string, starting
1276 * at pos. Return the index after the last character parsed. Do not
1277 * parse characters at or after limit.
1278 *
1279 * Important: The character at pos must be a non-whitespace character
1280 * that is not the comment character.
1281 *
1282 * This method handles quoting, escaping, and whitespace removal. It
1283 * parses the end-of-rule character. It recognizes context and cursor
1284 * indicators. Once it does a lexical breakdown of the rule at pos, it
1285 * creates a rule object and adds it to our rule list.
1286 */
parseRule(const UnicodeString & rule,int32_t pos,int32_t limit,UErrorCode & status)1287 int32_t TransliteratorParser::parseRule(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status) {
1288 // Locate the left side, operator, and right side
1289 int32_t start = pos;
1290 UChar op = 0;
1291 int32_t i;
1292
1293 // Set up segments data
1294 segmentStandins.truncate(0);
1295 segmentObjects.removeAllElements();
1296
1297 // Use pointers to automatics to make swapping possible.
1298 RuleHalf _left(*this), _right(*this);
1299 RuleHalf* left = &_left;
1300 RuleHalf* right = &_right;
1301
1302 undefinedVariableName.remove();
1303 pos = left->parse(rule, pos, limit, status);
1304 if (U_FAILURE(status)) {
1305 return start;
1306 }
1307
1308 if (pos == limit || u_strchr(gOPERATORS, (op = rule.charAt(--pos))) == NULL) {
1309 return syntaxError(U_MISSING_OPERATOR, rule, start, status);
1310 }
1311 ++pos;
1312
1313 // Found an operator char. Check for forward-reverse operator.
1314 if (op == REVERSE_RULE_OP &&
1315 (pos < limit && rule.charAt(pos) == FORWARD_RULE_OP)) {
1316 ++pos;
1317 op = FWDREV_RULE_OP;
1318 }
1319
1320 // Translate alternate op characters.
1321 switch (op) {
1322 case ALT_FORWARD_RULE_OP:
1323 op = FORWARD_RULE_OP;
1324 break;
1325 case ALT_REVERSE_RULE_OP:
1326 op = REVERSE_RULE_OP;
1327 break;
1328 case ALT_FWDREV_RULE_OP:
1329 op = FWDREV_RULE_OP;
1330 break;
1331 }
1332
1333 pos = right->parse(rule, pos, limit, status);
1334 if (U_FAILURE(status)) {
1335 return start;
1336 }
1337
1338 if (pos < limit) {
1339 if (rule.charAt(--pos) == END_OF_RULE) {
1340 ++pos;
1341 } else {
1342 // RuleHalf parser must have terminated at an operator
1343 return syntaxError(U_UNQUOTED_SPECIAL, rule, start, status);
1344 }
1345 }
1346
1347 if (op == VARIABLE_DEF_OP) {
1348 // LHS is the name. RHS is a single character, either a literal
1349 // or a set (already parsed). If RHS is longer than one
1350 // character, it is either a multi-character string, or multiple
1351 // sets, or a mixture of chars and sets -- syntax error.
1352
1353 // We expect to see a single undefined variable (the one being
1354 // defined).
1355 if (undefinedVariableName.length() == 0) {
1356 // "Missing '$' or duplicate definition"
1357 return syntaxError(U_BAD_VARIABLE_DEFINITION, rule, start, status);
1358 }
1359 if (left->text.length() != 1 || left->text.charAt(0) != variableLimit) {
1360 // "Malformed LHS"
1361 return syntaxError(U_MALFORMED_VARIABLE_DEFINITION, rule, start, status);
1362 }
1363 if (left->anchorStart || left->anchorEnd ||
1364 right->anchorStart || right->anchorEnd) {
1365 return syntaxError(U_MALFORMED_VARIABLE_DEFINITION, rule, start, status);
1366 }
1367 // We allow anything on the right, including an empty string.
1368 UnicodeString* value = new UnicodeString(right->text);
1369 // NULL pointer check
1370 if (value == NULL) {
1371 return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status);
1372 }
1373 variableNames.put(undefinedVariableName, value, status);
1374 ++variableLimit;
1375 return pos;
1376 }
1377
1378 // If this is not a variable definition rule, we shouldn't have
1379 // any undefined variable names.
1380 if (undefinedVariableName.length() != 0) {
1381 return syntaxError(// "Undefined variable $" + undefinedVariableName,
1382 U_UNDEFINED_VARIABLE,
1383 rule, start, status);
1384 }
1385
1386 // Verify segments
1387 if (segmentStandins.length() > segmentObjects.size()) {
1388 syntaxError(U_UNDEFINED_SEGMENT_REFERENCE, rule, start, status);
1389 }
1390 for (i=0; i<segmentStandins.length(); ++i) {
1391 if (segmentStandins.charAt(i) == 0) {
1392 syntaxError(U_INTERNAL_TRANSLITERATOR_ERROR, rule, start, status); // will never happen
1393 }
1394 }
1395 for (i=0; i<segmentObjects.size(); ++i) {
1396 if (segmentObjects.elementAt(i) == NULL) {
1397 syntaxError(U_INTERNAL_TRANSLITERATOR_ERROR, rule, start, status); // will never happen
1398 }
1399 }
1400
1401 // If the direction we want doesn't match the rule
1402 // direction, do nothing.
1403 if (op != FWDREV_RULE_OP &&
1404 ((direction == UTRANS_FORWARD) != (op == FORWARD_RULE_OP))) {
1405 return pos;
1406 }
1407
1408 // Transform the rule into a forward rule by swapping the
1409 // sides if necessary.
1410 if (direction == UTRANS_REVERSE) {
1411 left = &_right;
1412 right = &_left;
1413 }
1414
1415 // Remove non-applicable elements in forward-reverse
1416 // rules. Bidirectional rules ignore elements that do not
1417 // apply.
1418 if (op == FWDREV_RULE_OP) {
1419 right->removeContext();
1420 left->cursor = -1;
1421 left->cursorOffset = 0;
1422 }
1423
1424 // Normalize context
1425 if (left->ante < 0) {
1426 left->ante = 0;
1427 }
1428 if (left->post < 0) {
1429 left->post = left->text.length();
1430 }
1431
1432 // Context is only allowed on the input side. Cursors are only
1433 // allowed on the output side. Segment delimiters can only appear
1434 // on the left, and references on the right. Cursor offset
1435 // cannot appear without an explicit cursor. Cursor offset
1436 // cannot place the cursor outside the limits of the context.
1437 // Anchors are only allowed on the input side.
1438 if (right->ante >= 0 || right->post >= 0 || left->cursor >= 0 ||
1439 (right->cursorOffset != 0 && right->cursor < 0) ||
1440 // - The following two checks were used to ensure that the
1441 // - the cursor offset stayed within the ante- or postcontext.
1442 // - However, with the addition of quantifiers, we have to
1443 // - allow arbitrary cursor offsets and do runtime checking.
1444 //(right->cursorOffset > (left->text.length() - left->post)) ||
1445 //(-right->cursorOffset > left->ante) ||
1446 right->anchorStart || right->anchorEnd ||
1447 !left->isValidInput(*this) || !right->isValidOutput(*this) ||
1448 left->ante > left->post) {
1449
1450 return syntaxError(U_MALFORMED_RULE, rule, start, status);
1451 }
1452
1453 // Flatten segment objects vector to an array
1454 UnicodeFunctor** segmentsArray = NULL;
1455 if (segmentObjects.size() > 0) {
1456 segmentsArray = (UnicodeFunctor **)uprv_malloc(segmentObjects.size() * sizeof(UnicodeFunctor *));
1457 // Null pointer check
1458 if (segmentsArray == NULL) {
1459 return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status);
1460 }
1461 segmentObjects.toArray((void**) segmentsArray);
1462 }
1463 TransliterationRule* temptr = new TransliterationRule(
1464 left->text, left->ante, left->post,
1465 right->text, right->cursor, right->cursorOffset,
1466 segmentsArray,
1467 segmentObjects.size(),
1468 left->anchorStart, left->anchorEnd,
1469 curData,
1470 status);
1471 //Null pointer check
1472 if (temptr == NULL) {
1473 uprv_free(segmentsArray);
1474 return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status);
1475 }
1476
1477 curData->ruleSet.addRule(temptr, status);
1478
1479 return pos;
1480 }
1481
1482 /**
1483 * Called by main parser upon syntax error. Search the rule string
1484 * for the probable end of the rule. Of course, if the error is that
1485 * the end of rule marker is missing, then the rule end will not be found.
1486 * In any case the rule start will be correctly reported.
1487 * @param msg error description
1488 * @param rule pattern string
1489 * @param start position of first character of current rule
1490 */
syntaxError(UErrorCode parseErrorCode,const UnicodeString & rule,int32_t pos,UErrorCode & status)1491 int32_t TransliteratorParser::syntaxError(UErrorCode parseErrorCode,
1492 const UnicodeString& rule,
1493 int32_t pos,
1494 UErrorCode& status)
1495 {
1496 parseError.offset = pos;
1497 parseError.line = 0 ; /* we are not using line numbers */
1498
1499 // for pre-context
1500 const int32_t LEN = U_PARSE_CONTEXT_LEN - 1;
1501 int32_t start = uprv_max(pos - LEN, 0);
1502 int32_t stop = pos;
1503
1504 rule.extract(start,stop-start,parseError.preContext);
1505 //null terminate the buffer
1506 parseError.preContext[stop-start] = 0;
1507
1508 //for post-context
1509 start = pos;
1510 stop = uprv_min(pos + LEN, rule.length());
1511
1512 rule.extract(start,stop-start,parseError.postContext);
1513 //null terminate the buffer
1514 parseError.postContext[stop-start]= 0;
1515
1516 status = (UErrorCode)parseErrorCode;
1517 return pos;
1518
1519 }
1520
1521 /**
1522 * Parse a UnicodeSet out, store it, and return the stand-in character
1523 * used to represent it.
1524 */
parseSet(const UnicodeString & rule,ParsePosition & pos,UErrorCode & status)1525 UChar TransliteratorParser::parseSet(const UnicodeString& rule,
1526 ParsePosition& pos,
1527 UErrorCode& status) {
1528 UnicodeSet* set = new UnicodeSet(rule, pos, USET_IGNORE_SPACE, parseData, status);
1529 // Null pointer check
1530 if (set == NULL) {
1531 status = U_MEMORY_ALLOCATION_ERROR;
1532 return (UChar)0x0000; // Return empty character with error.
1533 }
1534 set->compact();
1535 return generateStandInFor(set, status);
1536 }
1537
1538 /**
1539 * Generate and return a stand-in for a new UnicodeFunctor. Store
1540 * the matcher (adopt it).
1541 */
generateStandInFor(UnicodeFunctor * adopted,UErrorCode & status)1542 UChar TransliteratorParser::generateStandInFor(UnicodeFunctor* adopted, UErrorCode& status) {
1543 // assert(obj != null);
1544
1545 // Look up previous stand-in, if any. This is a short list
1546 // (typical n is 0, 1, or 2); linear search is optimal.
1547 for (int32_t i=0; i<variablesVector.size(); ++i) {
1548 if (variablesVector.elementAt(i) == adopted) { // [sic] pointer comparison
1549 return (UChar) (curData->variablesBase + i);
1550 }
1551 }
1552
1553 if (variableNext >= variableLimit) {
1554 delete adopted;
1555 status = U_VARIABLE_RANGE_EXHAUSTED;
1556 return 0;
1557 }
1558 variablesVector.addElement(adopted, status);
1559 if (U_FAILURE(status)) {
1560 delete adopted;
1561 return 0;
1562 }
1563 return variableNext++;
1564 }
1565
1566 /**
1567 * Return the standin for segment seg (1-based).
1568 */
getSegmentStandin(int32_t seg,UErrorCode & status)1569 UChar TransliteratorParser::getSegmentStandin(int32_t seg, UErrorCode& status) {
1570 // Special character used to indicate an empty spot
1571 UChar empty = curData->variablesBase - 1;
1572 while (segmentStandins.length() < seg) {
1573 segmentStandins.append(empty);
1574 }
1575 UChar c = segmentStandins.charAt(seg-1);
1576 if (c == empty) {
1577 if (variableNext >= variableLimit) {
1578 status = U_VARIABLE_RANGE_EXHAUSTED;
1579 return 0;
1580 }
1581 c = variableNext++;
1582 // Set a placeholder in the primary variables vector that will be
1583 // filled in later by setSegmentObject(). We know that we will get
1584 // called first because setSegmentObject() will call us.
1585 variablesVector.addElement((void*) NULL, status);
1586 segmentStandins.setCharAt(seg-1, c);
1587 }
1588 return c;
1589 }
1590
1591 /**
1592 * Set the object for segment seg (1-based).
1593 */
setSegmentObject(int32_t seg,StringMatcher * adopted,UErrorCode & status)1594 void TransliteratorParser::setSegmentObject(int32_t seg, StringMatcher* adopted, UErrorCode& status) {
1595 // Since we call parseSection() recursively, nested
1596 // segments will result in segment i+1 getting parsed
1597 // and stored before segment i; be careful with the
1598 // vector handling here.
1599 if (segmentObjects.size() < seg) {
1600 segmentObjects.setSize(seg, status);
1601 }
1602 if (U_FAILURE(status)) {
1603 return;
1604 }
1605 int32_t index = getSegmentStandin(seg, status) - curData->variablesBase;
1606 if (segmentObjects.elementAt(seg-1) != NULL ||
1607 variablesVector.elementAt(index) != NULL) {
1608 // should never happen
1609 if (U_SUCCESS(status)) {status = U_INTERNAL_TRANSLITERATOR_ERROR;}
1610 return;
1611 }
1612 // Note: neither segmentObjects or variablesVector has an object deleter function.
1613 segmentObjects.setElementAt(adopted, seg-1);
1614 variablesVector.setElementAt(adopted, index);
1615 }
1616
1617 /**
1618 * Return the stand-in for the dot set. It is allocated the first
1619 * time and reused thereafter.
1620 */
getDotStandIn(UErrorCode & status)1621 UChar TransliteratorParser::getDotStandIn(UErrorCode& status) {
1622 if (dotStandIn == (UChar) -1) {
1623 UnicodeSet* tempus = new UnicodeSet(UnicodeString(TRUE, DOT_SET, -1), status);
1624 // Null pointer check.
1625 if (tempus == NULL) {
1626 status = U_MEMORY_ALLOCATION_ERROR;
1627 return (UChar)0x0000;
1628 }
1629 dotStandIn = generateStandInFor(tempus, status);
1630 }
1631 return dotStandIn;
1632 }
1633
1634 /**
1635 * Append the value of the given variable name to the given
1636 * UnicodeString.
1637 */
appendVariableDef(const UnicodeString & name,UnicodeString & buf,UErrorCode & status)1638 void TransliteratorParser::appendVariableDef(const UnicodeString& name,
1639 UnicodeString& buf,
1640 UErrorCode& status) {
1641 const UnicodeString* s = (const UnicodeString*) variableNames.get(name);
1642 if (s == NULL) {
1643 // We allow one undefined variable so that variable definition
1644 // statements work. For the first undefined variable we return
1645 // the special placeholder variableLimit-1, and save the variable
1646 // name.
1647 if (undefinedVariableName.length() == 0) {
1648 undefinedVariableName = name;
1649 if (variableNext >= variableLimit) {
1650 // throw new RuntimeException("Private use variables exhausted");
1651 status = U_ILLEGAL_ARGUMENT_ERROR;
1652 return;
1653 }
1654 buf.append((UChar) --variableLimit);
1655 } else {
1656 //throw new IllegalArgumentException("Undefined variable $"
1657 // + name);
1658 status = U_ILLEGAL_ARGUMENT_ERROR;
1659 return;
1660 }
1661 } else {
1662 buf.append(*s);
1663 }
1664 }
1665
1666 /**
1667 * Glue method to get around access restrictions in C++.
1668 */
1669 /*Transliterator* TransliteratorParser::createBasicInstance(const UnicodeString& id, const UnicodeString* canonID) {
1670 return Transliterator::createBasicInstance(id, canonID);
1671 }*/
1672
1673 U_NAMESPACE_END
1674
1675 U_CAPI int32_t
utrans_stripRules(const UChar * source,int32_t sourceLen,UChar * target,UErrorCode * status)1676 utrans_stripRules(const UChar *source, int32_t sourceLen, UChar *target, UErrorCode *status) {
1677 U_NAMESPACE_USE
1678
1679 //const UChar *sourceStart = source;
1680 const UChar *targetStart = target;
1681 const UChar *sourceLimit = source+sourceLen;
1682 UChar *targetLimit = target+sourceLen;
1683 UChar32 c = 0;
1684 UBool quoted = FALSE;
1685 int32_t index;
1686
1687 uprv_memset(target, 0, sourceLen*U_SIZEOF_UCHAR);
1688
1689 /* read the rules into the buffer */
1690 while (source < sourceLimit)
1691 {
1692 index=0;
1693 U16_NEXT_UNSAFE(source, index, c);
1694 source+=index;
1695 if(c == QUOTE) {
1696 quoted = (UBool)!quoted;
1697 }
1698 else if (!quoted) {
1699 if (c == RULE_COMMENT_CHAR) {
1700 /* skip comments and all preceding spaces */
1701 while (targetStart < target && *(target - 1) == 0x0020) {
1702 target--;
1703 }
1704 do {
1705 if (source == sourceLimit) {
1706 c = U_SENTINEL;
1707 break;
1708 }
1709 c = *(source++);
1710 }
1711 while (c != CR && c != LF);
1712 if (c < 0) {
1713 break;
1714 }
1715 }
1716 else if (c == ESCAPE && source < sourceLimit) {
1717 UChar32 c2 = *source;
1718 if (c2 == CR || c2 == LF) {
1719 /* A backslash at the end of a line. */
1720 /* Since we're stripping lines, ignore the backslash. */
1721 source++;
1722 continue;
1723 }
1724 if (c2 == 0x0075 && source+5 < sourceLimit) { /* \u seen. \U isn't unescaped. */
1725 int32_t escapeOffset = 0;
1726 UnicodeString escapedStr(source, 5);
1727 c2 = escapedStr.unescapeAt(escapeOffset);
1728
1729 if (c2 == (UChar32)0xFFFFFFFF || escapeOffset == 0)
1730 {
1731 *status = U_PARSE_ERROR;
1732 return 0;
1733 }
1734 if (!PatternProps::isWhiteSpace(c2) && !u_iscntrl(c2) && !u_ispunct(c2)) {
1735 /* It was escaped for a reason. Write what it was suppose to be. */
1736 source+=5;
1737 c = c2;
1738 }
1739 }
1740 else if (c2 == QUOTE) {
1741 /* \' seen. Make sure we don't do anything when we see it again. */
1742 quoted = (UBool)!quoted;
1743 }
1744 }
1745 }
1746 if (c == CR || c == LF)
1747 {
1748 /* ignore spaces carriage returns, and all leading spaces on the next line.
1749 * and line feed unless in the form \uXXXX
1750 */
1751 quoted = FALSE;
1752 while (source < sourceLimit) {
1753 c = *(source);
1754 if (c != CR && c != LF && c != 0x0020) {
1755 break;
1756 }
1757 source++;
1758 }
1759 continue;
1760 }
1761
1762 /* Append UChar * after dissembling if c > 0xffff*/
1763 index=0;
1764 U16_APPEND_UNSAFE(target, index, c);
1765 target+=index;
1766 }
1767 if (target < targetLimit) {
1768 *target = 0;
1769 }
1770 return (int32_t)(target-targetStart);
1771 }
1772
1773 #endif /* #if !UCONFIG_NO_TRANSLITERATION */
1774