1 //===- unittests/Lex/LexerTest.cpp ------ Lexer tests ---------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9
10 #include "clang/Lex/Lexer.h"
11 #include "clang/Basic/Diagnostic.h"
12 #include "clang/Basic/DiagnosticOptions.h"
13 #include "clang/Basic/FileManager.h"
14 #include "clang/Basic/LangOptions.h"
15 #include "clang/Basic/MemoryBufferCache.h"
16 #include "clang/Basic/SourceManager.h"
17 #include "clang/Basic/TargetInfo.h"
18 #include "clang/Basic/TargetOptions.h"
19 #include "clang/Lex/HeaderSearch.h"
20 #include "clang/Lex/HeaderSearchOptions.h"
21 #include "clang/Lex/MacroArgs.h"
22 #include "clang/Lex/MacroInfo.h"
23 #include "clang/Lex/ModuleLoader.h"
24 #include "clang/Lex/Preprocessor.h"
25 #include "clang/Lex/PreprocessorOptions.h"
26 #include "gtest/gtest.h"
27
28 using namespace clang;
29
30 namespace {
31
32 // The test fixture.
33 class LexerTest : public ::testing::Test {
34 protected:
LexerTest()35 LexerTest()
36 : FileMgr(FileMgrOpts),
37 DiagID(new DiagnosticIDs()),
38 Diags(DiagID, new DiagnosticOptions, new IgnoringDiagConsumer()),
39 SourceMgr(Diags, FileMgr),
40 TargetOpts(new TargetOptions)
41 {
42 TargetOpts->Triple = "x86_64-apple-darwin11.1.0";
43 Target = TargetInfo::CreateTargetInfo(Diags, TargetOpts);
44 }
45
CreatePP(StringRef Source,TrivialModuleLoader & ModLoader)46 std::unique_ptr<Preprocessor> CreatePP(StringRef Source,
47 TrivialModuleLoader &ModLoader) {
48 std::unique_ptr<llvm::MemoryBuffer> Buf =
49 llvm::MemoryBuffer::getMemBuffer(Source);
50 SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(Buf)));
51
52 MemoryBufferCache PCMCache;
53 HeaderSearch HeaderInfo(std::make_shared<HeaderSearchOptions>(), SourceMgr,
54 Diags, LangOpts, Target.get());
55 std::unique_ptr<Preprocessor> PP = llvm::make_unique<Preprocessor>(
56 std::make_shared<PreprocessorOptions>(), Diags, LangOpts, SourceMgr,
57 PCMCache, HeaderInfo, ModLoader,
58 /*IILookup =*/nullptr,
59 /*OwnsHeaderSearch =*/false);
60 PP->Initialize(*Target);
61 PP->EnterMainSourceFile();
62 return PP;
63 }
64
Lex(StringRef Source)65 std::vector<Token> Lex(StringRef Source) {
66 TrivialModuleLoader ModLoader;
67 auto PP = CreatePP(Source, ModLoader);
68
69 std::vector<Token> toks;
70 while (1) {
71 Token tok;
72 PP->Lex(tok);
73 if (tok.is(tok::eof))
74 break;
75 toks.push_back(tok);
76 }
77
78 return toks;
79 }
80
CheckLex(StringRef Source,ArrayRef<tok::TokenKind> ExpectedTokens)81 std::vector<Token> CheckLex(StringRef Source,
82 ArrayRef<tok::TokenKind> ExpectedTokens) {
83 auto toks = Lex(Source);
84 EXPECT_EQ(ExpectedTokens.size(), toks.size());
85 for (unsigned i = 0, e = ExpectedTokens.size(); i != e; ++i) {
86 EXPECT_EQ(ExpectedTokens[i], toks[i].getKind());
87 }
88
89 return toks;
90 }
91
getSourceText(Token Begin,Token End)92 std::string getSourceText(Token Begin, Token End) {
93 bool Invalid;
94 StringRef Str =
95 Lexer::getSourceText(CharSourceRange::getTokenRange(SourceRange(
96 Begin.getLocation(), End.getLocation())),
97 SourceMgr, LangOpts, &Invalid);
98 if (Invalid)
99 return "<INVALID>";
100 return Str;
101 }
102
103 FileSystemOptions FileMgrOpts;
104 FileManager FileMgr;
105 IntrusiveRefCntPtr<DiagnosticIDs> DiagID;
106 DiagnosticsEngine Diags;
107 SourceManager SourceMgr;
108 LangOptions LangOpts;
109 std::shared_ptr<TargetOptions> TargetOpts;
110 IntrusiveRefCntPtr<TargetInfo> Target;
111 };
112
TEST_F(LexerTest,GetSourceTextExpandsToMaximumInMacroArgument)113 TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgument) {
114 std::vector<tok::TokenKind> ExpectedTokens;
115 ExpectedTokens.push_back(tok::identifier);
116 ExpectedTokens.push_back(tok::l_paren);
117 ExpectedTokens.push_back(tok::identifier);
118 ExpectedTokens.push_back(tok::r_paren);
119
120 std::vector<Token> toks = CheckLex("#define M(x) x\n"
121 "M(f(M(i)))",
122 ExpectedTokens);
123
124 EXPECT_EQ("M(i)", getSourceText(toks[2], toks[2]));
125 }
126
TEST_F(LexerTest,GetSourceTextExpandsToMaximumInMacroArgumentForEndOfMacro)127 TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgumentForEndOfMacro) {
128 std::vector<tok::TokenKind> ExpectedTokens;
129 ExpectedTokens.push_back(tok::identifier);
130 ExpectedTokens.push_back(tok::identifier);
131
132 std::vector<Token> toks = CheckLex("#define M(x) x\n"
133 "M(M(i) c)",
134 ExpectedTokens);
135
136 EXPECT_EQ("M(i)", getSourceText(toks[0], toks[0]));
137 }
138
TEST_F(LexerTest,GetSourceTextExpandsInMacroArgumentForBeginOfMacro)139 TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForBeginOfMacro) {
140 std::vector<tok::TokenKind> ExpectedTokens;
141 ExpectedTokens.push_back(tok::identifier);
142 ExpectedTokens.push_back(tok::identifier);
143 ExpectedTokens.push_back(tok::identifier);
144
145 std::vector<Token> toks = CheckLex("#define M(x) x\n"
146 "M(c c M(i))",
147 ExpectedTokens);
148
149 EXPECT_EQ("c M(i)", getSourceText(toks[1], toks[2]));
150 }
151
TEST_F(LexerTest,GetSourceTextExpandsInMacroArgumentForEndOfMacro)152 TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForEndOfMacro) {
153 std::vector<tok::TokenKind> ExpectedTokens;
154 ExpectedTokens.push_back(tok::identifier);
155 ExpectedTokens.push_back(tok::identifier);
156 ExpectedTokens.push_back(tok::identifier);
157
158 std::vector<Token> toks = CheckLex("#define M(x) x\n"
159 "M(M(i) c c)",
160 ExpectedTokens);
161
162 EXPECT_EQ("M(i) c", getSourceText(toks[0], toks[1]));
163 }
164
TEST_F(LexerTest,GetSourceTextInSeparateFnMacros)165 TEST_F(LexerTest, GetSourceTextInSeparateFnMacros) {
166 std::vector<tok::TokenKind> ExpectedTokens;
167 ExpectedTokens.push_back(tok::identifier);
168 ExpectedTokens.push_back(tok::identifier);
169 ExpectedTokens.push_back(tok::identifier);
170 ExpectedTokens.push_back(tok::identifier);
171
172 std::vector<Token> toks = CheckLex("#define M(x) x\n"
173 "M(c M(i)) M(M(i) c)",
174 ExpectedTokens);
175
176 EXPECT_EQ("<INVALID>", getSourceText(toks[1], toks[2]));
177 }
178
TEST_F(LexerTest,GetSourceTextWorksAcrossTokenPastes)179 TEST_F(LexerTest, GetSourceTextWorksAcrossTokenPastes) {
180 std::vector<tok::TokenKind> ExpectedTokens;
181 ExpectedTokens.push_back(tok::identifier);
182 ExpectedTokens.push_back(tok::l_paren);
183 ExpectedTokens.push_back(tok::identifier);
184 ExpectedTokens.push_back(tok::r_paren);
185
186 std::vector<Token> toks = CheckLex("#define M(x) x\n"
187 "#define C(x) M(x##c)\n"
188 "M(f(C(i)))",
189 ExpectedTokens);
190
191 EXPECT_EQ("C(i)", getSourceText(toks[2], toks[2]));
192 }
193
TEST_F(LexerTest,GetSourceTextExpandsAcrossMultipleMacroCalls)194 TEST_F(LexerTest, GetSourceTextExpandsAcrossMultipleMacroCalls) {
195 std::vector<tok::TokenKind> ExpectedTokens;
196 ExpectedTokens.push_back(tok::identifier);
197 ExpectedTokens.push_back(tok::l_paren);
198 ExpectedTokens.push_back(tok::identifier);
199 ExpectedTokens.push_back(tok::r_paren);
200
201 std::vector<Token> toks = CheckLex("#define M(x) x\n"
202 "f(M(M(i)))",
203 ExpectedTokens);
204 EXPECT_EQ("M(M(i))", getSourceText(toks[2], toks[2]));
205 }
206
TEST_F(LexerTest,GetSourceTextInMiddleOfMacroArgument)207 TEST_F(LexerTest, GetSourceTextInMiddleOfMacroArgument) {
208 std::vector<tok::TokenKind> ExpectedTokens;
209 ExpectedTokens.push_back(tok::identifier);
210 ExpectedTokens.push_back(tok::l_paren);
211 ExpectedTokens.push_back(tok::identifier);
212 ExpectedTokens.push_back(tok::r_paren);
213
214 std::vector<Token> toks = CheckLex("#define M(x) x\n"
215 "M(f(i))",
216 ExpectedTokens);
217 EXPECT_EQ("i", getSourceText(toks[2], toks[2]));
218 }
219
TEST_F(LexerTest,GetSourceTextExpandsAroundDifferentMacroCalls)220 TEST_F(LexerTest, GetSourceTextExpandsAroundDifferentMacroCalls) {
221 std::vector<tok::TokenKind> ExpectedTokens;
222 ExpectedTokens.push_back(tok::identifier);
223 ExpectedTokens.push_back(tok::l_paren);
224 ExpectedTokens.push_back(tok::identifier);
225 ExpectedTokens.push_back(tok::r_paren);
226
227 std::vector<Token> toks = CheckLex("#define M(x) x\n"
228 "#define C(x) x\n"
229 "f(C(M(i)))",
230 ExpectedTokens);
231 EXPECT_EQ("C(M(i))", getSourceText(toks[2], toks[2]));
232 }
233
TEST_F(LexerTest,GetSourceTextOnlyExpandsIfFirstTokenInMacro)234 TEST_F(LexerTest, GetSourceTextOnlyExpandsIfFirstTokenInMacro) {
235 std::vector<tok::TokenKind> ExpectedTokens;
236 ExpectedTokens.push_back(tok::identifier);
237 ExpectedTokens.push_back(tok::l_paren);
238 ExpectedTokens.push_back(tok::identifier);
239 ExpectedTokens.push_back(tok::identifier);
240 ExpectedTokens.push_back(tok::r_paren);
241
242 std::vector<Token> toks = CheckLex("#define M(x) x\n"
243 "#define C(x) c x\n"
244 "f(C(M(i)))",
245 ExpectedTokens);
246 EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3]));
247 }
248
TEST_F(LexerTest,GetSourceTextExpandsRecursively)249 TEST_F(LexerTest, GetSourceTextExpandsRecursively) {
250 std::vector<tok::TokenKind> ExpectedTokens;
251 ExpectedTokens.push_back(tok::identifier);
252 ExpectedTokens.push_back(tok::identifier);
253 ExpectedTokens.push_back(tok::l_paren);
254 ExpectedTokens.push_back(tok::identifier);
255 ExpectedTokens.push_back(tok::r_paren);
256
257 std::vector<Token> toks = CheckLex("#define M(x) x\n"
258 "#define C(x) c M(x)\n"
259 "C(f(M(i)))",
260 ExpectedTokens);
261 EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3]));
262 }
263
TEST_F(LexerTest,LexAPI)264 TEST_F(LexerTest, LexAPI) {
265 std::vector<tok::TokenKind> ExpectedTokens;
266 ExpectedTokens.push_back(tok::l_square);
267 ExpectedTokens.push_back(tok::identifier);
268 ExpectedTokens.push_back(tok::r_square);
269 ExpectedTokens.push_back(tok::l_square);
270 ExpectedTokens.push_back(tok::identifier);
271 ExpectedTokens.push_back(tok::r_square);
272 ExpectedTokens.push_back(tok::identifier);
273 ExpectedTokens.push_back(tok::identifier);
274 ExpectedTokens.push_back(tok::identifier);
275 ExpectedTokens.push_back(tok::identifier);
276
277 std::vector<Token> toks = CheckLex("#define M(x) [x]\n"
278 "#define N(x) x\n"
279 "#define INN(x) x\n"
280 "#define NOF1 INN(val)\n"
281 "#define NOF2 val\n"
282 "M(foo) N([bar])\n"
283 "N(INN(val)) N(NOF1) N(NOF2) N(val)",
284 ExpectedTokens);
285
286 SourceLocation lsqrLoc = toks[0].getLocation();
287 SourceLocation idLoc = toks[1].getLocation();
288 SourceLocation rsqrLoc = toks[2].getLocation();
289 CharSourceRange macroRange = SourceMgr.getExpansionRange(lsqrLoc);
290
291 SourceLocation Loc;
292 EXPECT_TRUE(Lexer::isAtStartOfMacroExpansion(lsqrLoc, SourceMgr, LangOpts, &Loc));
293 EXPECT_EQ(Loc, macroRange.getBegin());
294 EXPECT_FALSE(Lexer::isAtStartOfMacroExpansion(idLoc, SourceMgr, LangOpts));
295 EXPECT_FALSE(Lexer::isAtEndOfMacroExpansion(idLoc, SourceMgr, LangOpts));
296 EXPECT_TRUE(Lexer::isAtEndOfMacroExpansion(rsqrLoc, SourceMgr, LangOpts, &Loc));
297 EXPECT_EQ(Loc, macroRange.getEnd());
298 EXPECT_TRUE(macroRange.isTokenRange());
299
300 CharSourceRange range = Lexer::makeFileCharRange(
301 CharSourceRange::getTokenRange(lsqrLoc, idLoc), SourceMgr, LangOpts);
302 EXPECT_TRUE(range.isInvalid());
303 range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(idLoc, rsqrLoc),
304 SourceMgr, LangOpts);
305 EXPECT_TRUE(range.isInvalid());
306 range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc),
307 SourceMgr, LangOpts);
308 EXPECT_TRUE(!range.isTokenRange());
309 EXPECT_EQ(range.getAsRange(),
310 SourceRange(macroRange.getBegin(),
311 macroRange.getEnd().getLocWithOffset(1)));
312
313 StringRef text = Lexer::getSourceText(
314 CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc),
315 SourceMgr, LangOpts);
316 EXPECT_EQ(text, "M(foo)");
317
318 SourceLocation macroLsqrLoc = toks[3].getLocation();
319 SourceLocation macroIdLoc = toks[4].getLocation();
320 SourceLocation macroRsqrLoc = toks[5].getLocation();
321 SourceLocation fileLsqrLoc = SourceMgr.getSpellingLoc(macroLsqrLoc);
322 SourceLocation fileIdLoc = SourceMgr.getSpellingLoc(macroIdLoc);
323 SourceLocation fileRsqrLoc = SourceMgr.getSpellingLoc(macroRsqrLoc);
324
325 range = Lexer::makeFileCharRange(
326 CharSourceRange::getTokenRange(macroLsqrLoc, macroIdLoc),
327 SourceMgr, LangOpts);
328 EXPECT_EQ(SourceRange(fileLsqrLoc, fileIdLoc.getLocWithOffset(3)),
329 range.getAsRange());
330
331 range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(macroIdLoc, macroRsqrLoc),
332 SourceMgr, LangOpts);
333 EXPECT_EQ(SourceRange(fileIdLoc, fileRsqrLoc.getLocWithOffset(1)),
334 range.getAsRange());
335
336 macroRange = SourceMgr.getExpansionRange(macroLsqrLoc);
337 range = Lexer::makeFileCharRange(
338 CharSourceRange::getTokenRange(macroLsqrLoc, macroRsqrLoc),
339 SourceMgr, LangOpts);
340 EXPECT_EQ(SourceRange(macroRange.getBegin(), macroRange.getEnd().getLocWithOffset(1)),
341 range.getAsRange());
342
343 text = Lexer::getSourceText(
344 CharSourceRange::getTokenRange(SourceRange(macroLsqrLoc, macroIdLoc)),
345 SourceMgr, LangOpts);
346 EXPECT_EQ(text, "[bar");
347
348
349 SourceLocation idLoc1 = toks[6].getLocation();
350 SourceLocation idLoc2 = toks[7].getLocation();
351 SourceLocation idLoc3 = toks[8].getLocation();
352 SourceLocation idLoc4 = toks[9].getLocation();
353 EXPECT_EQ("INN", Lexer::getImmediateMacroName(idLoc1, SourceMgr, LangOpts));
354 EXPECT_EQ("INN", Lexer::getImmediateMacroName(idLoc2, SourceMgr, LangOpts));
355 EXPECT_EQ("NOF2", Lexer::getImmediateMacroName(idLoc3, SourceMgr, LangOpts));
356 EXPECT_EQ("N", Lexer::getImmediateMacroName(idLoc4, SourceMgr, LangOpts));
357 }
358
TEST_F(LexerTest,DontMergeMacroArgsFromDifferentMacroFiles)359 TEST_F(LexerTest, DontMergeMacroArgsFromDifferentMacroFiles) {
360 std::vector<Token> toks =
361 Lex("#define helper1 0\n"
362 "void helper2(const char *, ...);\n"
363 "#define M1(a, ...) helper2(a, ##__VA_ARGS__)\n"
364 "#define M2(a, ...) M1(a, helper1, ##__VA_ARGS__)\n"
365 "void f1() { M2(\"a\", \"b\"); }");
366
367 // Check the file corresponding to the "helper1" macro arg in M2.
368 //
369 // The lexer used to report its size as 31, meaning that the end of the
370 // expansion would be on the *next line* (just past `M2("a", "b")`). Make
371 // sure that we get the correct end location (the comma after "helper1").
372 SourceLocation helper1ArgLoc = toks[20].getLocation();
373 EXPECT_EQ(SourceMgr.getFileIDSize(SourceMgr.getFileID(helper1ArgLoc)), 8U);
374 }
375
TEST_F(LexerTest,DontOverallocateStringifyArgs)376 TEST_F(LexerTest, DontOverallocateStringifyArgs) {
377 TrivialModuleLoader ModLoader;
378 auto PP = CreatePP("\"StrArg\", 5, 'C'", ModLoader);
379
380 llvm::BumpPtrAllocator Allocator;
381 std::array<IdentifierInfo *, 3> ParamList;
382 MacroInfo *MI = PP->AllocateMacroInfo({});
383 MI->setIsFunctionLike();
384 MI->setParameterList(ParamList, Allocator);
385 EXPECT_EQ(3u, MI->getNumParams());
386 EXPECT_TRUE(MI->isFunctionLike());
387
388 Token Eof;
389 Eof.setKind(tok::eof);
390 std::vector<Token> ArgTokens;
391 while (1) {
392 Token tok;
393 PP->Lex(tok);
394 if (tok.is(tok::eof)) {
395 ArgTokens.push_back(Eof);
396 break;
397 }
398 if (tok.is(tok::comma))
399 ArgTokens.push_back(Eof);
400 else
401 ArgTokens.push_back(tok);
402 }
403
404 auto MacroArgsDeleter = [&PP](MacroArgs *M) { M->destroy(*PP); };
405 std::unique_ptr<MacroArgs, decltype(MacroArgsDeleter)> MA(
406 MacroArgs::create(MI, ArgTokens, false, *PP), MacroArgsDeleter);
407 Token Result = MA->getStringifiedArgument(0, *PP, {}, {});
408 EXPECT_EQ(tok::string_literal, Result.getKind());
409 EXPECT_STREQ("\"\\\"StrArg\\\"\"", Result.getLiteralData());
410 Result = MA->getStringifiedArgument(1, *PP, {}, {});
411 EXPECT_EQ(tok::string_literal, Result.getKind());
412 EXPECT_STREQ("\"5\"", Result.getLiteralData());
413 Result = MA->getStringifiedArgument(2, *PP, {}, {});
414 EXPECT_EQ(tok::string_literal, Result.getKind());
415 EXPECT_STREQ("\"'C'\"", Result.getLiteralData());
416 #if !defined(NDEBUG) && GTEST_HAS_DEATH_TEST
417 EXPECT_DEATH(MA->getStringifiedArgument(3, *PP, {}, {}),
418 "Invalid argument number!");
419 #endif
420 }
421
TEST_F(LexerTest,IsNewLineEscapedValid)422 TEST_F(LexerTest, IsNewLineEscapedValid) {
423 auto hasNewLineEscaped = [](const char *S) {
424 return Lexer::isNewLineEscaped(S, S + strlen(S) - 1);
425 };
426
427 EXPECT_TRUE(hasNewLineEscaped("\\\r"));
428 EXPECT_TRUE(hasNewLineEscaped("\\\n"));
429 EXPECT_TRUE(hasNewLineEscaped("\\\r\n"));
430 EXPECT_TRUE(hasNewLineEscaped("\\\n\r"));
431 EXPECT_TRUE(hasNewLineEscaped("\\ \t\v\f\r"));
432 EXPECT_TRUE(hasNewLineEscaped("\\ \t\v\f\r\n"));
433
434 EXPECT_FALSE(hasNewLineEscaped("\\\r\r"));
435 EXPECT_FALSE(hasNewLineEscaped("\\\r\r\n"));
436 EXPECT_FALSE(hasNewLineEscaped("\\\n\n"));
437 EXPECT_FALSE(hasNewLineEscaped("\r"));
438 EXPECT_FALSE(hasNewLineEscaped("\n"));
439 EXPECT_FALSE(hasNewLineEscaped("\r\n"));
440 EXPECT_FALSE(hasNewLineEscaped("\n\r"));
441 EXPECT_FALSE(hasNewLineEscaped("\r\r"));
442 EXPECT_FALSE(hasNewLineEscaped("\n\n"));
443 }
444
TEST_F(LexerTest,GetBeginningOfTokenWithEscapedNewLine)445 TEST_F(LexerTest, GetBeginningOfTokenWithEscapedNewLine) {
446 // Each line should have the same length for
447 // further offset calculation to be more straightforward.
448 const unsigned IdentifierLength = 8;
449 std::string TextToLex = "rabarbar\n"
450 "foo\\\nbar\n"
451 "foo\\\rbar\n"
452 "fo\\\r\nbar\n"
453 "foo\\\n\rba\n";
454 std::vector<tok::TokenKind> ExpectedTokens{5, tok::identifier};
455 std::vector<Token> LexedTokens = CheckLex(TextToLex, ExpectedTokens);
456
457 for (const Token &Tok : LexedTokens) {
458 std::pair<FileID, unsigned> OriginalLocation =
459 SourceMgr.getDecomposedLoc(Tok.getLocation());
460 for (unsigned Offset = 0; Offset < IdentifierLength; ++Offset) {
461 SourceLocation LookupLocation =
462 Tok.getLocation().getLocWithOffset(Offset);
463
464 std::pair<FileID, unsigned> FoundLocation =
465 SourceMgr.getDecomposedExpansionLoc(
466 Lexer::GetBeginningOfToken(LookupLocation, SourceMgr, LangOpts));
467
468 // Check that location returned by the GetBeginningOfToken
469 // is the same as original token location reported by Lexer.
470 EXPECT_EQ(FoundLocation.second, OriginalLocation.second);
471 }
472 }
473 }
474
TEST_F(LexerTest,AvoidPastEndOfStringDereference)475 TEST_F(LexerTest, AvoidPastEndOfStringDereference) {
476 EXPECT_TRUE(Lex(" // \\\n").empty());
477 EXPECT_TRUE(Lex("#include <\\\\").empty());
478 EXPECT_TRUE(Lex("#include <\\\\\n").empty());
479 }
480
TEST_F(LexerTest,StringizingRasString)481 TEST_F(LexerTest, StringizingRasString) {
482 // For "std::string Lexer::Stringify(StringRef Str, bool Charify)".
483 std::string String1 = R"(foo
484 {"bar":[]}
485 baz)";
486 // For "void Lexer::Stringify(SmallVectorImpl<char> &Str)".
487 SmallString<128> String2;
488 String2 += String1.c_str();
489
490 // Corner cases.
491 std::string String3 = R"(\
492 \n
493 \\n
494 \\)";
495 SmallString<128> String4;
496 String4 += String3.c_str();
497 std::string String5 = R"(a\
498
499
500 \\b)";
501 SmallString<128> String6;
502 String6 += String5.c_str();
503
504 String1 = Lexer::Stringify(StringRef(String1));
505 Lexer::Stringify(String2);
506 String3 = Lexer::Stringify(StringRef(String3));
507 Lexer::Stringify(String4);
508 String5 = Lexer::Stringify(StringRef(String5));
509 Lexer::Stringify(String6);
510
511 EXPECT_EQ(String1, R"(foo\n {\"bar\":[]}\n baz)");
512 EXPECT_EQ(String2, R"(foo\n {\"bar\":[]}\n baz)");
513 EXPECT_EQ(String3, R"(\\\n \\n\n \\\\n\n \\\\)");
514 EXPECT_EQ(String4, R"(\\\n \\n\n \\\\n\n \\\\)");
515 EXPECT_EQ(String5, R"(a\\\n\n\n \\\\b)");
516 EXPECT_EQ(String6, R"(a\\\n\n\n \\\\b)");
517 }
518
519 } // anonymous namespace
520