1 // Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
2 
3 using System.Web.Razor.Tokenizer;
4 using System.Web.Razor.Tokenizer.Symbols;
5 using Xunit;
6 using Assert = Microsoft.TestCommon.AssertEx;
7 
8 namespace System.Web.Razor.Test.Tokenizer
9 {
10     public class CSharpTokenizerTest : CSharpTokenizerTestBase
11     {
12         [Fact]
Constructor_Throws_ArgNull_If_Null_Source_Provided()13         public void Constructor_Throws_ArgNull_If_Null_Source_Provided()
14         {
15             Assert.ThrowsArgumentNull(() => new CSharpTokenizer(null), "source");
16         }
17 
18         [Fact]
Next_Returns_Null_When_EOF_Reached()19         public void Next_Returns_Null_When_EOF_Reached()
20         {
21             TestTokenizer("");
22         }
23 
24         [Fact]
Next_Returns_Newline_Token_For_Single_CR()25         public void Next_Returns_Newline_Token_For_Single_CR()
26         {
27             TestTokenizer("\r\ra",
28                           new CSharpSymbol(0, 0, 0, "\r", CSharpSymbolType.NewLine),
29                           new CSharpSymbol(1, 1, 0, "\r", CSharpSymbolType.NewLine),
30                           IgnoreRemaining);
31         }
32 
33         [Fact]
Next_Returns_Newline_Token_For_Single_LF()34         public void Next_Returns_Newline_Token_For_Single_LF()
35         {
36             TestTokenizer("\n\na",
37                           new CSharpSymbol(0, 0, 0, "\n", CSharpSymbolType.NewLine),
38                           new CSharpSymbol(1, 1, 0, "\n", CSharpSymbolType.NewLine),
39                           IgnoreRemaining);
40         }
41 
42         [Fact]
Next_Returns_Newline_Token_For_Single_NEL()43         public void Next_Returns_Newline_Token_For_Single_NEL()
44         {
45             // NEL: Unicode "Next Line" U+0085
46             TestTokenizer("\u0085\u0085a",
47                           new CSharpSymbol(0, 0, 0, "\u0085", CSharpSymbolType.NewLine),
48                           new CSharpSymbol(1, 1, 0, "\u0085", CSharpSymbolType.NewLine),
49                           IgnoreRemaining);
50         }
51 
52         [Fact]
Next_Returns_Newline_Token_For_Single_Line_Separator()53         public void Next_Returns_Newline_Token_For_Single_Line_Separator()
54         {
55             // Unicode "Line Separator" U+2028
56             TestTokenizer("\u2028\u2028a",
57                           new CSharpSymbol(0, 0, 0, "\u2028", CSharpSymbolType.NewLine),
58                           new CSharpSymbol(1, 1, 0, "\u2028", CSharpSymbolType.NewLine),
59                           IgnoreRemaining);
60         }
61 
62         [Fact]
Next_Returns_Newline_Token_For_Single_Paragraph_Separator()63         public void Next_Returns_Newline_Token_For_Single_Paragraph_Separator()
64         {
65             // Unicode "Paragraph Separator" U+2029
66             TestTokenizer("\u2029\u2029a",
67                           new CSharpSymbol(0, 0, 0, "\u2029", CSharpSymbolType.NewLine),
68                           new CSharpSymbol(1, 1, 0, "\u2029", CSharpSymbolType.NewLine),
69                           IgnoreRemaining);
70         }
71 
72         [Fact]
Next_Returns_Single_Newline_Token_For_CRLF()73         public void Next_Returns_Single_Newline_Token_For_CRLF()
74         {
75             TestTokenizer("\r\n\r\na",
76                           new CSharpSymbol(0, 0, 0, "\r\n", CSharpSymbolType.NewLine),
77                           new CSharpSymbol(2, 1, 0, "\r\n", CSharpSymbolType.NewLine),
78                           IgnoreRemaining);
79         }
80 
81         [Fact]
Next_Returns_Token_For_Whitespace_Characters()82         public void Next_Returns_Token_For_Whitespace_Characters()
83         {
84             TestTokenizer(" \f\t\u000B \n ",
85                           new CSharpSymbol(0, 0, 0, " \f\t\u000B ", CSharpSymbolType.WhiteSpace),
86                           new CSharpSymbol(5, 0, 5, "\n", CSharpSymbolType.NewLine),
87                           new CSharpSymbol(6, 1, 0, " ", CSharpSymbolType.WhiteSpace));
88         }
89 
90         [Fact]
Transition_Is_Recognized()91         public void Transition_Is_Recognized()
92         {
93             TestSingleToken("@", CSharpSymbolType.Transition);
94         }
95 
96         [Fact]
Transition_Is_Recognized_As_SingleCharacter()97         public void Transition_Is_Recognized_As_SingleCharacter()
98         {
99             TestTokenizer("@(",
100                           new CSharpSymbol(0, 0, 0, "@", CSharpSymbolType.Transition),
101                           new CSharpSymbol(1, 0, 1, "(", CSharpSymbolType.LeftParenthesis));
102         }
103     }
104 }
105