1#! /usr/bin/python 2# 3# Protocol Buffers - Google's data interchange format 4# Copyright 2008 Google Inc. All rights reserved. 5# http://code.google.com/p/protobuf/ 6# 7# Redistribution and use in source and binary forms, with or without 8# modification, are permitted provided that the following conditions are 9# met: 10# 11# * Redistributions of source code must retain the above copyright 12# notice, this list of conditions and the following disclaimer. 13# * Redistributions in binary form must reproduce the above 14# copyright notice, this list of conditions and the following disclaimer 15# in the documentation and/or other materials provided with the 16# distribution. 17# * Neither the name of Google Inc. nor the names of its 18# contributors may be used to endorse or promote products derived from 19# this software without specific prior written permission. 20# 21# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 33"""Test for google.protobuf.text_format.""" 34 35__author__ = 'kenton@google.com (Kenton Varda)' 36 37import difflib 38import re 39 40import unittest 41from google.protobuf import text_format 42from google.protobuf.internal import test_util 43from google.protobuf import unittest_pb2 44from google.protobuf import unittest_mset_pb2 45 46 47class TextFormatTest(unittest.TestCase): 48 def ReadGolden(self, golden_filename): 49 f = test_util.GoldenFile(golden_filename) 50 golden_lines = f.readlines() 51 f.close() 52 return golden_lines 53 54 def CompareToGoldenFile(self, text, golden_filename): 55 golden_lines = self.ReadGolden(golden_filename) 56 self.CompareToGoldenLines(text, golden_lines) 57 58 def CompareToGoldenText(self, text, golden_text): 59 self.CompareToGoldenLines(text, golden_text.splitlines(1)) 60 61 def CompareToGoldenLines(self, text, golden_lines): 62 actual_lines = text.splitlines(1) 63 self.assertEqual(golden_lines, actual_lines, 64 "Text doesn't match golden. Diff:\n" + 65 ''.join(difflib.ndiff(golden_lines, actual_lines))) 66 67 def testPrintAllFields(self): 68 message = unittest_pb2.TestAllTypes() 69 test_util.SetAllFields(message) 70 self.CompareToGoldenFile( 71 self.RemoveRedundantZeros(text_format.MessageToString(message)), 72 'text_format_unittest_data.txt') 73 74 def testPrintAllExtensions(self): 75 message = unittest_pb2.TestAllExtensions() 76 test_util.SetAllExtensions(message) 77 self.CompareToGoldenFile( 78 self.RemoveRedundantZeros(text_format.MessageToString(message)), 79 'text_format_unittest_extensions_data.txt') 80 81 def testPrintMessageSet(self): 82 message = unittest_mset_pb2.TestMessageSetContainer() 83 ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension 84 ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension 85 message.message_set.Extensions[ext1].i = 23 86 message.message_set.Extensions[ext2].str = 'foo' 87 self.CompareToGoldenText(text_format.MessageToString(message), 88 'message_set {\n' 89 ' [protobuf_unittest.TestMessageSetExtension1] {\n' 90 ' i: 23\n' 91 ' }\n' 92 ' [protobuf_unittest.TestMessageSetExtension2] {\n' 93 ' str: \"foo\"\n' 94 ' }\n' 95 '}\n') 96 97 def testPrintBadEnumValue(self): 98 message = unittest_pb2.TestAllTypes() 99 message.optional_nested_enum = 100 100 message.optional_foreign_enum = 101 101 message.optional_import_enum = 102 102 self.CompareToGoldenText( 103 text_format.MessageToString(message), 104 'optional_nested_enum: 100\n' 105 'optional_foreign_enum: 101\n' 106 'optional_import_enum: 102\n') 107 108 def testPrintBadEnumValueExtensions(self): 109 message = unittest_pb2.TestAllExtensions() 110 message.Extensions[unittest_pb2.optional_nested_enum_extension] = 100 111 message.Extensions[unittest_pb2.optional_foreign_enum_extension] = 101 112 message.Extensions[unittest_pb2.optional_import_enum_extension] = 102 113 self.CompareToGoldenText( 114 text_format.MessageToString(message), 115 '[protobuf_unittest.optional_nested_enum_extension]: 100\n' 116 '[protobuf_unittest.optional_foreign_enum_extension]: 101\n' 117 '[protobuf_unittest.optional_import_enum_extension]: 102\n') 118 119 def testPrintExotic(self): 120 message = unittest_pb2.TestAllTypes() 121 message.repeated_int64.append(-9223372036854775808) 122 message.repeated_uint64.append(18446744073709551615) 123 message.repeated_double.append(123.456) 124 message.repeated_double.append(1.23e22) 125 message.repeated_double.append(1.23e-18) 126 message.repeated_string.append('\000\001\a\b\f\n\r\t\v\\\'"') 127 message.repeated_string.append(u'\u00fc\ua71f') 128 self.CompareToGoldenText( 129 self.RemoveRedundantZeros(text_format.MessageToString(message)), 130 'repeated_int64: -9223372036854775808\n' 131 'repeated_uint64: 18446744073709551615\n' 132 'repeated_double: 123.456\n' 133 'repeated_double: 1.23e+22\n' 134 'repeated_double: 1.23e-18\n' 135 'repeated_string: ' 136 '"\\000\\001\\007\\010\\014\\n\\r\\t\\013\\\\\\\'\\""\n' 137 'repeated_string: "\\303\\274\\352\\234\\237"\n') 138 139 def testPrintNestedMessageAsOneLine(self): 140 message = unittest_pb2.TestAllTypes() 141 msg = message.repeated_nested_message.add() 142 msg.bb = 42; 143 self.CompareToGoldenText( 144 text_format.MessageToString(message, as_one_line=True), 145 'repeated_nested_message { bb: 42 }') 146 147 def testPrintRepeatedFieldsAsOneLine(self): 148 message = unittest_pb2.TestAllTypes() 149 message.repeated_int32.append(1) 150 message.repeated_int32.append(1) 151 message.repeated_int32.append(3) 152 message.repeated_string.append("Google") 153 message.repeated_string.append("Zurich") 154 self.CompareToGoldenText( 155 text_format.MessageToString(message, as_one_line=True), 156 'repeated_int32: 1 repeated_int32: 1 repeated_int32: 3 ' 157 'repeated_string: "Google" repeated_string: "Zurich"') 158 159 def testPrintNestedNewLineInStringAsOneLine(self): 160 message = unittest_pb2.TestAllTypes() 161 message.optional_string = "a\nnew\nline" 162 self.CompareToGoldenText( 163 text_format.MessageToString(message, as_one_line=True), 164 'optional_string: "a\\nnew\\nline"') 165 166 def testPrintMessageSetAsOneLine(self): 167 message = unittest_mset_pb2.TestMessageSetContainer() 168 ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension 169 ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension 170 message.message_set.Extensions[ext1].i = 23 171 message.message_set.Extensions[ext2].str = 'foo' 172 self.CompareToGoldenText( 173 text_format.MessageToString(message, as_one_line=True), 174 'message_set {' 175 ' [protobuf_unittest.TestMessageSetExtension1] {' 176 ' i: 23' 177 ' }' 178 ' [protobuf_unittest.TestMessageSetExtension2] {' 179 ' str: \"foo\"' 180 ' }' 181 ' }') 182 183 def testPrintExoticAsOneLine(self): 184 message = unittest_pb2.TestAllTypes() 185 message.repeated_int64.append(-9223372036854775808) 186 message.repeated_uint64.append(18446744073709551615) 187 message.repeated_double.append(123.456) 188 message.repeated_double.append(1.23e22) 189 message.repeated_double.append(1.23e-18) 190 message.repeated_string.append('\000\001\a\b\f\n\r\t\v\\\'"') 191 message.repeated_string.append(u'\u00fc\ua71f') 192 self.CompareToGoldenText( 193 self.RemoveRedundantZeros( 194 text_format.MessageToString(message, as_one_line=True)), 195 'repeated_int64: -9223372036854775808' 196 ' repeated_uint64: 18446744073709551615' 197 ' repeated_double: 123.456' 198 ' repeated_double: 1.23e+22' 199 ' repeated_double: 1.23e-18' 200 ' repeated_string: ' 201 '"\\000\\001\\007\\010\\014\\n\\r\\t\\013\\\\\\\'\\""' 202 ' repeated_string: "\\303\\274\\352\\234\\237"') 203 204 def testRoundTripExoticAsOneLine(self): 205 message = unittest_pb2.TestAllTypes() 206 message.repeated_int64.append(-9223372036854775808) 207 message.repeated_uint64.append(18446744073709551615) 208 message.repeated_double.append(123.456) 209 message.repeated_double.append(1.23e22) 210 message.repeated_double.append(1.23e-18) 211 message.repeated_string.append('\000\001\a\b\f\n\r\t\v\\\'"') 212 message.repeated_string.append(u'\u00fc\ua71f') 213 214 # Test as_utf8 = False. 215 wire_text = text_format.MessageToString( 216 message, as_one_line=True, as_utf8=False) 217 parsed_message = unittest_pb2.TestAllTypes() 218 text_format.Merge(wire_text, parsed_message) 219 self.assertEquals(message, parsed_message) 220 221 # Test as_utf8 = True. 222 wire_text = text_format.MessageToString( 223 message, as_one_line=True, as_utf8=True) 224 parsed_message = unittest_pb2.TestAllTypes() 225 text_format.Merge(wire_text, parsed_message) 226 self.assertEquals(message, parsed_message) 227 228 def testPrintRawUtf8String(self): 229 message = unittest_pb2.TestAllTypes() 230 message.repeated_string.append(u'\u00fc\ua71f') 231 text = text_format.MessageToString(message, as_utf8 = True) 232 self.CompareToGoldenText(text, 'repeated_string: "\303\274\352\234\237"\n') 233 parsed_message = unittest_pb2.TestAllTypes() 234 text_format.Merge(text, parsed_message) 235 self.assertEquals(message, parsed_message) 236 237 def testMessageToString(self): 238 message = unittest_pb2.ForeignMessage() 239 message.c = 123 240 self.assertEqual('c: 123\n', str(message)) 241 242 def RemoveRedundantZeros(self, text): 243 # Some platforms print 1e+5 as 1e+005. This is fine, but we need to remove 244 # these zeros in order to match the golden file. 245 text = text.replace('e+0','e+').replace('e+0','e+') \ 246 .replace('e-0','e-').replace('e-0','e-') 247 # Floating point fields are printed with .0 suffix even if they are 248 # actualy integer numbers. 249 text = re.compile('\.0$', re.MULTILINE).sub('', text) 250 return text 251 252 def testMergeGolden(self): 253 golden_text = '\n'.join(self.ReadGolden('text_format_unittest_data.txt')) 254 parsed_message = unittest_pb2.TestAllTypes() 255 text_format.Merge(golden_text, parsed_message) 256 257 message = unittest_pb2.TestAllTypes() 258 test_util.SetAllFields(message) 259 self.assertEquals(message, parsed_message) 260 261 def testMergeGoldenExtensions(self): 262 golden_text = '\n'.join(self.ReadGolden( 263 'text_format_unittest_extensions_data.txt')) 264 parsed_message = unittest_pb2.TestAllExtensions() 265 text_format.Merge(golden_text, parsed_message) 266 267 message = unittest_pb2.TestAllExtensions() 268 test_util.SetAllExtensions(message) 269 self.assertEquals(message, parsed_message) 270 271 def testMergeAllFields(self): 272 message = unittest_pb2.TestAllTypes() 273 test_util.SetAllFields(message) 274 ascii_text = text_format.MessageToString(message) 275 276 parsed_message = unittest_pb2.TestAllTypes() 277 text_format.Merge(ascii_text, parsed_message) 278 self.assertEqual(message, parsed_message) 279 test_util.ExpectAllFieldsSet(self, message) 280 281 def testMergeAllExtensions(self): 282 message = unittest_pb2.TestAllExtensions() 283 test_util.SetAllExtensions(message) 284 ascii_text = text_format.MessageToString(message) 285 286 parsed_message = unittest_pb2.TestAllExtensions() 287 text_format.Merge(ascii_text, parsed_message) 288 self.assertEqual(message, parsed_message) 289 290 def testMergeMessageSet(self): 291 message = unittest_pb2.TestAllTypes() 292 text = ('repeated_uint64: 1\n' 293 'repeated_uint64: 2\n') 294 text_format.Merge(text, message) 295 self.assertEqual(1, message.repeated_uint64[0]) 296 self.assertEqual(2, message.repeated_uint64[1]) 297 298 message = unittest_mset_pb2.TestMessageSetContainer() 299 text = ('message_set {\n' 300 ' [protobuf_unittest.TestMessageSetExtension1] {\n' 301 ' i: 23\n' 302 ' }\n' 303 ' [protobuf_unittest.TestMessageSetExtension2] {\n' 304 ' str: \"foo\"\n' 305 ' }\n' 306 '}\n') 307 text_format.Merge(text, message) 308 ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension 309 ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension 310 self.assertEquals(23, message.message_set.Extensions[ext1].i) 311 self.assertEquals('foo', message.message_set.Extensions[ext2].str) 312 313 def testMergeExotic(self): 314 message = unittest_pb2.TestAllTypes() 315 text = ('repeated_int64: -9223372036854775808\n' 316 'repeated_uint64: 18446744073709551615\n' 317 'repeated_double: 123.456\n' 318 'repeated_double: 1.23e+22\n' 319 'repeated_double: 1.23e-18\n' 320 'repeated_string: \n' 321 '"\\000\\001\\007\\010\\014\\n\\r\\t\\013\\\\\\\'\\""\n' 322 'repeated_string: "foo" \'corge\' "grault"\n' 323 'repeated_string: "\\303\\274\\352\\234\\237"\n' 324 'repeated_string: "\\xc3\\xbc"\n' 325 'repeated_string: "\xc3\xbc"\n') 326 text_format.Merge(text, message) 327 328 self.assertEqual(-9223372036854775808, message.repeated_int64[0]) 329 self.assertEqual(18446744073709551615, message.repeated_uint64[0]) 330 self.assertEqual(123.456, message.repeated_double[0]) 331 self.assertEqual(1.23e22, message.repeated_double[1]) 332 self.assertEqual(1.23e-18, message.repeated_double[2]) 333 self.assertEqual( 334 '\000\001\a\b\f\n\r\t\v\\\'"', message.repeated_string[0]) 335 self.assertEqual('foocorgegrault', message.repeated_string[1]) 336 self.assertEqual(u'\u00fc\ua71f', message.repeated_string[2]) 337 self.assertEqual(u'\u00fc', message.repeated_string[3]) 338 339 def testMergeEmptyText(self): 340 message = unittest_pb2.TestAllTypes() 341 text = '' 342 text_format.Merge(text, message) 343 self.assertEquals(unittest_pb2.TestAllTypes(), message) 344 345 def testMergeInvalidUtf8(self): 346 message = unittest_pb2.TestAllTypes() 347 text = 'repeated_string: "\\xc3\\xc3"' 348 self.assertRaises(text_format.ParseError, text_format.Merge, text, message) 349 350 def testMergeSingleWord(self): 351 message = unittest_pb2.TestAllTypes() 352 text = 'foo' 353 self.assertRaisesWithMessage( 354 text_format.ParseError, 355 ('1:1 : Message type "protobuf_unittest.TestAllTypes" has no field named ' 356 '"foo".'), 357 text_format.Merge, text, message) 358 359 def testMergeUnknownField(self): 360 message = unittest_pb2.TestAllTypes() 361 text = 'unknown_field: 8\n' 362 self.assertRaisesWithMessage( 363 text_format.ParseError, 364 ('1:1 : Message type "protobuf_unittest.TestAllTypes" has no field named ' 365 '"unknown_field".'), 366 text_format.Merge, text, message) 367 368 def testMergeBadExtension(self): 369 message = unittest_pb2.TestAllExtensions() 370 text = '[unknown_extension]: 8\n' 371 self.assertRaisesWithMessage( 372 text_format.ParseError, 373 '1:2 : Extension "unknown_extension" not registered.', 374 text_format.Merge, text, message) 375 message = unittest_pb2.TestAllTypes() 376 self.assertRaisesWithMessage( 377 text_format.ParseError, 378 ('1:2 : Message type "protobuf_unittest.TestAllTypes" does not have ' 379 'extensions.'), 380 text_format.Merge, text, message) 381 382 def testMergeGroupNotClosed(self): 383 message = unittest_pb2.TestAllTypes() 384 text = 'RepeatedGroup: <' 385 self.assertRaisesWithMessage( 386 text_format.ParseError, '1:16 : Expected ">".', 387 text_format.Merge, text, message) 388 389 text = 'RepeatedGroup: {' 390 self.assertRaisesWithMessage( 391 text_format.ParseError, '1:16 : Expected "}".', 392 text_format.Merge, text, message) 393 394 def testMergeEmptyGroup(self): 395 message = unittest_pb2.TestAllTypes() 396 text = 'OptionalGroup: {}' 397 text_format.Merge(text, message) 398 self.assertTrue(message.HasField('optionalgroup')) 399 400 message.Clear() 401 402 message = unittest_pb2.TestAllTypes() 403 text = 'OptionalGroup: <>' 404 text_format.Merge(text, message) 405 self.assertTrue(message.HasField('optionalgroup')) 406 407 def testMergeBadEnumValue(self): 408 message = unittest_pb2.TestAllTypes() 409 text = 'optional_nested_enum: BARR' 410 self.assertRaisesWithMessage( 411 text_format.ParseError, 412 ('1:23 : Enum type "protobuf_unittest.TestAllTypes.NestedEnum" ' 413 'has no value named BARR.'), 414 text_format.Merge, text, message) 415 416 message = unittest_pb2.TestAllTypes() 417 text = 'optional_nested_enum: 100' 418 self.assertRaisesWithMessage( 419 text_format.ParseError, 420 ('1:23 : Enum type "protobuf_unittest.TestAllTypes.NestedEnum" ' 421 'has no value with number 100.'), 422 text_format.Merge, text, message) 423 424 def testMergeBadIntValue(self): 425 message = unittest_pb2.TestAllTypes() 426 text = 'optional_int32: bork' 427 self.assertRaisesWithMessage( 428 text_format.ParseError, 429 ('1:17 : Couldn\'t parse integer: bork'), 430 text_format.Merge, text, message) 431 432 def testMergeStringFieldUnescape(self): 433 message = unittest_pb2.TestAllTypes() 434 text = r'''repeated_string: "\xf\x62" 435 repeated_string: "\\xf\\x62" 436 repeated_string: "\\\xf\\\x62" 437 repeated_string: "\\\\xf\\\\x62" 438 repeated_string: "\\\\\xf\\\\\x62" 439 repeated_string: "\x5cx20"''' 440 text_format.Merge(text, message) 441 442 SLASH = '\\' 443 self.assertEqual('\x0fb', message.repeated_string[0]) 444 self.assertEqual(SLASH + 'xf' + SLASH + 'x62', message.repeated_string[1]) 445 self.assertEqual(SLASH + '\x0f' + SLASH + 'b', message.repeated_string[2]) 446 self.assertEqual(SLASH + SLASH + 'xf' + SLASH + SLASH + 'x62', 447 message.repeated_string[3]) 448 self.assertEqual(SLASH + SLASH + '\x0f' + SLASH + SLASH + 'b', 449 message.repeated_string[4]) 450 self.assertEqual(SLASH + 'x20', message.repeated_string[5]) 451 452 def assertRaisesWithMessage(self, e_class, e, func, *args, **kwargs): 453 """Same as assertRaises, but also compares the exception message.""" 454 if hasattr(e_class, '__name__'): 455 exc_name = e_class.__name__ 456 else: 457 exc_name = str(e_class) 458 459 try: 460 func(*args, **kwargs) 461 except e_class as expr: 462 if str(expr) != e: 463 msg = '%s raised, but with wrong message: "%s" instead of "%s"' 464 raise self.failureException(msg % (exc_name, 465 str(expr).encode('string_escape'), 466 e.encode('string_escape'))) 467 return 468 else: 469 raise self.failureException('%s not raised' % exc_name) 470 471 472class TokenizerTest(unittest.TestCase): 473 474 def testSimpleTokenCases(self): 475 text = ('identifier1:"string1"\n \n\n' 476 'identifier2 : \n \n123 \n identifier3 :\'string\'\n' 477 'identifiER_4 : 1.1e+2 ID5:-0.23 ID6:\'aaaa\\\'bbbb\'\n' 478 'ID7 : "aa\\"bb"\n\n\n\n ID8: {A:inf B:-inf C:true D:false}\n' 479 'ID9: 22 ID10: -111111111111111111 ID11: -22\n' 480 'ID12: 2222222222222222222 ID13: 1.23456f ID14: 1.2e+2f ' 481 'false_bool: 0 true_BOOL:t \n true_bool1: 1 false_BOOL1:f ' ) 482 tokenizer = text_format._Tokenizer(text) 483 methods = [(tokenizer.ConsumeIdentifier, 'identifier1'), 484 ':', 485 (tokenizer.ConsumeString, 'string1'), 486 (tokenizer.ConsumeIdentifier, 'identifier2'), 487 ':', 488 (tokenizer.ConsumeInt32, 123), 489 (tokenizer.ConsumeIdentifier, 'identifier3'), 490 ':', 491 (tokenizer.ConsumeString, 'string'), 492 (tokenizer.ConsumeIdentifier, 'identifiER_4'), 493 ':', 494 (tokenizer.ConsumeFloat, 1.1e+2), 495 (tokenizer.ConsumeIdentifier, 'ID5'), 496 ':', 497 (tokenizer.ConsumeFloat, -0.23), 498 (tokenizer.ConsumeIdentifier, 'ID6'), 499 ':', 500 (tokenizer.ConsumeString, 'aaaa\'bbbb'), 501 (tokenizer.ConsumeIdentifier, 'ID7'), 502 ':', 503 (tokenizer.ConsumeString, 'aa\"bb'), 504 (tokenizer.ConsumeIdentifier, 'ID8'), 505 ':', 506 '{', 507 (tokenizer.ConsumeIdentifier, 'A'), 508 ':', 509 (tokenizer.ConsumeFloat, float('inf')), 510 (tokenizer.ConsumeIdentifier, 'B'), 511 ':', 512 (tokenizer.ConsumeFloat, -float('inf')), 513 (tokenizer.ConsumeIdentifier, 'C'), 514 ':', 515 (tokenizer.ConsumeBool, True), 516 (tokenizer.ConsumeIdentifier, 'D'), 517 ':', 518 (tokenizer.ConsumeBool, False), 519 '}', 520 (tokenizer.ConsumeIdentifier, 'ID9'), 521 ':', 522 (tokenizer.ConsumeUint32, 22), 523 (tokenizer.ConsumeIdentifier, 'ID10'), 524 ':', 525 (tokenizer.ConsumeInt64, -111111111111111111), 526 (tokenizer.ConsumeIdentifier, 'ID11'), 527 ':', 528 (tokenizer.ConsumeInt32, -22), 529 (tokenizer.ConsumeIdentifier, 'ID12'), 530 ':', 531 (tokenizer.ConsumeUint64, 2222222222222222222), 532 (tokenizer.ConsumeIdentifier, 'ID13'), 533 ':', 534 (tokenizer.ConsumeFloat, 1.23456), 535 (tokenizer.ConsumeIdentifier, 'ID14'), 536 ':', 537 (tokenizer.ConsumeFloat, 1.2e+2), 538 (tokenizer.ConsumeIdentifier, 'false_bool'), 539 ':', 540 (tokenizer.ConsumeBool, False), 541 (tokenizer.ConsumeIdentifier, 'true_BOOL'), 542 ':', 543 (tokenizer.ConsumeBool, True), 544 (tokenizer.ConsumeIdentifier, 'true_bool1'), 545 ':', 546 (tokenizer.ConsumeBool, True), 547 (tokenizer.ConsumeIdentifier, 'false_BOOL1'), 548 ':', 549 (tokenizer.ConsumeBool, False)] 550 551 i = 0 552 while not tokenizer.AtEnd(): 553 m = methods[i] 554 if type(m) == str: 555 token = tokenizer.token 556 self.assertEqual(token, m) 557 tokenizer.NextToken() 558 else: 559 self.assertEqual(m[1], m[0]()) 560 i += 1 561 562 def testConsumeIntegers(self): 563 # This test only tests the failures in the integer parsing methods as well 564 # as the '0' special cases. 565 int64_max = (1 << 63) - 1 566 uint32_max = (1 << 32) - 1 567 text = '-1 %d %d' % (uint32_max + 1, int64_max + 1) 568 tokenizer = text_format._Tokenizer(text) 569 self.assertRaises(text_format.ParseError, tokenizer.ConsumeUint32) 570 self.assertRaises(text_format.ParseError, tokenizer.ConsumeUint64) 571 self.assertEqual(-1, tokenizer.ConsumeInt32()) 572 573 self.assertRaises(text_format.ParseError, tokenizer.ConsumeUint32) 574 self.assertRaises(text_format.ParseError, tokenizer.ConsumeInt32) 575 self.assertEqual(uint32_max + 1, tokenizer.ConsumeInt64()) 576 577 self.assertRaises(text_format.ParseError, tokenizer.ConsumeInt64) 578 self.assertEqual(int64_max + 1, tokenizer.ConsumeUint64()) 579 self.assertTrue(tokenizer.AtEnd()) 580 581 text = '-0 -0 0 0' 582 tokenizer = text_format._Tokenizer(text) 583 self.assertEqual(0, tokenizer.ConsumeUint32()) 584 self.assertEqual(0, tokenizer.ConsumeUint64()) 585 self.assertEqual(0, tokenizer.ConsumeUint32()) 586 self.assertEqual(0, tokenizer.ConsumeUint64()) 587 self.assertTrue(tokenizer.AtEnd()) 588 589 def testConsumeByteString(self): 590 text = '"string1\'' 591 tokenizer = text_format._Tokenizer(text) 592 self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString) 593 594 text = 'string1"' 595 tokenizer = text_format._Tokenizer(text) 596 self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString) 597 598 text = '\n"\\xt"' 599 tokenizer = text_format._Tokenizer(text) 600 self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString) 601 602 text = '\n"\\"' 603 tokenizer = text_format._Tokenizer(text) 604 self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString) 605 606 text = '\n"\\x"' 607 tokenizer = text_format._Tokenizer(text) 608 self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString) 609 610 def testConsumeBool(self): 611 text = 'not-a-bool' 612 tokenizer = text_format._Tokenizer(text) 613 self.assertRaises(text_format.ParseError, tokenizer.ConsumeBool) 614 615 616if __name__ == '__main__': 617 unittest.main() 618