1package parser_test 2 3import ( 4 "fmt" 5 "path/filepath" 6 "strings" 7 "testing" 8 9 "github.com/goccy/go-yaml/ast" 10 "github.com/goccy/go-yaml/lexer" 11 "github.com/goccy/go-yaml/parser" 12) 13 14func TestParser(t *testing.T) { 15 sources := []string{ 16 "null\n", 17 "{}\n", 18 "v: hi\n", 19 "v: \"true\"\n", 20 "v: \"false\"\n", 21 "v: true\n", 22 "v: false\n", 23 "v: 10\n", 24 "v: -10\n", 25 "v: 42\n", 26 "v: 4294967296\n", 27 "v: \"10\"\n", 28 "v: 0.1\n", 29 "v: 0.99\n", 30 "v: -0.1\n", 31 "v: .inf\n", 32 "v: -.inf\n", 33 "v: .nan\n", 34 "v: null\n", 35 "v: \"\"\n", 36 "v:\n- A\n- B\n", 37 "a: '-'\n", 38 "123\n", 39 "hello: world\n", 40 "a: null\n", 41 "v:\n- A\n- 1\n- B:\n - 2\n - 3\n", 42 "a:\n b: c\n", 43 "a: {x: 1}\n", 44 "t2: 2018-01-09T10:40:47Z\nt4: 2098-01-09T10:40:47Z\n", 45 "a: [1, 2]\n", 46 "a: {b: c, d: e}\n", 47 "a: 3s\n", 48 "a: <foo>\n", 49 "a: \"1:1\"\n", 50 "a: 1.2.3.4\n", 51 "a: \"2015-02-24T18:19:39Z\"\n", 52 "a: 'b: c'\n", 53 "a: 'Hello #comment'\n", 54 "a: abc <<def>> ghi", 55 "a: <<abcd", 56 "a: <<:abcd", 57 "a: << :abcd", 58 "a: 100.5\n", 59 "a: bogus\n", 60 "a: \"\\0\"\n", 61 "b: 2\na: 1\nd: 4\nc: 3\nsub:\n e: 5\n", 62 " a : b \n", 63 "a: b # comment\nb: c\n", 64 "---\na: b\n", 65 "a: b\n...\n", 66 "%YAML 1.2\n---\n", 67 "a: !!binary gIGC\n", 68 "a: !!binary |\n " + strings.Repeat("kJCQ", 17) + "kJ\n CQ\n", 69 "- !tag\n a: b\n c: d\n", 70 "v:\n- A\n- |-\n B\n C\n", 71 "v:\n- A\n- >-\n B\n C\n", 72 "v: |-\n 0\n", 73 "v: |-\n 0\nx: 0", 74 `"a\n1\nb"`, 75 `{"a":"b"}`, 76 `!!map { 77 ? !!str "explicit":!!str "entry", 78 ? !!str "implicit" : !!str "entry", 79 ? !!null "" : !!null "", 80}`, 81 } 82 for _, src := range sources { 83 if _, err := parser.Parse(lexer.Tokenize(src), 0); err != nil { 84 t.Fatalf("parse error: source [%s]: %+v", src, err) 85 } 86 } 87} 88 89func TestParseComplicatedDocument(t *testing.T) { 90 tests := []struct { 91 source string 92 expect string 93 }{ 94 { 95 ` 96american: 97 - Boston Red Sox 98 - Detroit Tigers 99 - New York Yankees 100national: 101 - New York Mets 102 - Chicago Cubs 103 - Atlanta Braves 104`, ` 105american: 106 - Boston Red Sox 107 - Detroit Tigers 108 - New York Yankees 109national: 110 - New York Mets 111 - Chicago Cubs 112 - Atlanta Braves 113`, 114 }, 115 { 116 ` 117a: 118 b: c 119 d: e 120 f: g 121h: 122 i: j 123 k: 124 l: m 125 n: o 126 p: q 127r: s 128`, ` 129a: 130 b: c 131 d: e 132 f: g 133h: 134 i: j 135 k: 136 l: m 137 n: o 138 p: q 139r: s 140`, 141 }, 142 { 143 ` 144- a: 145 - b 146 - c 147- d 148`, ` 149- a: 150 - b 151 - c 152- d 153`, 154 }, 155 { 156 ` 157- a 158- b 159- c 160 - d 161 - e 162- f 163`, ` 164- a 165- b 166- c - d - e 167- f 168`, 169 }, 170 { 171 ` 172a: 0 - 1 173`, 174 ` 175a: 0 - 1 176`, 177 }, 178 {` 179- a: 180 b: c 181 d: e 182- f: 183 g: h 184`, 185 ` 186- a: 187 b: c 188 d: e 189- f: null 190 g: h 191`, 192 }, 193 { 194 ` 195a: 196 b 197 c 198d: e 199`, ` 200a: b c 201d: e 202`, 203 }, 204 { 205 ` 206a 207b 208c 209`, ` 210a b c 211`, 212 }, 213 { 214 ` 215a: 216 - b 217 - c 218`, ` 219a: 220 - b 221 - c 222`, 223 }, 224 { 225 ` 226- a : 227 b: c 228`, ` 229- a: null 230 b: c 231`, 232 }, 233 { 234 ` 235- a: 236 b 237 c 238 d 239 hoge: fuga 240`, ` 241- a: b c d 242 hoge: fuga 243`, 244 }, 245 { 246 ` 247- a # ' " # - : % 248- b # " # - : % ' 249- c # # - : % ' " 250- d # - : % ' " # 251- e # : % ' " # - 252- f # % ' : # - : 253`, 254 ` 255- a 256- b 257- c 258- d 259- e 260- f 261`, 262 }, 263 { 264 ` 265# comment 266a: # comment 267# comment 268 b: c # comment 269 # comment 270d: e # comment 271# comment 272`, 273 ` 274a: 275 b: c 276d: e 277`, 278 }, 279 { 280 ` 281a: b#notcomment 282`, 283 ` 284a: b#notcomment 285`, 286 }, 287 { 288 ` 289anchored: &anchor foo 290aliased: *anchor 291`, 292 ` 293anchored: &anchor foo 294aliased: *anchor 295`, 296 }, 297 { 298 ` 299--- 300- &CENTER { x: 1, y: 2 } 301- &LEFT { x: 0, y: 2 } 302- &BIG { r: 10 } 303- &SMALL { r: 1 } 304 305# All the following maps are equal: 306 307- # Explicit keys 308 x: 1 309 y: 2 310 r: 10 311 label: center/big 312 313- # Merge one map 314 << : *CENTER 315 r: 10 316 label: center/big 317 318- # Merge multiple maps 319 << : [ *CENTER, *BIG ] 320 label: center/big 321 322- # Override 323 << : [ *BIG, *LEFT, *SMALL ] 324 x: 1 325 label: center/big 326`, 327 ` 328--- 329- &CENTER {x: 1, y: 2} 330- &LEFT {x: 0, y: 2} 331- &BIG {r: 10} 332- &SMALL {r: 1} 333- x: 1 334 y: 2 335 r: 10 336 label: center/big 337- <<: *CENTER 338 r: 10 339 label: center/big 340- <<: [*CENTER, *BIG] 341 label: center/big 342- <<: [*BIG, *LEFT, *SMALL] 343 x: 1 344 label: center/big 345`, 346 }, 347 { 348 ` 349a: 350- - b 351- - c 352 - d 353`, 354 ` 355a: 356- - b 357- - c 358 - d 359`, 360 }, 361 { 362 ` 363a: 364 b: 365 c: d 366 e: 367 f: g 368 h: i 369j: k 370`, 371 ` 372a: 373 b: 374 c: d 375 e: 376 f: g 377 h: i 378j: k 379`, 380 }, 381 { 382 ` 383--- 384a: 1 385b: 2 386... 387--- 388c: 3 389d: 4 390... 391`, 392 ` 393--- 394a: 1 395b: 2 396... 397--- 398c: 3 399d: 4 400... 401`, 402 }, 403 { 404 ` 405a: 406 b: | 407 { 408 [ 1, 2 ] 409 } 410 c: d 411`, 412 ` 413a: 414 b: | 415 { 416 [ 1, 2 ] 417 } 418 c: d 419`, 420 }, 421 { 422 ` 423| 424 hoge 425 fuga 426 piyo`, 427 ` 428| 429 hoge 430 fuga 431 piyo 432`, 433 }, 434 { 435 ` 436a: | 437 bbbbbbb 438 439 440 ccccccc 441d: eeeeeeeeeeeeeeeee 442`, 443 ` 444a: | 445 bbbbbbb 446 447 448 ccccccc 449d: eeeeeeeeeeeeeeeee 450`, 451 }, 452 { 453 ` 454a: b 455 c 456`, 457 ` 458a: b c 459`, 460 }, 461 { 462 ` 463a: 464 b: c 465`, 466 ` 467a: 468 b: c 469`, 470 }, 471 { 472 ` 473a: b 474c: d 475`, 476 ` 477a: b 478c: d 479`, 480 }, 481 { 482 ` 483- ab - cd 484- ef - gh 485`, 486 ` 487- ab - cd 488- ef - gh 489`, 490 }, 491 { 492 ` 493- 0 - 1 494 - 2 - 3 495`, 496 ` 497- 0 - 1 - 2 - 3 498`, 499 }, 500 { 501 ` 502a - b - c: value 503`, 504 ` 505a - b - c: value 506`, 507 }, 508 { 509 ` 510a: 511- 512 b: c 513 d: e 514- 515 f: g 516 h: i 517`, 518 ` 519a: 520- b: c 521 d: e 522- f: g 523 h: i 524`, 525 }, 526 { 527 ` 528a: |- 529 value 530b: c 531`, 532 ` 533a: |- 534 value 535b: c 536`, 537 }, 538 { 539 ` 540a: |+ 541 value 542b: c 543`, 544 ` 545a: |+ 546 value 547b: c 548`, 549 }, 550 } 551 552 for _, test := range tests { 553 tokens := lexer.Tokenize(test.source) 554 f, err := parser.Parse(tokens, 0) 555 if err != nil { 556 t.Fatalf("%+v", err) 557 } 558 var v Visitor 559 for _, doc := range f.Docs { 560 ast.Walk(&v, doc.Body) 561 } 562 expect := fmt.Sprintf("\n%+v\n", f) 563 if test.expect != expect { 564 tokens.Dump() 565 t.Fatalf("unexpected output: [%s] != [%s]", test.expect, expect) 566 } 567 } 568} 569 570func TestNewLineChar(t *testing.T) { 571 for _, f := range []string{ 572 "lf.yml", 573 "cr.yml", 574 "crlf.yml", 575 } { 576 ast, err := parser.ParseFile(filepath.Join("testdata", f), 0) 577 if err != nil { 578 t.Fatalf("%+v", err) 579 } 580 actual := fmt.Sprintf("%v\n", ast) 581 expect := `a: "a" 582b: 1 583` 584 if expect != actual { 585 t.Fatal("unexpected result") 586 } 587 } 588} 589 590func TestSyntaxError(t *testing.T) { 591 tests := []struct { 592 source string 593 expect string 594 }{ 595 { 596 ` 597a: 598- b 599 c: d 600 e: f 601 g: h`, 602 ` 603[3:3] unexpected key name 604 2 | a: 605> 3 | - b 606 4 | c: d 607 ^ 608 5 | e: f 609 6 | g: h`, 610 }, 611 { 612 ` 613a 614- b: c`, 615 ` 616[2:1] unexpected key name 617> 2 | a 618 3 | - b: c 619 ^ 620`, 621 }, 622 { 623 `%YAML 1.1 {}`, 624 ` 625[1:2] unexpected directive value. document not started 626> 1 | %YAML 1.1 {} 627 ^ 628`, 629 }, 630 { 631 `{invalid`, 632 ` 633[1:2] unexpected map 634> 1 | {invalid 635 ^ 636`, 637 }, 638 } 639 for _, test := range tests { 640 t.Run(test.source, func(t *testing.T) { 641 _, err := parser.ParseBytes([]byte(test.source), 0) 642 if err == nil { 643 t.Fatal("cannot catch syntax error") 644 } 645 actual := "\n" + err.Error() 646 if test.expect != actual { 647 t.Fatalf("expected: [%s] but got [%s]", test.expect, actual) 648 } 649 }) 650 } 651} 652 653func TestComment(t *testing.T) { 654 tests := []struct { 655 name string 656 yaml string 657 }{ 658 { 659 name: "map with comment", 660 yaml: ` 661# commentA 662a: #commentB 663 # commentC 664 b: c # commentD 665 # commentE 666 d: e # commentF 667 # commentG 668 f: g # commentH 669# commentI 670f: g # commentJ 671# commentK 672`, 673 }, 674 { 675 name: "sequence with comment", 676 yaml: ` 677# commentA 678- a # commentB 679# commentC 680- b: # commentD 681 # commentE 682 - d # commentF 683 - e # commentG 684# commentH 685`, 686 }, 687 { 688 name: "anchor and alias", 689 yaml: ` 690a: &x b # commentA 691c: *x # commentB 692`, 693 }, 694 } 695 for _, test := range tests { 696 t.Run(test.name, func(t *testing.T) { 697 f, err := parser.ParseBytes([]byte(test.yaml), parser.ParseComments) 698 if err != nil { 699 t.Fatalf("%+v", err) 700 } 701 var v Visitor 702 for _, doc := range f.Docs { 703 ast.Walk(&v, doc.Body) 704 } 705 }) 706 } 707} 708 709type Visitor struct { 710} 711 712func (v *Visitor) Visit(node ast.Node) ast.Visitor { 713 tk := node.GetToken() 714 tk.Prev = nil 715 tk.Next = nil 716 if comment := node.GetComment(); comment != nil { 717 comment.Prev = nil 718 comment.Next = nil 719 } 720 return v 721} 722