1// Copyright 2018 The CUE Authors 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package cmd 16 17import ( 18 "bytes" 19 "io" 20 "os" 21 "path/filepath" 22 "regexp" 23 "strings" 24 25 "github.com/spf13/pflag" 26 "golang.org/x/text/language" 27 "golang.org/x/text/message" 28 29 "cuelang.org/go/cue" 30 "cuelang.org/go/cue/ast" 31 "cuelang.org/go/cue/build" 32 "cuelang.org/go/cue/errors" 33 "cuelang.org/go/cue/load" 34 "cuelang.org/go/cue/parser" 35 "cuelang.org/go/cue/token" 36 "cuelang.org/go/internal/encoding" 37 "cuelang.org/go/internal/filetypes" 38 "cuelang.org/go/internal/value" 39) 40 41// Disallow 42// - block comments 43// - old-style field comprehensions 44// - space separator syntax 45const syntaxVersion = -1000 + 100*2 + 1 46 47var requestedVersion = os.Getenv("CUE_SYNTAX_OVERRIDE") 48 49var defaultConfig = config{ 50 loadCfg: &load.Config{ 51 ParseFile: func(name string, src interface{}) (*ast.File, error) { 52 version := syntaxVersion 53 if requestedVersion != "" { 54 switch { 55 case strings.HasPrefix(requestedVersion, "v0.1"): 56 version = -1000 + 100 57 } 58 } 59 return parser.ParseFile(name, src, 60 parser.FromVersion(version), 61 parser.ParseComments, 62 ) 63 }, 64 }, 65} 66 67var runtime = &cue.Runtime{} 68 69var inTest = false 70 71func exitIfErr(cmd *Command, inst *cue.Instance, err error, fatal bool) { 72 exitOnErr(cmd, err, fatal) 73} 74 75func getLang() language.Tag { 76 loc := os.Getenv("LC_ALL") 77 if loc == "" { 78 loc = os.Getenv("LANG") 79 } 80 loc = strings.Split(loc, ".")[0] 81 return language.Make(loc) 82} 83 84func exitOnErr(cmd *Command, err error, fatal bool) { 85 if err == nil { 86 return 87 } 88 89 // Link x/text as our localizer. 90 p := message.NewPrinter(getLang()) 91 format := func(w io.Writer, format string, args ...interface{}) { 92 p.Fprintf(w, format, args...) 93 } 94 95 cwd, _ := os.Getwd() 96 97 w := &bytes.Buffer{} 98 errors.Print(w, err, &errors.Config{ 99 Format: format, 100 Cwd: cwd, 101 ToSlash: inTest, 102 }) 103 104 b := w.Bytes() 105 _, _ = cmd.Stderr().Write(b) 106 if fatal { 107 exit() 108 } 109} 110 111func loadFromArgs(cmd *Command, args []string, cfg *load.Config) []*build.Instance { 112 binst := load.Instances(args, cfg) 113 if len(binst) == 0 { 114 return nil 115 } 116 117 return binst 118} 119 120// A buildPlan defines what should be done based on command line 121// arguments and flags. 122// 123// TODO: allow --merge/-m to mix in other packages. 124type buildPlan struct { 125 cmd *Command 126 insts []*build.Instance 127 128 // instance is a pre-compiled instance, which exists if value files are 129 // being processed, which may require a schema to decode. 130 instance *cue.Instance 131 132 cfg *config 133 134 // If orphanFiles are mixed with CUE files and/or if placement flags are used, 135 // the instance is also included in insts. 136 importing bool 137 mergeData bool // do not merge individual data files. 138 orphaned []*decoderInfo 139 orphanInstance *build.Instance 140 // imported files are files that were orphaned in the build instance, but 141 // were placed in the instance by using one the --files, --list or --path 142 // flags. 143 imported []*ast.File 144 145 expressions []ast.Expr // only evaluate these expressions within results 146 schema ast.Expr // selects schema in instance for orphaned values 147 148 // orphan placement flags. 149 perFile bool 150 useList bool 151 path []ast.Label 152 useContext bool 153 154 // outFile defines the file to output to. Default is CUE stdout. 155 outFile *build.File 156 157 encConfig *encoding.Config 158} 159 160// instances iterates either over a list of instances, or a list of 161// data files. In the latter case, there must be either 0 or 1 other 162// instance, with which the data instance may be merged. 163func (b *buildPlan) instances() iterator { 164 var i iterator 165 switch { 166 case len(b.orphaned) > 0: 167 i = newStreamingIterator(b) 168 case len(b.insts) > 0: 169 i = &instanceIterator{ 170 inst: b.instance, 171 a: buildInstances(b.cmd, b.insts), 172 i: -1, 173 } 174 default: 175 i = &instanceIterator{ 176 a: []*cue.Instance{b.instance}, 177 i: -1, 178 } 179 b.instance = nil 180 } 181 if len(b.expressions) > 0 { 182 return &expressionIter{ 183 iter: i, 184 expr: b.expressions, 185 i: len(b.expressions), 186 } 187 } 188 return i 189} 190 191type iterator interface { 192 scan() bool 193 value() cue.Value 194 instance() *cue.Instance // may return nil 195 file() *ast.File // may return nil 196 err() error 197 close() 198 id() string 199} 200 201type instanceIterator struct { 202 inst *cue.Instance 203 a []*cue.Instance 204 i int 205 e error 206} 207 208func (i *instanceIterator) scan() bool { 209 i.i++ 210 return i.i < len(i.a) && i.e == nil 211} 212 213func (i *instanceIterator) close() {} 214func (i *instanceIterator) err() error { return i.e } 215func (i *instanceIterator) value() cue.Value { 216 v := i.a[i.i].Value() 217 if i.inst != nil { 218 v = v.Unify(i.inst.Value()) 219 } 220 return v 221} 222func (i *instanceIterator) instance() *cue.Instance { 223 if i.i >= len(i.a) { 224 return nil 225 } 226 return i.a[i.i] 227} 228func (i *instanceIterator) file() *ast.File { return nil } 229func (i *instanceIterator) id() string { return i.a[i.i].Dir } 230 231type streamingIterator struct { 232 r *cue.Runtime 233 b *buildPlan 234 cfg *encoding.Config 235 a []*decoderInfo 236 dec *encoding.Decoder 237 v cue.Value 238 f *ast.File 239 e error 240} 241 242func newStreamingIterator(b *buildPlan) *streamingIterator { 243 i := &streamingIterator{ 244 cfg: b.encConfig, 245 a: b.orphaned, 246 b: b, 247 } 248 249 // TODO: use orphanedSchema 250 i.r = &cue.Runtime{} 251 if v := b.encConfig.Schema; v.Exists() { 252 i.r = value.ConvertToRuntime(v.Context()) 253 } 254 255 return i 256} 257 258func (i *streamingIterator) file() *ast.File { return i.f } 259func (i *streamingIterator) value() cue.Value { return i.v } 260func (i *streamingIterator) instance() *cue.Instance { return nil } 261 262func (i *streamingIterator) id() string { 263 return "" 264} 265 266func (i *streamingIterator) scan() bool { 267 if i.e != nil { 268 return false 269 } 270 271 // advance to next value 272 if i.dec != nil && !i.dec.Done() { 273 i.dec.Next() 274 } 275 276 // advance to next stream if necessary 277 for i.dec == nil || i.dec.Done() { 278 if i.dec != nil { 279 i.dec.Close() 280 i.dec = nil 281 } 282 if len(i.a) == 0 { 283 return false 284 } 285 286 i.dec = i.a[0].dec(i.b) 287 if i.e = i.dec.Err(); i.e != nil { 288 return false 289 } 290 i.a = i.a[1:] 291 } 292 293 // compose value 294 i.f = i.dec.File() 295 inst, err := i.r.CompileFile(i.f) 296 if err != nil { 297 i.e = err 298 return false 299 } 300 i.v = inst.Value() 301 if schema := i.b.encConfig.Schema; schema.Exists() { 302 i.e = schema.Err() 303 if i.e == nil { 304 i.v = i.v.Unify(schema) // TODO(required fields): don't merge in schema 305 i.e = i.v.Err() 306 } 307 i.f = nil 308 } 309 return i.e == nil 310} 311 312func (i *streamingIterator) close() { 313 if i.dec != nil { 314 i.dec.Close() 315 i.dec = nil 316 } 317} 318 319func (i *streamingIterator) err() error { 320 if i.dec != nil { 321 if err := i.dec.Err(); err != nil { 322 return err 323 } 324 } 325 return i.e 326} 327 328type expressionIter struct { 329 iter iterator 330 expr []ast.Expr 331 i int 332} 333 334func (i *expressionIter) err() error { return i.iter.err() } 335func (i *expressionIter) close() { i.iter.close() } 336func (i *expressionIter) id() string { return i.iter.id() } 337 338func (i *expressionIter) scan() bool { 339 i.i++ 340 if i.i < len(i.expr) { 341 return true 342 } 343 if !i.iter.scan() { 344 return false 345 } 346 i.i = 0 347 return true 348} 349 350func (i *expressionIter) file() *ast.File { return nil } 351func (i *expressionIter) instance() *cue.Instance { return nil } 352 353func (i *expressionIter) value() cue.Value { 354 if len(i.expr) == 0 { 355 return i.iter.value() 356 } 357 v := i.iter.value() 358 path := "" 359 if inst := i.iter.instance(); inst != nil { 360 path = inst.ID() 361 } 362 return v.Context().BuildExpr(i.expr[i.i], 363 cue.Scope(v), 364 cue.InferBuiltins(true), 365 cue.ImportPath(path), 366 ) 367} 368 369type config struct { 370 outMode filetypes.Mode 371 372 fileFilter string 373 reFile *regexp.Regexp 374 encoding build.Encoding 375 interpretation build.Interpretation 376 377 overrideDefault bool 378 379 noMerge bool // do not merge individual data files. 380 381 loadCfg *load.Config 382} 383 384func newBuildPlan(cmd *Command, args []string, cfg *config) (p *buildPlan, err error) { 385 if cfg == nil { 386 cfg = &defaultConfig 387 } 388 if cfg.loadCfg == nil { 389 cfg.loadCfg = defaultConfig.loadCfg 390 } 391 cfg.loadCfg.Stdin = cmd.InOrStdin() 392 393 p = &buildPlan{cfg: cfg, cmd: cmd, importing: cfg.loadCfg.DataFiles} 394 395 if err := p.parseFlags(); err != nil { 396 return nil, err 397 } 398 re, err := regexp.Compile(p.cfg.fileFilter) 399 if err != nil { 400 return nil, err 401 } 402 cfg.reFile = re 403 404 if err := setTags(cmd.Flags(), cfg.loadCfg); err != nil { 405 return nil, err 406 } 407 408 return p, nil 409} 410 411func (p *buildPlan) matchFile(file string) bool { 412 return p.cfg.reFile.MatchString(file) 413} 414 415func setTags(f *pflag.FlagSet, cfg *load.Config) error { 416 tags, _ := f.GetStringArray(string(flagInject)) 417 cfg.Tags = tags 418 if b, _ := f.GetBool(string(flagInjectVars)); b { 419 cfg.TagVars = load.DefaultTagVars() 420 } 421 return nil 422} 423 424type decoderInfo struct { 425 file *build.File 426 d *encoding.Decoder // may be nil if delayed 427} 428 429func (d *decoderInfo) dec(b *buildPlan) *encoding.Decoder { 430 if d.d == nil { 431 d.d = encoding.NewDecoder(d.file, b.encConfig) 432 } 433 return d.d 434} 435 436func (d *decoderInfo) close() { 437 if d.d != nil { 438 d.d.Close() 439 } 440} 441 442// getDecoders takes the orphaned files of the given instance and splits them in 443// schemas and values, saving the build.File and encoding.Decoder in the 444// returned slices. It is up to the caller to Close any of the decoders that are 445// returned. 446func (p *buildPlan) getDecoders(b *build.Instance) (schemas, values []*decoderInfo, err error) { 447 files := b.OrphanedFiles 448 if p.cfg.overrideDefault { 449 files = append(files, b.UnknownFiles...) 450 } 451 for _, f := range files { 452 if !b.User && !p.matchFile(f.Filename) { 453 continue 454 } 455 if p.cfg.overrideDefault { 456 f.Encoding = p.cfg.encoding 457 f.Interpretation = p.cfg.interpretation 458 } 459 switch f.Encoding { 460 case build.Protobuf, build.YAML, build.JSON, build.JSONL, 461 build.Text, build.Binary: 462 if f.Interpretation == build.ProtobufJSON { 463 // Need a schema. 464 values = append(values, &decoderInfo{f, nil}) 465 continue 466 } 467 case build.TextProto: 468 if p.importing { 469 return schemas, values, errors.Newf(token.NoPos, 470 "cannot import textproto files") 471 } 472 // Needs to be decoded after any schema. 473 values = append(values, &decoderInfo{f, nil}) 474 continue 475 default: 476 return schemas, values, errors.Newf(token.NoPos, 477 "unsupported encoding %q", f.Encoding) 478 } 479 480 // We add the module root to the path if there is a module defined. 481 c := *p.encConfig 482 if b.Module != "" { 483 c.ProtoPath = append(c.ProtoPath, b.Root) 484 } 485 d := encoding.NewDecoder(f, &c) 486 487 fi, err := filetypes.FromFile(f, p.cfg.outMode) 488 if err != nil { 489 return schemas, values, err 490 } 491 switch { 492 // case !fi.Schema: // TODO: value/schema/auto 493 // values = append(values, d) 494 case fi.Form != build.Schema && fi.Form != build.Final: 495 values = append(values, &decoderInfo{f, d}) 496 497 case f.Interpretation != build.Auto: 498 schemas = append(schemas, &decoderInfo{f, d}) 499 500 case d.Interpretation() == "": 501 values = append(values, &decoderInfo{f, d}) 502 503 default: 504 schemas = append(schemas, &decoderInfo{f, d}) 505 } 506 } 507 return schemas, values, nil 508} 509 510// importFiles imports orphan files for existing instances. Note that during 511// import, both schemas and non-schemas are placed (TODO: should we allow schema 512// mode here as well? It seems that the existing package should have enough 513// typing to allow for schemas). 514// 515// It is a separate call to allow closing decoders between processing each 516// package. 517func (p *buildPlan) importFiles(b *build.Instance) error { 518 // TODO: assume textproto is imported at top-level or just ignore them. 519 520 schemas, values, err := p.getDecoders(b) 521 for _, d := range append(schemas, values...) { 522 defer d.close() 523 } 524 if err != nil { 525 return err 526 } 527 return p.placeOrphans(b, append(schemas, values...)) 528} 529 530func parseArgs(cmd *Command, args []string, cfg *config) (p *buildPlan, err error) { 531 p, err = newBuildPlan(cmd, args, cfg) 532 if err != nil { 533 return nil, err 534 } 535 536 builds := loadFromArgs(cmd, args, cfg.loadCfg) 537 if builds == nil { 538 return nil, errors.Newf(token.NoPos, "invalid args") 539 } 540 541 if err := p.parsePlacementFlags(); err != nil { 542 return nil, err 543 } 544 545 for _, b := range builds { 546 if b.Err != nil { 547 return nil, b.Err 548 } 549 switch { 550 case !b.User: 551 if p.importing { 552 if err := p.importFiles(b); err != nil { 553 return nil, err 554 } 555 } 556 p.insts = append(p.insts, b) 557 558 case p.orphanInstance != nil: 559 return nil, errors.Newf(token.NoPos, 560 "builds contain two file packages") 561 562 default: 563 p.orphanInstance = b 564 } 565 } 566 567 if len(p.insts) == 0 && flagGlob.String(p.cmd) != "" { 568 return nil, errors.Newf(token.NoPos, 569 "use of -n/--name flag without a directory") 570 } 571 572 if b := p.orphanInstance; b != nil { 573 schemas, values, err := p.getDecoders(b) 574 for _, d := range append(schemas, values...) { 575 defer d.close() 576 } 577 if err != nil { 578 return nil, err 579 } 580 581 if values == nil { 582 values, schemas = schemas, values 583 } 584 585 for _, di := range schemas { 586 d := di.dec(p) 587 for ; !d.Done(); d.Next() { 588 if err := b.AddSyntax(d.File()); err != nil { 589 return nil, err 590 } 591 } 592 if err := d.Err(); err != nil { 593 return nil, err 594 } 595 } 596 597 if len(p.insts) > 1 && p.schema != nil { 598 return nil, errors.Newf(token.NoPos, 599 "cannot use --schema/-d with flag more than one schema") 600 } 601 602 var schema *build.Instance 603 switch n := len(p.insts); n { 604 default: 605 return nil, errors.Newf(token.NoPos, 606 "too many packages defined (%d) in combination with files", n) 607 case 1: 608 if len(schemas) > 0 { 609 return nil, errors.Newf(token.NoPos, 610 "cannot combine packages with individual schema files") 611 } 612 schema = p.insts[0] 613 p.insts = nil 614 615 case 0: 616 bb := *b 617 schema = &bb 618 b.BuildFiles = nil 619 b.Files = nil 620 } 621 622 if schema != nil && len(schema.Files) > 0 { 623 inst := buildInstances(p.cmd, []*build.Instance{schema})[0] 624 625 if inst.Err != nil { 626 return nil, err 627 } 628 p.instance = inst 629 p.encConfig.Schema = inst.Value() 630 if p.schema != nil { 631 v := inst.Eval(p.schema) 632 if err := v.Err(); err != nil { 633 return nil, err 634 } 635 p.encConfig.Schema = v 636 } 637 } else if p.schema != nil { 638 return nil, errors.Newf(token.NoPos, 639 "-d/--schema flag specified without a schema") 640 } 641 642 switch { 643 default: 644 fallthrough 645 646 case p.schema != nil: 647 p.orphaned = values 648 649 case p.mergeData, p.usePlacement(), p.importing: 650 if err = p.placeOrphans(b, values); err != nil { 651 return nil, err 652 } 653 654 } 655 656 if len(b.Files) > 0 { 657 p.insts = append(p.insts, b) 658 } 659 } 660 661 if len(p.expressions) > 1 { 662 p.encConfig.Stream = true 663 } 664 return p, nil 665} 666 667func (b *buildPlan) parseFlags() (err error) { 668 b.mergeData = !b.cfg.noMerge && flagMerge.Bool(b.cmd) 669 670 out := flagOut.String(b.cmd) 671 outFile := flagOutFile.String(b.cmd) 672 673 if strings.Contains(out, ":") && strings.Contains(outFile, ":") { 674 return errors.Newf(token.NoPos, 675 "cannot specify qualifier in both --out and --outfile") 676 } 677 if outFile == "" { 678 outFile = "-" 679 } 680 if out != "" { 681 outFile = out + ":" + outFile 682 } 683 b.outFile, err = filetypes.ParseFile(outFile, b.cfg.outMode) 684 if err != nil { 685 return err 686 } 687 688 for _, e := range flagExpression.StringArray(b.cmd) { 689 expr, err := parser.ParseExpr("--expression", e) 690 if err != nil { 691 return err 692 } 693 b.expressions = append(b.expressions, expr) 694 } 695 if s := flagSchema.String(b.cmd); s != "" { 696 b.schema, err = parser.ParseExpr("--schema", s) 697 if err != nil { 698 return err 699 } 700 } 701 if s := flagGlob.String(b.cmd); s != "" { 702 // Set a default file filter to only include json and yaml files 703 b.cfg.fileFilter = s 704 } 705 b.encConfig = &encoding.Config{ 706 Force: flagForce.Bool(b.cmd), 707 Mode: b.cfg.outMode, 708 Stdin: b.cmd.InOrStdin(), 709 Stdout: b.cmd.OutOrStdout(), 710 ProtoPath: flagProtoPath.StringArray(b.cmd), 711 AllErrors: flagAllErrors.Bool(b.cmd), 712 PkgName: flagPackage.String(b.cmd), 713 Strict: flagStrict.Bool(b.cmd), 714 } 715 return nil 716} 717 718func buildInstances(cmd *Command, binst []*build.Instance) []*cue.Instance { 719 // TODO: 720 // If there are no files and User is true, then use those? 721 // Always use all files in user mode? 722 instances := cue.Build(binst) 723 for _, inst := range instances { 724 // TODO: consider merging errors of multiple files, but ensure 725 // duplicates are removed. 726 exitIfErr(cmd, inst, inst.Err, true) 727 } 728 729 if flagIgnore.Bool(cmd) { 730 return instances 731 } 732 733 // TODO check errors after the fact in case of ignore. 734 for _, inst := range instances { 735 // TODO: consider merging errors of multiple files, but ensure 736 // duplicates are removed. 737 exitIfErr(cmd, inst, inst.Value().Validate(), !flagIgnore.Bool(cmd)) 738 } 739 return instances 740} 741 742func buildToolInstances(cmd *Command, binst []*build.Instance) ([]*cue.Instance, error) { 743 instances := cue.Build(binst) 744 for _, inst := range instances { 745 if inst.Err != nil { 746 return nil, inst.Err 747 } 748 } 749 750 // TODO check errors after the fact in case of ignore. 751 for _, inst := range instances { 752 if err := inst.Value().Validate(); err != nil { 753 return nil, err 754 } 755 } 756 return instances, nil 757} 758 759func buildTools(cmd *Command, args []string) (*cue.Instance, error) { 760 761 cfg := &load.Config{ 762 Tools: true, 763 } 764 f := cmd.cmd.Flags() 765 if err := setTags(f, cfg); err != nil { 766 return nil, err 767 } 768 769 binst := loadFromArgs(cmd, args, cfg) 770 if len(binst) == 0 { 771 return nil, nil 772 } 773 included := map[string]bool{} 774 775 ti := binst[0].Context().NewInstance(binst[0].Root, nil) 776 for _, inst := range binst { 777 k := 0 778 for _, f := range inst.Files { 779 if strings.HasSuffix(f.Filename, "_tool.cue") { 780 if !included[f.Filename] { 781 _ = ti.AddSyntax(f) 782 included[f.Filename] = true 783 } 784 continue 785 } 786 inst.Files[k] = f 787 k++ 788 } 789 inst.Files = inst.Files[:k] 790 } 791 792 insts, err := buildToolInstances(cmd, binst) 793 if err != nil { 794 return nil, err 795 } 796 797 inst := insts[0] 798 if len(insts) > 1 { 799 inst = cue.Merge(insts...) 800 } 801 802 r := value.ConvertToRuntime(inst.Value().Context()) 803 for _, b := range binst { 804 for _, i := range b.Imports { 805 if _, err := r.Build(i); err != nil { 806 return nil, err 807 } 808 } 809 } 810 811 // Set path equal to the package from which it is loading. 812 ti.ImportPath = binst[0].ImportPath 813 814 inst = inst.Build(ti) 815 return inst, inst.Err 816} 817 818func shortFile(root string, f *build.File) string { 819 dir, _ := filepath.Rel(root, f.Filename) 820 if dir == "" { 821 return f.Filename 822 } 823 if !filepath.IsAbs(dir) { 824 dir = "." + string(filepath.Separator) + dir 825 } 826 return dir 827} 828