1// Copyright 2018 The CUE Authors 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package cmd 16 17import ( 18 "bytes" 19 "fmt" 20 "go/ast" 21 "go/token" 22 "go/types" 23 "io" 24 "io/ioutil" 25 "os" 26 "path" 27 "path/filepath" 28 "reflect" 29 "regexp" 30 "strconv" 31 "strings" 32 "unicode" 33 34 "github.com/spf13/cobra" 35 "golang.org/x/tools/go/packages" 36 37 cueast "cuelang.org/go/cue/ast" 38 "cuelang.org/go/cue/ast/astutil" 39 "cuelang.org/go/cue/format" 40 "cuelang.org/go/cue/literal" 41 "cuelang.org/go/cue/load" 42 "cuelang.org/go/cue/parser" 43 cuetoken "cuelang.org/go/cue/token" 44 "cuelang.org/go/internal" 45) 46 47// TODO: 48// Document: 49// - Use ast package. 50// - how to deal with "oneOf" or sum types? 51// - generate cue files for cue field tags? 52// - cue go get or cue get go 53// - include generation report in doc_gen.cue or report.txt. 54// Possible enums: 55// package foo 56// Type: enumType 57 58func newGoCmd(c *Command) *cobra.Command { 59 cmd := &cobra.Command{ 60 Use: "go [packages]", 61 Short: "add Go dependencies to the current module", 62 Long: `go converts Go types into CUE definitions 63 64The command "cue get go" is like "go get", but converts the retrieved Go 65packages to CUE. The retrieved packages are put in the CUE module's pkg 66directory at the import path of the corresponding Go package. The converted 67definitions are available to any CUE file within the CUE module by using 68this import path. 69 70The Go type definitions are converted to CUE based on how they would be 71interpreted by Go's encoding/json package. Definitions for a Go file foo.go 72are written to a CUE file named foo_go_gen.cue. 73 74It is safe for users to add additional files to the generated directories, 75as long as their name does not end with _gen.*. 76 77 78Rules of Converting Go types to CUE 79 80Go structs are converted to cue structs adhering to the following conventions: 81 82 - field names are translated based on the definition of a "json" or "yaml" 83 tag, in that order. 84 85 - embedded structs marked with a json inline tag unify with struct 86 definition. For instance, the Go struct 87 88 struct MyStruct { 89 Common ` + "json:\",inline\"" + ` 90 Field string 91 } 92 93 translates to the CUE struct 94 95 #MyStruct: Common & { 96 Field: string 97 } 98 99 - a type that implements MarshalJSON, UnmarshalJSON, MarshalYAML, or 100 UnmarshalYAML is translated to top (_) to indicate it may be any 101 value. For some Go core types for which the implementation of these 102 methods is known, like time.Time, the type may be more specific. 103 104 - a type implementing MarshalText or UnmarshalText is represented as 105 the CUE type string 106 107 - slices and arrays convert to CUE lists, except when the element type is 108 byte, in which case it translates to the CUE bytes type. 109 In the case of arrays, the length of the CUE value is constrained 110 accordingly, when possible. 111 112 - Maps translate to a CUE struct, where all elements are constrained to 113 be of Go map element type. Like for JSON, maps may only have string keys. 114 115 - Pointers translate to a sum type with the default value of null and 116 the Go type as an alternative value. 117 118 - Field tags are translated to CUE's field attributes. In some cases, 119 the contents are rewritten to reflect the corresponding types in CUE. 120 The @go attribute is added if the field name or type definition differs 121 between the generated CUE and the original Go. 122 123 124Native CUE Constraints 125 126Native CUE constraints may be defined in separate cue files alongside the 127generated files either in the original Go directory or in the generated 128directory. These files can impose additional constraints on types and values 129that are not otherwise expressible in Go. The package name for these CUE files 130must be the same as that of the Go package. 131 132For instance, for the type 133 134 package foo 135 136 type IP4String string 137 138defined in the Go package, one could add a cue file foo.cue with the following 139contents to allow IP4String to assume only valid IP4 addresses: 140 141 package foo 142 143 // IP4String defines a valid IP4 address. 144 #IP4String: =~#"^\#(byte)\.\#(byte)\.\#(byte)\.\#(byte)$"# 145 146 // byte defines string allowing integer values of 0-255. 147 byte = #"([01]?\d?\d|2[0-4]\d|25[0-5])"# 148 149 150The "cue get go" command copies any cue files in the original Go package 151directory that has a package clause with the same name as the Go package to the 152destination directory, replacing its .cue ending with _gen.cue. 153 154Alternatively, the additional native constraints can be added to the generated 155package, as long as the file name does not end with _gen.cue. 156Running cue get go again to regenerate the package will never overwrite any 157files not ending with _gen.*. 158 159 160Constants and Enums 161 162Go does not have an enum or sum type. Conventionally, a type that is supposed 163to be an enum is followed by a const block with the allowed values for that 164type. However, as that is only a guideline and not a hard rule, these cases 165cannot be translated to CUE disjunctions automatically. 166 167Constant values, however, are generated in a way that makes it easy to convert 168a type to a proper enum using native CUE constraints. For instance, the Go type 169 170 package foo 171 172 type Switch int 173 174 const ( 175 Off Switch = iota 176 On 177 ) 178 179translates into the following CUE definitions: 180 181 package foo 182 183 #Switch: int // enumSwitch 184 185 enumSwitch: Off | On 186 187 Off: 0 188 On: 1 189 190This definition allows any integer value for Switch, while the enumSwitch value 191defines all defined constants for Switch and thus all valid values if Switch 192were to be interpreted as an enum type. To turn Switch into an enum, 193include the following constraint in, say, enum.cue, in either the original 194source directory or the generated directory: 195 196 package foo 197 198 // limit the valid values for Switch to those existing as constants with 199 // the same type. 200 #Switch: enumSwitch 201 202This tells CUE that only the values enumerated by enumSwitch are valid 203values for Switch. Note that there are now two definitions of Switch. 204CUE handles this in the usual way by unifying the two definitions, in which case 205the more restrictive enum interpretation of Switch remains. 206`, 207 // - TODO: interpret cuego's struct tags and annotations. 208 209 RunE: mkRunE(c, extract), 210 } 211 212 cmd.Flags().StringP(string(flagExclude), "e", "", 213 "comma-separated list of regexps of entries") 214 215 cmd.Flags().Bool(string(flagLocal), false, 216 "generates files in the main module locally") 217 218 cmd.Flags().StringP(string(flagPackage), "p", "", "package name for generated CUE files") 219 220 return cmd 221} 222 223const ( 224 flagExclude flagName = "exclude" 225 flagLocal flagName = "local" 226) 227 228func (e *extractor) initExclusions(str string) { 229 e.exclude = str 230 for _, re := range strings.Split(str, ",") { 231 if re != "" { 232 e.exclusions = append(e.exclusions, regexp.MustCompile(re)) 233 } 234 } 235} 236 237func (e *extractor) filter(name string) bool { 238 for _, ex := range e.exclusions { 239 if ex.MatchString(name) { 240 return true 241 } 242 } 243 return false 244} 245 246type extractor struct { 247 cmd *Command 248 249 stderr io.Writer 250 pkgs []*packages.Package 251 done map[string]bool 252 253 // per package 254 orig map[types.Type]*ast.StructType 255 usedPkgs map[string]bool 256 257 // per file 258 cmap ast.CommentMap 259 pkg *packages.Package 260 consts map[string][]string 261 pkgNames map[string]pkgInfo 262 263 exclusions []*regexp.Regexp 264 exclude string 265} 266 267type pkgInfo struct { 268 id string 269 name string 270} 271 272func (e *extractor) logf(format string, args ...interface{}) { 273 if flagVerbose.Bool(e.cmd) { 274 fmt.Fprintf(e.stderr, format+"\n", args...) 275 } 276} 277 278func (e *extractor) usedPkg(pkg string) { 279 e.usedPkgs[pkg] = true 280} 281 282const cueGoMod = ` 283module cuelang.org/go 284 285go 1.14 286` 287 288//go:generate go run cuelang.org/go/internal/cmd/embedpkg cuelang.org/go/cmd/cue/cmd/interfaces 289 290func initInterfaces() (err error) { 291 // tempdir needed for overlay 292 tmpDir, err := ioutil.TempDir("", "cuelang") 293 if err != nil { 294 return err 295 } 296 297 defer func() { 298 rerr := os.RemoveAll(tmpDir) 299 if err == nil { 300 err = rerr 301 } 302 }() 303 304 // write the cuelang go.mod 305 err = ioutil.WriteFile(filepath.Join(tmpDir, "go.mod"), []byte(cueGoMod), 0666) 306 if err != nil { 307 return err 308 } 309 310 for fn, contents := range interfacesFiles { 311 fn = filepath.Join(tmpDir, filepath.FromSlash(fn)) 312 dir := filepath.Dir(fn) 313 if err := os.MkdirAll(dir, 0777); err != nil { 314 return err 315 } 316 317 if err = ioutil.WriteFile(fn, contents, 0666); err != nil { 318 return err 319 } 320 } 321 322 cfg := &packages.Config{ 323 Mode: packages.NeedName | packages.NeedFiles | packages.NeedCompiledGoFiles | 324 packages.NeedImports | packages.NeedTypes | packages.NeedTypesSizes | 325 packages.NeedSyntax | packages.NeedTypesInfo | packages.NeedDeps, 326 Dir: filepath.Join(tmpDir), 327 } 328 329 p, err := packages.Load(cfg, "cuelang.org/go/cmd/cue/cmd/interfaces") 330 if err != nil { 331 return fmt.Errorf("error loading embedded cuelang.org/go/cmd/cue/cmd/interfaces package: %w", err) 332 } 333 if len(p[0].Errors) > 0 { 334 var buf bytes.Buffer 335 for _, e := range p[0].Errors { 336 fmt.Fprintf(&buf, "\t%v\n", e) 337 } 338 return fmt.Errorf("error loading embedded cuelang.org/go/cmd/cue/cmd/interfaces package:\n%s", buf.String()) 339 } 340 341 for e, tt := range p[0].TypesInfo.Types { 342 if n, ok := tt.Type.(*types.Named); ok && n.String() == "error" { 343 continue 344 } 345 if tt.Type.Underlying().String() == "interface{}" { 346 continue 347 } 348 349 switch tt.Type.Underlying().(type) { 350 case *types.Interface: 351 file := p[0].Fset.Position(e.Pos()).Filename 352 switch filepath.Base(file) { 353 case "top.go": 354 toTop = append(toTop, tt.Type) 355 case "text.go": 356 toString = append(toString, tt.Type) 357 } 358 } 359 } 360 return nil 361} 362 363var ( 364 toTop []types.Type 365 toString []types.Type 366) 367 368// TODO: 369// - consider not including types with any dropped fields. 370 371func extract(cmd *Command, args []string) error { 372 // TODO the CUE load using "." (below) assumes that a CUE module and a Go 373 // module will exist within the same directory (more precisely a Go module 374 // could be nested within a CUE module), such that the module path in any 375 // subdirectory below the current directory will be the same. This seems an 376 // entirely reasonable restriction, but also one that we should enforce. 377 // 378 // Enforcing this restriction also makes --local entirely redundant. 379 380 // command specifies a Go package(s) that belong to the main module 381 // and where for some reason the 382 // determine module root: 383 binst := loadFromArgs(cmd, []string{"."}, nil)[0] 384 385 if err := initInterfaces(); err != nil { 386 return err 387 } 388 389 // TODO: require explicitly set root. 390 root := binst.Root 391 392 cfg := &packages.Config{ 393 Mode: packages.NeedName | packages.NeedFiles | packages.NeedCompiledGoFiles | 394 packages.NeedImports | packages.NeedTypes | packages.NeedTypesSizes | 395 packages.NeedSyntax | packages.NeedTypesInfo | packages.NeedDeps | 396 packages.NeedModule, 397 } 398 pkgs, err := packages.Load(cfg, args...) 399 if err != nil { 400 return err 401 } 402 var errs []string 403 for _, p := range pkgs { 404 for _, e := range p.Errors { 405 switch e.Kind { 406 case packages.ParseError, packages.TypeError: 407 default: 408 errs = append(errs, fmt.Sprintf("\t%s: %v", p.PkgPath, e)) 409 } 410 } 411 } 412 if len(errs) > 0 { 413 return fmt.Errorf("could not load Go packages:\n%s", strings.Join(errs, "\n")) 414 } 415 416 e := extractor{ 417 cmd: cmd, 418 stderr: cmd.Stderr(), 419 pkgs: pkgs, 420 orig: map[types.Type]*ast.StructType{}, 421 } 422 423 e.initExclusions(flagExclude.String(cmd)) 424 425 e.done = map[string]bool{} 426 427 for _, p := range pkgs { 428 e.done[p.PkgPath] = true 429 } 430 431 for _, p := range pkgs { 432 if err := e.extractPkg(root, p); err != nil { 433 return err 434 } 435 } 436 return nil 437} 438 439func (e *extractor) recordTypeInfo(p *packages.Package) { 440 for _, f := range p.Syntax { 441 ast.Inspect(f, func(n ast.Node) bool { 442 switch x := n.(type) { 443 case *ast.StructType: 444 e.orig[p.TypesInfo.TypeOf(x)] = x 445 } 446 return true 447 }) 448 } 449} 450 451func (e *extractor) extractPkg(root string, p *packages.Package) error { 452 e.pkg = p 453 e.logf("--- Package %s", p.PkgPath) 454 455 e.recordTypeInfo(p) 456 457 e.consts = map[string][]string{} 458 459 for _, f := range p.Syntax { 460 for _, d := range f.Decls { 461 switch x := d.(type) { 462 case *ast.GenDecl: 463 e.recordConsts(x) 464 } 465 } 466 } 467 468 pkg := p.PkgPath 469 dir := filepath.Join(load.GenPath(root), filepath.FromSlash(pkg)) 470 471 isMain := flagLocal.Bool(e.cmd) && p.Module != nil && p.Module.Main 472 if isMain { 473 dir = p.Module.Dir 474 sub := p.PkgPath[len(p.Module.Path):] 475 if sub != "" { 476 dir = filepath.FromSlash(dir + sub) 477 } 478 } 479 480 if err := os.MkdirAll(dir, 0777); err != nil { 481 return err 482 } 483 484 e.usedPkgs = map[string]bool{} 485 486 args := pkg 487 if e.exclude != "" { 488 args += " --exclude=" + e.exclude 489 } 490 491 for i, f := range p.Syntax { 492 e.cmap = ast.NewCommentMap(p.Fset, f, f.Comments) 493 494 e.pkgNames = map[string]pkgInfo{} 495 496 for _, spec := range f.Imports { 497 pkgPath, _ := strconv.Unquote(spec.Path.Value) 498 pkg := p.Imports[pkgPath] 499 500 info := pkgInfo{id: pkgPath, name: pkg.Name} 501 if path.Base(pkgPath) != pkg.Name { 502 info.id += ":" + pkg.Name 503 } 504 505 if spec.Name != nil { 506 info.name = spec.Name.Name 507 } 508 509 e.pkgNames[pkgPath] = info 510 } 511 512 decls := []cueast.Decl{} 513 for _, d := range f.Decls { 514 switch x := d.(type) { 515 case *ast.GenDecl: 516 decls = append(decls, e.reportDecl(x)...) 517 } 518 } 519 520 if len(decls) == 0 && f.Doc == nil { 521 continue 522 } 523 524 pName := flagPackage.String(e.cmd) 525 if pName == "" { 526 pName = p.Name 527 } 528 529 pkg := &cueast.Package{Name: e.ident(pName, false)} 530 addDoc(f.Doc, pkg) 531 532 f := &cueast.File{Decls: []cueast.Decl{ 533 internal.NewComment(false, "Code generated by cue get go. DO NOT EDIT."), 534 &cueast.CommentGroup{List: []*cueast.Comment{ 535 {Text: "//cue:generate cue get go " + args}, 536 }}, 537 pkg, 538 }} 539 f.Decls = append(f.Decls, decls...) 540 541 if err := astutil.Sanitize(f); err != nil { 542 return err 543 } 544 545 file := filepath.Base(p.CompiledGoFiles[i]) 546 547 file = strings.Replace(file, ".go", "_go", 1) 548 file += "_gen.cue" 549 b, err := format.Node(f, format.Simplify()) 550 if err != nil { 551 return err 552 } 553 err = ioutil.WriteFile(filepath.Join(dir, file), b, 0666) 554 if err != nil { 555 return err 556 } 557 } 558 559 if !isMain { 560 if err := e.importCUEFiles(p, dir, args); err != nil { 561 return err 562 } 563 } 564 565 for path := range e.usedPkgs { 566 if !e.done[path] { 567 e.done[path] = true 568 p := p.Imports[path] 569 if err := e.extractPkg(root, p); err != nil { 570 return err 571 } 572 } 573 } 574 575 return nil 576} 577 578func (e *extractor) importCUEFiles(p *packages.Package, dir, args string) error { 579 for _, o := range p.CompiledGoFiles { 580 root := filepath.Dir(o) 581 err := filepath.Walk(root, func(path string, fi os.FileInfo, err error) error { 582 if fi.IsDir() && path != root { 583 return filepath.SkipDir 584 } 585 if filepath.Ext(path) != ".cue" { 586 return nil 587 } 588 f, err := parser.ParseFile(path, nil) 589 if err != nil { 590 return err 591 } 592 593 if _, pkg, _ := internal.PackageInfo(f); pkg != "" && pkg == p.Name { 594 file := filepath.Base(path) 595 file = file[:len(file)-len(".cue")] 596 file += "_gen.cue" 597 598 w := &bytes.Buffer{} 599 fmt.Fprintln(w, "// Code generated by cue get go. DO NOT EDIT.") 600 fmt.Fprintln(w) 601 fmt.Fprintln(w, "//cue:generate cue get go", args) 602 fmt.Fprintln(w) 603 604 b, err := ioutil.ReadFile(path) 605 if err != nil { 606 return err 607 } 608 w.Write(b) 609 610 dst := filepath.Join(dir, file) 611 if err := ioutil.WriteFile(dst, w.Bytes(), 0666); err != nil { 612 return err 613 } 614 } 615 return nil 616 }) 617 if err != nil { 618 return err 619 } 620 } 621 return nil 622} 623 624func (e *extractor) recordConsts(x *ast.GenDecl) { 625 if x.Tok != token.CONST { 626 return 627 } 628 for _, s := range x.Specs { 629 v, ok := s.(*ast.ValueSpec) 630 if !ok { 631 continue 632 } 633 for _, n := range v.Names { 634 typ := e.pkg.TypesInfo.TypeOf(n).String() 635 e.consts[typ] = append(e.consts[typ], n.Name) 636 } 637 } 638} 639 640func (e *extractor) strLabel(name string) cueast.Label { 641 return cueast.NewString(name) 642} 643 644func (e *extractor) ident(name string, isDef bool) *cueast.Ident { 645 if isDef { 646 r := []rune(name)[0] 647 name = "#" + name 648 if !unicode.Is(unicode.Lu, r) { 649 name = "_" + name 650 } 651 } 652 return cueast.NewIdent(name) 653} 654 655func (e *extractor) def(doc *ast.CommentGroup, name string, value cueast.Expr, newline bool) *cueast.Field { 656 f := &cueast.Field{ 657 Label: e.ident(name, true), // Go identifiers are always valid CUE identifiers. 658 Value: value, 659 } 660 addDoc(doc, f) 661 if newline { 662 cueast.SetRelPos(f, cuetoken.NewSection) 663 } 664 return f 665} 666 667func (e *extractor) reportDecl(x *ast.GenDecl) (a []cueast.Decl) { 668 switch x.Tok { 669 case token.TYPE: 670 for _, s := range x.Specs { 671 v, ok := s.(*ast.TypeSpec) 672 if !ok || e.filter(v.Name.Name) { 673 continue 674 } 675 676 typ := e.pkg.TypesInfo.TypeOf(v.Name) 677 enums := e.consts[typ.String()] 678 name := v.Name.Name 679 mapNamed := false 680 underlying := e.pkg.TypesInfo.TypeOf(v.Type) 681 if b, ok := underlying.Underlying().(*types.Basic); ok && b.Kind() != types.String { 682 mapNamed = true 683 } 684 685 switch tn, ok := e.pkg.TypesInfo.Defs[v.Name].(*types.TypeName); { 686 case ok: 687 if altType := e.altType(tn.Type()); altType != nil { 688 // TODO: add the underlying tag as a Go tag once we have 689 // proper string escaping for CUE. 690 a = append(a, e.def(x.Doc, name, altType, true)) 691 break 692 } 693 fallthrough 694 695 default: 696 if !supportedType(nil, typ) { 697 e.logf(" Dropped declaration %v of unsupported type %v", name, typ) 698 continue 699 } 700 if s := e.altType(types.NewPointer(typ)); s != nil { 701 a = append(a, e.def(x.Doc, name, s, true)) 702 break 703 } 704 705 f, _ := e.makeField(name, cuetoken.ISA, underlying, x.Doc, true) 706 a = append(a, f) 707 cueast.SetRelPos(f, cuetoken.NewSection) 708 709 } 710 711 if len(enums) > 0 && ast.IsExported(name) { 712 enumName := "#enum" + name 713 cueast.AddComment(a[len(a)-1], internal.NewComment(false, enumName)) 714 715 // Constants are mapped as definitions. 716 var exprs []cueast.Expr 717 var named []cueast.Decl 718 for _, v := range enums { 719 label := cueast.NewString(v) 720 cueast.SetRelPos(label, cuetoken.Blank) 721 722 var x cueast.Expr = e.ident(v, true) 723 cueast.SetRelPos(x, cuetoken.Newline) 724 exprs = append(exprs, x) 725 726 if !mapNamed { 727 continue 728 } 729 730 named = append(named, &cueast.Field{ 731 Label: label, 732 Value: e.ident(v, true), 733 }) 734 } 735 736 addField := func(label string, exprs []cueast.Expr) { 737 f := &cueast.Field{ 738 Label: cueast.NewIdent(label), 739 Value: cueast.NewBinExpr(cuetoken.OR, exprs...), 740 } 741 cueast.SetRelPos(f, cuetoken.NewSection) 742 a = append(a, f) 743 } 744 745 addField(enumName, exprs) 746 if len(named) > 0 { 747 f := &cueast.Field{ 748 Label: cueast.NewIdent("#values_" + name), 749 Value: &cueast.StructLit{Elts: named}, 750 } 751 cueast.SetRelPos(f, cuetoken.NewSection) 752 a = append(a, f) 753 } 754 } 755 } 756 757 case token.CONST: 758 // TODO: copy over comments for constant blocks. 759 760 for k, s := range x.Specs { 761 // TODO: determine type name and filter. 762 v, ok := s.(*ast.ValueSpec) 763 if !ok { 764 continue 765 } 766 767 for i, name := range v.Names { 768 if name.Name == "_" { 769 continue 770 } 771 f := e.def(v.Doc, name.Name, nil, k == 0) 772 a = append(a, f) 773 774 val := "" 775 if i < len(v.Values) { 776 if lit, ok := v.Values[i].(*ast.BasicLit); ok { 777 val = lit.Value 778 } 779 } 780 781 c := e.pkg.TypesInfo.Defs[v.Names[i]].(*types.Const) 782 sv := c.Val().ExactString() 783 cv, err := parser.ParseExpr("", sv) 784 if err != nil { 785 panic(fmt.Errorf("failed to parse %v: %v", sv, err)) 786 } 787 788 // Use orignal Go value if compatible with CUE (octal is okay) 789 if b, ok := cv.(*cueast.BasicLit); ok { 790 if b.Kind == cuetoken.INT && val != "" && val[0] != '\'' { 791 b.Value = val 792 } 793 if b.Value != val { 794 cv.AddComment(internal.NewComment(false, val)) 795 } 796 } 797 798 typ := e.pkg.TypesInfo.TypeOf(name) 799 if s := typ.String(); !strings.Contains(s, "untyped") { 800 switch s { 801 case "byte", "string", "error": 802 default: 803 cv = cueast.NewBinExpr(cuetoken.AND, e.makeType(typ), cv) 804 } 805 } 806 807 f.Value = cv 808 } 809 } 810 } 811 return a 812} 813 814func shortTypeName(t types.Type) string { 815 if n, ok := t.(*types.Named); ok { 816 return n.Obj().Name() 817 } 818 return t.String() 819} 820 821func (e *extractor) altType(typ types.Type) cueast.Expr { 822 ptr := types.NewPointer(typ) 823 for _, x := range toTop { 824 i := x.Underlying().(*types.Interface) 825 if types.Implements(typ, i) || types.Implements(ptr, i) { 826 t := shortTypeName(typ) 827 e.logf(" %v implements %s; setting type to _", t, x) 828 return e.ident("_", false) 829 } 830 } 831 for _, x := range toString { 832 i := x.Underlying().(*types.Interface) 833 if types.Implements(typ, i) || types.Implements(ptr, i) { 834 t := shortTypeName(typ) 835 e.logf(" %v implements %s; setting type to string", t, x) 836 return e.ident("string", false) 837 } 838 } 839 return nil 840} 841 842func addDoc(g *ast.CommentGroup, x cueast.Node) bool { 843 doc := makeDoc(g, true) 844 if doc != nil { 845 x.AddComment(doc) 846 return true 847 } 848 return false 849} 850 851func makeDoc(g *ast.CommentGroup, isDoc bool) *cueast.CommentGroup { 852 if g == nil { 853 return nil 854 } 855 856 a := []*cueast.Comment{} 857 858 for _, comment := range g.List { 859 c := comment.Text 860 861 // Remove comment markers. 862 // The parser has given us exactly the comment text. 863 switch c[1] { 864 case '/': 865 //-style comment (no newline at the end) 866 a = append(a, &cueast.Comment{Text: c}) 867 868 case '*': 869 /*-style comment */ 870 c = c[2 : len(c)-2] 871 if len(c) > 0 && c[0] == '\n' { 872 c = c[1:] 873 } 874 875 lines := strings.Split(c, "\n") 876 877 // Find common space prefix 878 i := 0 879 line := lines[0] 880 for ; i < len(line); i++ { 881 if c := line[i]; c != ' ' && c != '\t' { 882 break 883 } 884 } 885 886 for _, l := range lines { 887 for j := 0; j < i && j < len(l); j++ { 888 if line[j] != l[j] { 889 i = j 890 break 891 } 892 } 893 } 894 895 // Strip last line if empty. 896 if n := len(lines); n > 1 && len(lines[n-1]) < i { 897 lines = lines[:n-1] 898 } 899 900 // Print lines. 901 for _, l := range lines { 902 if i >= len(l) { 903 a = append(a, &cueast.Comment{Text: "//"}) 904 continue 905 } 906 a = append(a, &cueast.Comment{Text: "// " + l[i:]}) 907 } 908 } 909 } 910 return &cueast.CommentGroup{Doc: isDoc, List: a} 911} 912 913func supportedType(stack []types.Type, t types.Type) (ok bool) { 914 // handle recursive types 915 for _, t0 := range stack { 916 if t0 == t { 917 return true 918 } 919 } 920 stack = append(stack, t) 921 922 if named, ok := t.(*types.Named); ok { 923 obj := named.Obj() 924 925 // Redirect or drop Go standard library types. 926 if obj.Pkg() == nil { 927 // error interface 928 return true 929 } 930 switch obj.Pkg().Path() { 931 case "time": 932 switch named.Obj().Name() { 933 case "Time", "Duration", "Location", "Month", "Weekday": 934 return true 935 } 936 return false 937 case "math/big": 938 switch named.Obj().Name() { 939 case "Int", "Float": 940 return true 941 } 942 // case "net": 943 // // TODO: IP, Host, SRV, etc. 944 // case "url": 945 // // TODO: URL and Values 946 } 947 } 948 949 t = t.Underlying() 950 switch x := t.(type) { 951 case *types.Basic: 952 return x.String() != "invalid type" 953 case *types.Named: 954 return true 955 case *types.Pointer: 956 return supportedType(stack, x.Elem()) 957 case *types.Slice: 958 return supportedType(stack, x.Elem()) 959 case *types.Array: 960 return supportedType(stack, x.Elem()) 961 case *types.Map: 962 if b, ok := x.Key().Underlying().(*types.Basic); !ok || b.Kind() != types.String { 963 return false 964 } 965 return supportedType(stack, x.Elem()) 966 case *types.Struct: 967 // Eliminate structs with fields for which all fields are filtered. 968 if x.NumFields() == 0 { 969 return true 970 } 971 for i := 0; i < x.NumFields(); i++ { 972 f := x.Field(i) 973 if f.Exported() && supportedType(stack, f.Type()) { 974 return true 975 } 976 } 977 case *types.Interface: 978 return true 979 } 980 return false 981} 982 983func (e *extractor) makeField(name string, kind cuetoken.Token, expr types.Type, doc *ast.CommentGroup, newline bool) (f *cueast.Field, typename string) { 984 typ := e.makeType(expr) 985 var label cueast.Label 986 if kind == cuetoken.ISA { 987 label = e.ident(name, true) 988 } else { 989 label = e.strLabel(name) 990 } 991 f = &cueast.Field{Label: label, Value: typ} 992 if doc := makeDoc(doc, newline); doc != nil { 993 f.AddComment(doc) 994 cueast.SetRelPos(doc, cuetoken.NewSection) 995 } 996 997 if kind == cuetoken.OPTION { 998 f.Token = cuetoken.COLON 999 f.Optional = cuetoken.Blank.Pos() 1000 } 1001 b, _ := format.Node(typ) 1002 return f, string(b) 1003} 1004 1005func (e *extractor) makeType(expr types.Type) (result cueast.Expr) { 1006 if x, ok := expr.(*types.Named); ok { 1007 obj := x.Obj() 1008 if obj.Pkg() == nil { 1009 return e.ident("_", false) 1010 } 1011 // Check for builtin packages. 1012 // TODO: replace these literal types with a reference to the fixed 1013 // builtin type. 1014 switch obj.Type().String() { 1015 case "time.Time": 1016 ref := e.ident(e.pkgNames[obj.Pkg().Path()].name, false) 1017 var name *cueast.Ident 1018 if ref.Name != "time" { 1019 name = e.ident(ref.Name, false) 1020 } 1021 ref.Node = cueast.NewImport(name, "time") 1022 return cueast.NewSel(ref, obj.Name()) 1023 1024 case "math/big.Int": 1025 return e.ident("int", false) 1026 1027 default: 1028 if !strings.ContainsAny(obj.Pkg().Path(), ".") { 1029 // Drop any standard library type if they haven't been handled 1030 // above. 1031 // TODO: Doc? 1032 if s := e.altType(obj.Type()); s != nil { 1033 return s 1034 } 1035 } 1036 } 1037 1038 result = e.ident(obj.Name(), true) 1039 if pkg := obj.Pkg(); pkg != nil && pkg != e.pkg.Types { 1040 info := e.pkgNames[pkg.Path()] 1041 if info.name == "" { 1042 info.name = pkg.Name() 1043 } 1044 p := e.ident(info.name, false) 1045 var name *cueast.Ident 1046 if info.name != pkg.Name() { 1047 name = e.ident(info.name, false) 1048 } 1049 if info.id == "" { 1050 // This may happen if an alias is defined in a different file 1051 // within this package referring to yet another package. 1052 info.id = pkg.Path() 1053 } 1054 p.Node = cueast.NewImport(name, info.id) 1055 // makeType is always called to describe a type, so whatever 1056 // this is referring to, it must be a definition. 1057 result = cueast.NewSel(p, "#"+obj.Name()) 1058 e.usedPkg(pkg.Path()) 1059 } 1060 return 1061 } 1062 1063 switch x := expr.(type) { 1064 case *types.Pointer: 1065 return &cueast.BinaryExpr{ 1066 X: cueast.NewNull(), 1067 Op: cuetoken.OR, 1068 Y: e.makeType(x.Elem()), 1069 } 1070 1071 case *types.Struct: 1072 st := &cueast.StructLit{ 1073 Lbrace: cuetoken.Blank.Pos(), 1074 Rbrace: cuetoken.Newline.Pos(), 1075 } 1076 e.addFields(x, st) 1077 return st 1078 1079 case *types.Slice: 1080 // TODO: should this be x.Elem().Underlying().String()? One could 1081 // argue either way. 1082 if x.Elem().String() == "byte" { 1083 return e.ident("bytes", false) 1084 } 1085 return cueast.NewList(&cueast.Ellipsis{Type: e.makeType(x.Elem())}) 1086 1087 case *types.Array: 1088 if x.Elem().String() == "byte" { 1089 // TODO: no way to constraint lengths of bytes for now, as regexps 1090 // operate on Unicode, not bytes. So we need 1091 // fmt.Fprint(e.w, fmt.Sprintf("=~ '^\C{%d}$'", x.Len())), 1092 // but regexp does not support that. 1093 // But translate to bytes, instead of [...byte] to be consistent. 1094 return e.ident("bytes", false) 1095 } else { 1096 return &cueast.BinaryExpr{ 1097 X: &cueast.BasicLit{ 1098 Kind: cuetoken.INT, 1099 Value: strconv.Itoa(int(x.Len())), 1100 }, 1101 Op: cuetoken.MUL, 1102 Y: cueast.NewList(e.makeType(x.Elem())), 1103 } 1104 } 1105 1106 case *types.Map: 1107 if b, ok := x.Key().Underlying().(*types.Basic); !ok || b.Kind() != types.String { 1108 panic(fmt.Sprintf("unsupported map key type %T", x.Key())) 1109 } 1110 1111 f := &cueast.Field{ 1112 Label: cueast.NewList(e.ident("string", false)), 1113 Value: e.makeType(x.Elem()), 1114 } 1115 cueast.SetRelPos(f, cuetoken.Blank) 1116 return &cueast.StructLit{ 1117 Lbrace: cuetoken.Blank.Pos(), 1118 Elts: []cueast.Decl{f}, 1119 Rbrace: cuetoken.Blank.Pos(), 1120 } 1121 1122 case *types.Basic: 1123 return e.ident(x.String(), false) 1124 1125 case *types.Interface: 1126 return e.ident("_", false) 1127 1128 default: 1129 // record error 1130 panic(fmt.Sprintf("unsupported type %T", x)) 1131 } 1132} 1133 1134func (e *extractor) addAttr(f *cueast.Field, tag, body string) { 1135 s := fmt.Sprintf("@%s(%s)", tag, body) 1136 f.Attrs = append(f.Attrs, &cueast.Attribute{Text: s}) 1137} 1138 1139func (e *extractor) addFields(x *types.Struct, st *cueast.StructLit) { 1140 add := func(x cueast.Decl) { 1141 st.Elts = append(st.Elts, x) 1142 } 1143 1144 s := e.orig[x] 1145 docs := []*ast.CommentGroup{} 1146 for _, f := range s.Fields.List { 1147 if len(f.Names) == 0 { 1148 docs = append(docs, f.Doc) 1149 } else { 1150 for range f.Names { 1151 docs = append(docs, f.Doc) 1152 } 1153 } 1154 } 1155 count := 0 1156 for i := 0; i < x.NumFields(); i++ { 1157 f := x.Field(i) 1158 if !ast.IsExported(f.Name()) { 1159 continue 1160 } 1161 if !supportedType(nil, f.Type()) { 1162 e.logf(" Dropped field %v for unsupported type %v", f.Name(), f.Type()) 1163 continue 1164 } 1165 if f.Anonymous() && e.isInline(x.Tag(i)) { 1166 typ := f.Type() 1167 for { 1168 p, ok := typ.(*types.Pointer) 1169 if !ok { 1170 break 1171 } 1172 typ = p.Elem() 1173 } 1174 if _, ok := typ.(*types.Named); ok { 1175 embed := &cueast.EmbedDecl{Expr: e.makeType(typ)} 1176 if i > 0 { 1177 cueast.SetRelPos(embed, cuetoken.NewSection) 1178 } 1179 add(embed) 1180 } else { 1181 switch x := typ.(type) { 1182 case *types.Struct: 1183 e.addFields(x, st) 1184 default: 1185 panic(fmt.Sprintf("unimplemented embedding for type %T", x)) 1186 } 1187 } 1188 continue 1189 } 1190 tag := x.Tag(i) 1191 name := getName(f.Name(), tag) 1192 if name == "-" { 1193 continue 1194 } 1195 // TODO: check referrers 1196 kind := cuetoken.COLON 1197 if e.isOptional(tag) { 1198 kind = cuetoken.OPTION 1199 } 1200 if _, ok := f.Type().(*types.Pointer); ok { 1201 kind = cuetoken.OPTION 1202 } 1203 field, cueType := e.makeField(name, kind, f.Type(), docs[i], count > 0) 1204 add(field) 1205 1206 if s := reflect.StructTag(tag).Get("cue"); s != "" { 1207 expr, err := parser.ParseExpr("get go", s) 1208 if err != nil { 1209 e.logf("error parsing struct tag %q:", s, err) 1210 } 1211 field.Value = cueast.NewBinExpr(cuetoken.AND, field.Value, expr) 1212 } 1213 1214 // Add field tag to convert back to Go. 1215 typeName := f.Type().String() 1216 // simplify type names: 1217 for path, info := range e.pkgNames { 1218 typeName = strings.Replace(typeName, path+".", info.name+".", -1) 1219 } 1220 typeName = strings.Replace(typeName, e.pkg.Types.Path()+".", "", -1) 1221 1222 cueStr := strings.Replace(cueType, "_#", "", -1) 1223 cueStr = strings.Replace(cueStr, "#", "", -1) 1224 1225 // TODO: remove fields in @go attr that are the same as printed? 1226 if name != f.Name() || typeName != cueStr { 1227 buf := &strings.Builder{} 1228 if name != f.Name() { 1229 buf.WriteString(f.Name()) 1230 } 1231 1232 if typeName != cueStr { 1233 if strings.ContainsAny(typeName, `#"',()=`) { 1234 typeName = literal.String.Quote(typeName) 1235 } 1236 fmt.Fprint(buf, ",", typeName) 1237 } 1238 e.addAttr(field, "go", buf.String()) 1239 } 1240 1241 // Carry over protobuf field tags with modifications. 1242 // TODO: consider trashing the protobuf tag, as the Go versions are 1243 // lossy and will not allow for an accurate translation in some cases. 1244 tags := reflect.StructTag(tag) 1245 if t := tags.Get("protobuf"); t != "" { 1246 split := strings.Split(t, ",") 1247 k := 0 1248 for _, s := range split { 1249 if strings.HasPrefix(s, "name=") && s[len("name="):] == name { 1250 continue 1251 } 1252 split[k] = s 1253 k++ 1254 } 1255 split = split[:k] 1256 1257 // Put tag first, as type could potentially be elided and is 1258 // "more optional". 1259 if len(split) >= 2 { 1260 split[0], split[1] = split[1], split[0] 1261 } 1262 1263 // Interpret as map? 1264 if len(split) > 2 && split[1] == "bytes" { 1265 tk := tags.Get("protobuf_key") 1266 tv := tags.Get("protobuf_val") 1267 if tk != "" && tv != "" { 1268 tk = strings.SplitN(tk, ",", 2)[0] 1269 tv = strings.SplitN(tv, ",", 2)[0] 1270 split[1] = fmt.Sprintf("map[%s]%s", tk, tv) 1271 } 1272 } 1273 1274 e.addAttr(field, "protobuf", strings.Join(split, ",")) 1275 } 1276 1277 // Carry over XML tags. 1278 if t := reflect.StructTag(tag).Get("xml"); t != "" { 1279 e.addAttr(field, "xml", t) 1280 } 1281 1282 // Carry over TOML tags. 1283 if t := reflect.StructTag(tag).Get("toml"); t != "" { 1284 e.addAttr(field, "toml", t) 1285 } 1286 1287 // TODO: should we in general carry over any unknown tag verbatim? 1288 1289 count++ 1290 } 1291} 1292 1293func (e *extractor) isInline(tag string) bool { 1294 return hasFlag(tag, "json", "inline", 1) || 1295 hasFlag(tag, "yaml", "inline", 1) 1296} 1297 1298func (e *extractor) isOptional(tag string) bool { 1299 // TODO: also when the type is a list or other kind of pointer. 1300 return hasFlag(tag, "json", "omitempty", 1) || 1301 hasFlag(tag, "yaml", "omitempty", 1) 1302} 1303 1304func hasFlag(tag, key, flag string, offset int) bool { 1305 if t := reflect.StructTag(tag).Get(key); t != "" { 1306 split := strings.Split(t, ",") 1307 if offset >= len(split) { 1308 return false 1309 } 1310 for _, str := range split[offset:] { 1311 if str == flag { 1312 return true 1313 } 1314 } 1315 } 1316 return false 1317} 1318 1319func getName(name string, tag string) string { 1320 tags := reflect.StructTag(tag) 1321 for _, s := range []string{"json", "yaml"} { 1322 if tag, ok := tags.Lookup(s); ok { 1323 if p := strings.Index(tag, ","); p >= 0 { 1324 tag = tag[:p] 1325 } 1326 if tag != "" { 1327 return tag 1328 } 1329 } 1330 } 1331 // TODO: should we also consider to protobuf name? Probably not. 1332 1333 return name 1334} 1335