1package interp 2 3import ( 4 "bytes" 5 "context" 6 "crypto/md5" 7 "embed" 8 "encoding/base64" 9 "encoding/hex" 10 "errors" 11 "fmt" 12 "io" 13 "io/fs" 14 "io/ioutil" 15 "math/big" 16 "path/filepath" 17 "strconv" 18 "strings" 19 "time" 20 21 "github.com/mitchellh/mapstructure" 22 "github.com/wader/fq/internal/ansi" 23 "github.com/wader/fq/internal/colorjson" 24 "github.com/wader/fq/internal/ctxstack" 25 "github.com/wader/fq/internal/ioextra" 26 "github.com/wader/fq/internal/num" 27 "github.com/wader/fq/internal/pos" 28 "github.com/wader/fq/pkg/bitio" 29 "github.com/wader/fq/pkg/decode" 30 "github.com/wader/fq/pkg/registry" 31 32 "github.com/wader/gojq" 33) 34 35//go:embed interp.jq 36//go:embed internal.jq 37//go:embed options.jq 38//go:embed buffer.jq 39//go:embed decode.jq 40//go:embed match.jq 41//go:embed funcs.jq 42//go:embed grep.jq 43//go:embed args.jq 44//go:embed query.jq 45//go:embed repl.jq 46//go:embed formats.jq 47var builtinFS embed.FS 48 49var initSource = `include "@builtin/interp";` 50 51var functionRegisterFns []func(i *Interp) []Function 52 53func init() { 54 functionRegisterFns = append(functionRegisterFns, func(i *Interp) []Function { 55 return []Function{ 56 {"_readline", 0, 2, i.readline, nil}, 57 {"eval", 1, 2, nil, i.eval}, 58 {"stdin", 0, 0, nil, i.makeStdioFn(i.os.Stdin())}, 59 {"stdout", 0, 0, nil, i.makeStdioFn(i.os.Stdout())}, 60 {"stderr", 0, 0, nil, i.makeStdioFn(i.os.Stderr())}, 61 {"_extkeys", 0, 0, i._extKeys, nil}, 62 {"_exttype", 0, 0, i._extType, nil}, 63 {"_global_state", 0, 1, i.makeStateFn(i.state), nil}, 64 {"_registry", 0, 0, i._registry, nil}, 65 {"history", 0, 0, i.history, nil}, 66 {"_display", 1, 1, nil, i._display}, 67 {"_can_display", 0, 0, i._canDisplay, nil}, 68 {"_print_color_json", 0, 1, nil, i._printColorJSON}, 69 } 70 }) 71} 72 73type valueError struct { 74 v interface{} 75} 76 77func (v valueError) Error() string { return fmt.Sprintf("error: %v", v.v) } 78func (v valueError) Value() interface{} { return v.v } 79 80type compileError struct { 81 err error 82 what string 83 filename string 84 pos pos.Pos 85} 86 87func (ce compileError) Value() interface{} { 88 return map[string]interface{}{ 89 "error": ce.err.Error(), 90 "what": ce.what, 91 "filename": ce.filename, 92 "line": ce.pos.Line, 93 "column": ce.pos.Column, 94 } 95} 96func (ce compileError) Error() string { 97 filename := ce.filename 98 if filename == "" { 99 filename = "src" 100 } 101 return fmt.Sprintf("%s:%d:%d: %s: %s", filename, ce.pos.Line, ce.pos.Column, ce.what, ce.err.Error()) 102} 103 104var ErrEOF = io.EOF 105var ErrInterrupt = errors.New("Interrupt") 106 107// gojq errors can implement this to signal exit code 108type Exiter interface { 109 ExitCode() int 110} 111 112// gojq halt_error uses this 113type IsEmptyErrorer interface { 114 IsEmptyError() bool 115} 116 117type Terminal interface { 118 Size() (int, int) 119 IsTerminal() bool 120} 121 122type Input interface { 123 fs.File 124 Terminal 125} 126 127type Output interface { 128 io.Writer 129 Terminal 130} 131 132type OS interface { 133 Stdin() Input 134 Stdout() Output 135 Stderr() Output 136 InterruptChan() chan struct{} 137 Args() []string 138 Environ() []string 139 ConfigDir() (string, error) 140 // FS.File returned by FS().Open() can optionally implement io.Seeker 141 FS() fs.FS 142 Readline(prompt string, complete func(line string, pos int) (newLine []string, shared int)) (string, error) 143 History() ([]string, error) 144} 145 146type FixedFileInfo struct { 147 FName string 148 FSize int64 149 FMode fs.FileMode 150 FModTime time.Time 151 FIsDir bool 152 FSys interface{} 153} 154 155func (ffi FixedFileInfo) Name() string { return ffi.FName } 156func (ffi FixedFileInfo) Size() int64 { return ffi.FSize } 157func (ffi FixedFileInfo) Mode() fs.FileMode { return ffi.FMode } 158func (ffi FixedFileInfo) ModTime() time.Time { return ffi.FModTime } 159func (ffi FixedFileInfo) IsDir() bool { return ffi.FIsDir } 160func (ffi FixedFileInfo) Sys() interface{} { return ffi.FSys } 161 162type FileReader struct { 163 R io.Reader 164 FileInfo FixedFileInfo 165} 166 167func (rf FileReader) Stat() (fs.FileInfo, error) { return rf.FileInfo, nil } 168func (rf FileReader) Read(p []byte) (int, error) { return rf.R.Read(p) } 169func (FileReader) Close() error { return nil } 170 171type Value interface { 172 gojq.JQValue 173 174 ExtType() string 175 ExtKeys() []string 176} 177 178type Display interface { 179 Display(w io.Writer, opts Options) error 180} 181 182type JQValueEx interface { 183 JQValueToGoJQEx(optsFn func() Options) interface{} 184} 185 186func valuePath(v *decode.Value) []interface{} { 187 var parts []interface{} 188 189 for v.Parent != nil { 190 switch vv := v.Parent.V.(type) { 191 case *decode.Compound: 192 if vv.IsArray { 193 parts = append([]interface{}{v.Index}, parts...) 194 } else { 195 parts = append([]interface{}{v.Name}, parts...) 196 } 197 } 198 v = v.Parent 199 } 200 201 return parts 202} 203 204func valuePathDecorated(v *decode.Value, d Decorator) string { 205 var parts []string 206 207 for _, p := range valuePath(v) { 208 switch p := p.(type) { 209 case string: 210 parts = append(parts, ".", d.ObjectKey.Wrap(p)) 211 case int: 212 indexStr := strconv.Itoa(p) 213 parts = append(parts, fmt.Sprintf("%s%s%s", d.Index.F("["), d.Number.F(indexStr), d.Index.F("]"))) 214 } 215 } 216 217 if len(parts) == 0 { 218 return "." 219 } 220 221 return strings.Join(parts, "") 222} 223 224type iterFn func() (interface{}, bool) 225 226func (i iterFn) Next() (interface{}, bool) { return i() } 227 228type loadModule struct { 229 init func() ([]*gojq.Query, error) 230 load func(name string) (*gojq.Query, error) 231} 232 233func (l loadModule) LoadInitModules() ([]*gojq.Query, error) { return l.init() } 234func (l loadModule) LoadModule(name string) (*gojq.Query, error) { return l.load(name) } 235 236func toString(v interface{}) (string, error) { 237 switch v := v.(type) { 238 case string: 239 return v, nil 240 case gojq.JQValue: 241 return toString(v.JQValueToGoJQ()) 242 default: 243 b, err := toBytes(v) 244 if err != nil { 245 return "", fmt.Errorf("value can't be a string") 246 } 247 248 return string(b), nil 249 } 250} 251 252func toBigInt(v interface{}) (*big.Int, error) { 253 switch v := v.(type) { 254 case int: 255 return new(big.Int).SetInt64(int64(v)), nil 256 case float64: 257 return new(big.Int).SetInt64(int64(v)), nil 258 case *big.Int: 259 return v, nil 260 default: 261 return nil, fmt.Errorf("value is not a number") 262 } 263} 264 265func toBytes(v interface{}) ([]byte, error) { 266 switch v := v.(type) { 267 default: 268 bb, err := toBitBuf(v) 269 if err != nil { 270 return nil, fmt.Errorf("value is not bytes") 271 } 272 buf := &bytes.Buffer{} 273 if _, err := io.Copy(buf, bb); err != nil { 274 return nil, err 275 } 276 277 return buf.Bytes(), nil 278 } 279} 280 281func queryErrorPosition(src string, v error) pos.Pos { 282 var offset int 283 284 if tokIf, ok := v.(interface{ Token() (string, int) }); ok { //nolint:errorlint 285 _, offset = tokIf.Token() 286 } 287 if offset >= 0 { 288 return pos.NewFromOffset(src, offset) 289 } 290 return pos.Pos{} 291} 292 293type Variable struct { 294 Name string 295 Value interface{} 296} 297 298type Function struct { 299 Name string 300 MinArity int 301 MaxArity int 302 Fn func(interface{}, []interface{}) interface{} 303 IterFn func(interface{}, []interface{}) gojq.Iter 304} 305 306type RunMode int 307 308const ( 309 ScriptMode RunMode = iota 310 REPLMode 311 CompletionMode 312) 313 314type evalContext struct { 315 ctx context.Context 316 output io.Writer 317} 318 319type Interp struct { 320 registry *registry.Registry 321 os OS 322 initFqQuery *gojq.Query 323 includeCache map[string]*gojq.Query 324 interruptStack *ctxstack.Stack 325 // global state, is ref as Interp i cloned per eval 326 state *interface{} 327 328 // new for each run, other values are copied by value 329 evalContext evalContext 330} 331 332func New(os OS, registry *registry.Registry) (*Interp, error) { 333 var err error 334 335 i := &Interp{ 336 os: os, 337 registry: registry, 338 } 339 340 i.includeCache = map[string]*gojq.Query{} 341 i.initFqQuery, err = gojq.Parse(initSource) 342 if err != nil { 343 return nil, fmt.Errorf("init:%s: %w", queryErrorPosition(initSource, err), err) 344 } 345 // TODO: refactor ctxstack have a CancelTop and return c context to Stop? 346 i.interruptStack = ctxstack.New(func(stopCh chan struct{}) { 347 select { 348 case <-stopCh: 349 return 350 case <-os.InterruptChan(): 351 return 352 } 353 }) 354 i.state = new(interface{}) 355 356 return i, nil 357} 358 359func (i *Interp) Stop() { 360 // TODO: cancel all run instances? 361 i.interruptStack.Stop() 362} 363 364func (i *Interp) Main(ctx context.Context, output Output, version string) error { 365 var args []interface{} 366 for _, a := range i.os.Args() { 367 args = append(args, a) 368 } 369 370 input := map[string]interface{}{ 371 "args": args, 372 "version": version, 373 } 374 375 iter, err := i.EvalFunc(ctx, input, "_main", nil, output) 376 if err != nil { 377 fmt.Fprintln(i.os.Stderr(), err) 378 return err 379 } 380 for { 381 v, ok := iter.Next() 382 if !ok { 383 break 384 } 385 386 switch v := v.(type) { 387 case error: 388 if emptyErr, ok := v.(IsEmptyErrorer); ok && emptyErr.IsEmptyError() { //nolint:errorlint 389 // no output 390 } else if errors.Is(v, context.Canceled) { 391 // ignore context cancel here for now, which means user somehow interrupted the interpreter 392 // TODO: handle this inside interp.jq instead but then we probably have to do nested 393 // eval and or also use different contexts for the interpreter and reading/decoding 394 } else { 395 fmt.Fprintln(i.os.Stderr(), v) 396 } 397 return v 398 case [2]interface{}: 399 fmt.Fprintln(i.os.Stderr(), v[:]...) 400 default: 401 // TODO: can this happen? 402 fmt.Fprintln(i.os.Stderr(), v) 403 } 404 } 405 406 return nil 407} 408 409func (i *Interp) readline(c interface{}, a []interface{}) interface{} { 410 var opts struct { 411 Complete string `mapstructure:"complete"` 412 Timeout float64 `mapstructure:"timeout"` 413 } 414 415 var err error 416 prompt := "" 417 418 if len(a) > 0 { 419 prompt, err = toString(a[0]) 420 if err != nil { 421 return fmt.Errorf("prompt: %w", err) 422 } 423 } 424 if len(a) > 1 { 425 _ = mapstructure.Decode(a[1], &opts) 426 } 427 428 src, err := i.os.Readline( 429 prompt, 430 func(line string, pos int) (newLine []string, shared int) { 431 completeCtx := i.evalContext.ctx 432 if opts.Timeout > 0 { 433 var completeCtxCancelFn context.CancelFunc 434 completeCtx, completeCtxCancelFn = context.WithTimeout(i.evalContext.ctx, time.Duration(opts.Timeout*float64(time.Second))) 435 defer completeCtxCancelFn() 436 } 437 438 names, shared, err := func() (newLine []string, shared int, err error) { 439 // c | opts.Complete(line; pos) 440 vs, err := i.EvalFuncValues( 441 completeCtx, 442 c, 443 opts.Complete, 444 []interface{}{line, pos}, 445 ioextra.DiscardCtxWriter{Ctx: completeCtx}, 446 ) 447 if err != nil { 448 return nil, pos, err 449 } 450 if len(vs) < 1 { 451 return nil, pos, fmt.Errorf("no values") 452 } 453 v := vs[0] 454 if vErr, ok := v.(error); ok { 455 return nil, pos, vErr 456 } 457 458 // {abc: 123, abd: 123} | complete(".ab"; 3) will return {prefix: "ab", names: ["abc", "abd"]} 459 460 var result struct { 461 Names []string `mapstructure:"names"` 462 Prefix string `mapstructure:"prefix"` 463 } 464 465 _ = mapstructure.Decode(v, &result) 466 if len(result.Names) == 0 { 467 return nil, pos, nil 468 } 469 470 sharedLen := len(result.Prefix) 471 472 return result.Names, sharedLen, nil 473 }() 474 475 // TODO: how to report err? 476 _ = err 477 478 return names, shared 479 }, 480 ) 481 482 if errors.Is(err, ErrInterrupt) { 483 return valueError{"interrupt"} 484 } else if errors.Is(err, ErrEOF) { 485 return valueError{"eof"} 486 } else if err != nil { 487 return err 488 } 489 490 return src 491} 492 493func (i *Interp) eval(c interface{}, a []interface{}) gojq.Iter { 494 var err error 495 src, err := toString(a[0]) 496 if err != nil { 497 return gojq.NewIter(fmt.Errorf("src: %w", err)) 498 } 499 var filenameHint string 500 if len(a) >= 2 { 501 filenameHint, err = toString(a[1]) 502 if err != nil { 503 return gojq.NewIter(fmt.Errorf("filename hint: %w", err)) 504 } 505 } 506 507 iter, err := i.Eval(i.evalContext.ctx, c, src, filenameHint, i.evalContext.output) 508 if err != nil { 509 return gojq.NewIter(err) 510 } 511 512 return iter 513} 514 515func (i *Interp) _extKeys(c interface{}, a []interface{}) interface{} { 516 if v, ok := c.(Value); ok { 517 var vs []interface{} 518 for _, s := range v.ExtKeys() { 519 vs = append(vs, s) 520 } 521 return vs 522 } 523 return nil 524} 525 526func (i *Interp) _extType(c interface{}, a []interface{}) interface{} { 527 if v, ok := c.(Value); ok { 528 return v.ExtType() 529 } 530 return nil 531} 532 533func (i *Interp) makeStateFn(state *interface{}) func(c interface{}, a []interface{}) interface{} { 534 return func(c interface{}, a []interface{}) interface{} { 535 if len(a) > 0 { 536 *state = a[0] 537 } 538 return *state 539 } 540} 541 542func (i *Interp) _registry(c interface{}, a []interface{}) interface{} { 543 uniqueFormats := map[string]decode.Format{} 544 545 groups := map[string]interface{}{} 546 formats := map[string]interface{}{} 547 548 for fsName := range i.registry.Groups { 549 var group []interface{} 550 551 for _, f := range i.registry.MustGroup(fsName) { 552 group = append(group, f.Name) 553 if _, ok := uniqueFormats[f.Name]; ok { 554 continue 555 } 556 uniqueFormats[f.Name] = f 557 } 558 559 groups[fsName] = group 560 } 561 562 for _, f := range uniqueFormats { 563 vf := map[string]interface{}{ 564 "name": f.Name, 565 "description": f.Description, 566 "probe_order": f.ProbeOrder, 567 "root_name": f.RootName, 568 "root_array": f.RootArray, 569 } 570 571 var dependenciesVs []interface{} 572 for _, d := range f.Dependencies { 573 var dNamesVs []interface{} 574 for _, n := range d.Names { 575 dNamesVs = append(dNamesVs, n) 576 } 577 dependenciesVs = append(dependenciesVs, dNamesVs) 578 } 579 if len(dependenciesVs) > 0 { 580 vf["dependencies"] = dependenciesVs 581 } 582 var groupsVs []interface{} 583 for _, n := range f.Groups { 584 groupsVs = append(groupsVs, n) 585 } 586 if len(groupsVs) > 0 { 587 vf["groups"] = groupsVs 588 } 589 590 if f.Files != nil { 591 files := map[string]interface{}{} 592 593 entries, err := f.Files.ReadDir(".") 594 if err != nil { 595 return err 596 } 597 598 for _, e := range entries { 599 f, err := f.Files.Open(e.Name()) 600 if err != nil { 601 return err 602 } 603 b, err := ioutil.ReadAll(f) 604 if err != nil { 605 return err 606 } 607 files[e.Name()] = string(b) 608 } 609 610 vf["files"] = files 611 } 612 613 formats[f.Name] = vf 614 } 615 616 return map[string]interface{}{ 617 "groups": groups, 618 "formats": formats, 619 } 620} 621 622func (i *Interp) makeStdioFn(t Terminal) func(c interface{}, a []interface{}) gojq.Iter { 623 return func(c interface{}, a []interface{}) gojq.Iter { 624 if c == nil { 625 w, h := t.Size() 626 return gojq.NewIter(map[string]interface{}{ 627 "is_terminal": t.IsTerminal(), 628 "width": w, 629 "height": h, 630 }) 631 } 632 633 if w, ok := t.(io.Writer); ok { 634 if _, err := fmt.Fprint(w, c); err != nil { 635 return gojq.NewIter(err) 636 } 637 return gojq.NewIter() 638 } 639 640 return gojq.NewIter(fmt.Errorf("%v: it not writeable", c)) 641 } 642} 643 644func (i *Interp) history(c interface{}, a []interface{}) interface{} { 645 hs, err := i.os.History() 646 if err != nil { 647 return err 648 } 649 var vs []interface{} 650 for _, s := range hs { 651 vs = append(vs, s) 652 } 653 return vs 654} 655 656func (i *Interp) _display(c interface{}, a []interface{}) gojq.Iter { 657 opts := i.Options(a[0]) 658 659 switch v := c.(type) { 660 case Display: 661 if err := v.Display(i.evalContext.output, opts); err != nil { 662 return gojq.NewIter(err) 663 } 664 return gojq.NewIter() 665 default: 666 return gojq.NewIter(fmt.Errorf("%+#v: not displayable", c)) 667 } 668} 669 670func (i *Interp) _printColorJSON(c interface{}, a []interface{}) gojq.Iter { 671 opts := i.Options(a[0]) 672 673 cj, err := i.NewColorJSON(opts) 674 if err != nil { 675 return gojq.NewIter(err) 676 } 677 if err := cj.Marshal(c, i.evalContext.output); err != nil { 678 return gojq.NewIter(err) 679 } 680 681 return gojq.NewIter() 682} 683 684func (i *Interp) _canDisplay(c interface{}, a []interface{}) interface{} { 685 _, ok := c.(Display) 686 return ok 687} 688 689func (i *Interp) Eval(ctx context.Context, c interface{}, src string, srcFilename string, output io.Writer) (gojq.Iter, error) { 690 gq, err := gojq.Parse(src) 691 if err != nil { 692 p := queryErrorPosition(src, err) 693 return nil, compileError{ 694 err: err, 695 what: "parse", 696 filename: srcFilename, 697 pos: p, 698 } 699 } 700 701 // make copy of interp and give it its own eval context 702 ci := *i 703 ni := &ci 704 ni.evalContext = evalContext{} 705 706 var variableNames []string 707 var variableValues []interface{} 708 for k, v := range i.variables() { 709 variableNames = append(variableNames, "$"+k) 710 variableValues = append(variableValues, v) 711 } 712 713 var funcCompilerOpts []gojq.CompilerOption 714 for _, frFn := range functionRegisterFns { 715 for _, f := range frFn(ni) { 716 if f.IterFn != nil { 717 funcCompilerOpts = append(funcCompilerOpts, 718 gojq.WithIterFunction(f.Name, f.MinArity, f.MaxArity, f.IterFn)) 719 } else { 720 funcCompilerOpts = append(funcCompilerOpts, 721 gojq.WithFunction(f.Name, f.MinArity, f.MaxArity, f.Fn)) 722 } 723 } 724 } 725 726 compilerOpts := append([]gojq.CompilerOption{}, funcCompilerOpts...) 727 compilerOpts = append(compilerOpts, gojq.WithEnvironLoader(ni.os.Environ)) 728 compilerOpts = append(compilerOpts, gojq.WithVariables(variableNames)) 729 compilerOpts = append(compilerOpts, gojq.WithModuleLoader(loadModule{ 730 init: func() ([]*gojq.Query, error) { 731 return []*gojq.Query{i.initFqQuery}, nil 732 }, 733 load: func(name string) (*gojq.Query, error) { 734 if err := ctx.Err(); err != nil { 735 return nil, err 736 } 737 738 var filename string 739 // support include "nonexisting?" to ignore include error 740 var isTry bool 741 if strings.HasSuffix(name, "?") { 742 isTry = true 743 filename = name[0 : len(name)-1] 744 } else { 745 filename = name 746 } 747 filename = filename + ".jq" 748 749 pathPrefixes := []struct { 750 prefix string 751 fn func(filename string) (io.ReadCloser, error) 752 }{ 753 { 754 "@builtin/", func(filename string) (io.ReadCloser, error) { 755 return builtinFS.Open(filename) 756 }, 757 }, 758 { 759 "@config/", func(filename string) (io.ReadCloser, error) { 760 configDir, err := i.os.ConfigDir() 761 if err != nil { 762 return nil, err 763 } 764 return i.os.FS().Open(filepath.Join(configDir, filename)) 765 }, 766 }, 767 { 768 "", func(filename string) (io.ReadCloser, error) { 769 // TODO: jq $ORIGIN 770 771 if filepath.IsAbs(filename) { 772 return i.os.FS().Open(filename) 773 } 774 775 for _, path := range append([]string{"./"}, i.includePaths()...) { 776 if f, err := i.os.FS().Open(filepath.Join(path, filename)); err == nil { 777 return f, nil 778 } 779 } 780 781 return nil, &fs.PathError{Op: "open", Path: filename, Err: fs.ErrNotExist} 782 }, 783 }, 784 } 785 786 for _, p := range pathPrefixes { 787 if !strings.HasPrefix(filename, p.prefix) { 788 continue 789 } 790 791 if q, ok := ni.includeCache[filename]; ok { 792 return q, nil 793 } 794 795 filenamePart := strings.TrimPrefix(filename, p.prefix) 796 f, err := p.fn(filenamePart) 797 if err != nil { 798 if !isTry { 799 return nil, err 800 } 801 f = io.NopCloser(&bytes.Buffer{}) 802 } 803 defer f.Close() 804 805 b, err := io.ReadAll(f) 806 if err != nil { 807 return nil, err 808 } 809 s := string(b) 810 q, err := gojq.Parse(s) 811 if err != nil { 812 p := queryErrorPosition(s, err) 813 return nil, compileError{ 814 err: err, 815 what: "parse", 816 filename: filenamePart, 817 pos: p, 818 } 819 } 820 821 // not identity body means it returns something, threat as dynamic include 822 if q.Term.Type != gojq.TermTypeIdentity { 823 gc, err := gojq.Compile(q, funcCompilerOpts...) 824 if err != nil { 825 return nil, err 826 } 827 iter := gc.RunWithContext(context.Background(), nil) 828 var vs []interface{} 829 for { 830 v, ok := iter.Next() 831 if !ok { 832 break 833 } 834 if err, ok := v.(error); ok { 835 return nil, err 836 } 837 vs = append(vs, v) 838 } 839 if len(vs) != 1 { 840 return nil, fmt.Errorf("dynamic include: must output one string, got: %#v", vs) 841 } 842 s, sOk := vs[0].(string) 843 if !sOk { 844 return nil, fmt.Errorf("dynamic include: must be string, got %#v", s) 845 } 846 q, err = gojq.Parse(s) 847 if err != nil { 848 p := queryErrorPosition(s, err) 849 return nil, compileError{ 850 err: err, 851 what: "parse", 852 filename: filenamePart, 853 pos: p, 854 } 855 } 856 } 857 858 // TODO: some better way of handling relative includes that 859 // works with @builtin etc 860 basePath := filepath.Dir(name) 861 for _, i := range q.Imports { 862 rewritePath := func(base, path string) string { 863 if strings.HasPrefix(i.IncludePath, "@") { 864 return path 865 } 866 if filepath.IsAbs(i.IncludePath) { 867 return path 868 } 869 return filepath.Join(base, path) 870 } 871 i.IncludePath = rewritePath(basePath, i.IncludePath) 872 i.ImportPath = rewritePath(basePath, i.ImportPath) 873 } 874 875 i.includeCache[filename] = q 876 877 return q, nil 878 } 879 880 panic("unreachable") 881 }, 882 })) 883 884 gc, err := gojq.Compile(gq, compilerOpts...) 885 if err != nil { 886 p := queryErrorPosition(src, err) 887 return nil, compileError{ 888 err: err, 889 what: "compile", 890 filename: srcFilename, 891 pos: p, 892 } 893 } 894 895 runCtx, runCtxCancelFn := i.interruptStack.Push(ctx) 896 ni.evalContext.ctx = runCtx 897 ni.evalContext.output = ioextra.CtxWriter{Writer: output, Ctx: runCtx} 898 iter := gc.RunWithContext(runCtx, c, variableValues...) 899 900 iterWrapper := iterFn(func() (interface{}, bool) { 901 v, ok := iter.Next() 902 // gojq ctx cancel will not return ok=false, just cancelled error 903 if !ok { 904 runCtxCancelFn() 905 } 906 return v, ok 907 }) 908 909 return iterWrapper, nil 910} 911 912func (i *Interp) EvalFunc(ctx context.Context, c interface{}, name string, args []interface{}, output io.Writer) (gojq.Iter, error) { 913 var argsExpr []string 914 for i := range args { 915 argsExpr = append(argsExpr, fmt.Sprintf("$_args[%d]", i)) 916 } 917 argExpr := "" 918 if len(argsExpr) > 0 { 919 argExpr = "(" + strings.Join(argsExpr, ";") + ")" 920 } 921 922 trampolineInput := map[string]interface{}{ 923 "input": c, 924 "args": args, 925 } 926 // _args to mark variable as internal and hide it from completion 927 // {input: ..., args: [...]} | .args as {args: $_args} | .input | name[($_args[0]; ...)] 928 trampolineExpr := fmt.Sprintf(". as {args: $_args} | .input | %s%s", name, argExpr) 929 iter, err := i.Eval(ctx, trampolineInput, trampolineExpr, "", output) 930 if err != nil { 931 return nil, err 932 } 933 return iter, nil 934} 935 936func (i *Interp) EvalFuncValues(ctx context.Context, c interface{}, name string, args []interface{}, output io.Writer) ([]interface{}, error) { 937 iter, err := i.EvalFunc(ctx, c, name, args, output) 938 if err != nil { 939 return nil, err 940 } 941 942 var vs []interface{} 943 for { 944 v, ok := iter.Next() 945 if !ok { 946 break 947 } 948 vs = append(vs, v) 949 } 950 951 return vs, nil 952} 953 954type Options struct { 955 Depth int `mapstructure:"depth"` 956 ArrayTruncate int `mapstructure:"array_truncate"` 957 Verbose bool `mapstructure:"verbose"` 958 DecodeProgress bool `mapstructure:"decode_progress"` 959 Color bool `mapstructure:"color"` 960 Colors string `mapstructure:"colors"` 961 ByteColors string `mapstructure:"byte_colors"` 962 Unicode bool `mapstructure:"unicode"` 963 RawOutput bool `mapstructure:"raw_output"` 964 REPL bool `mapstructure:"repl"` 965 RawString bool `mapstructure:"raw_string"` 966 JoinString string `mapstructure:"join_string"` 967 Compact bool `mapstructure:"compact"` 968 BitsFormat string `mapstructure:"bits_format"` 969 LineBytes int `mapstructure:"line_bytes"` 970 DisplayBytes int `mapstructure:"display_bytes"` 971 AddrBase int `mapstructure:"addrbase"` 972 SizeBase int `mapstructure:"sizebase"` 973 974 Decorator Decorator 975 BitsFormatFn func(bb *bitio.Buffer) (interface{}, error) 976} 977 978func bitsFormatFnFromOptions(opts Options) func(bb *bitio.Buffer) (interface{}, error) { 979 switch opts.BitsFormat { 980 case "md5": 981 return func(bb *bitio.Buffer) (interface{}, error) { 982 d := md5.New() 983 if _, err := io.Copy(d, bb); err != nil { 984 return "", err 985 } 986 return hex.EncodeToString(d.Sum(nil)), nil 987 } 988 case "base64": 989 return func(bb *bitio.Buffer) (interface{}, error) { 990 b := &bytes.Buffer{} 991 e := base64.NewEncoder(base64.StdEncoding, b) 992 if _, err := io.Copy(e, bb); err != nil { 993 return "", err 994 } 995 e.Close() 996 return b.String(), nil 997 } 998 case "truncate": 999 // TODO: configure 1000 return func(bb *bitio.Buffer) (interface{}, error) { 1001 b := &bytes.Buffer{} 1002 if _, err := io.Copy(b, io.LimitReader(bb, 1024)); err != nil { 1003 return "", err 1004 } 1005 return b.String(), nil 1006 } 1007 case "string": 1008 return func(bb *bitio.Buffer) (interface{}, error) { 1009 b := &bytes.Buffer{} 1010 if _, err := io.Copy(b, bb); err != nil { 1011 return "", err 1012 } 1013 return b.String(), nil 1014 } 1015 case "snippet": 1016 fallthrough 1017 default: 1018 return func(bb *bitio.Buffer) (interface{}, error) { 1019 b := &bytes.Buffer{} 1020 e := base64.NewEncoder(base64.StdEncoding, b) 1021 if _, err := io.Copy(e, io.LimitReader(bb, 256)); err != nil { 1022 return "", err 1023 } 1024 e.Close() 1025 return fmt.Sprintf("<%s>%s", num.Bits(bb.Len()).StringByteBits(opts.SizeBase), b.String()), nil 1026 } 1027 } 1028} 1029 1030func (i *Interp) lookupState(key string) interface{} { 1031 if i.state == nil { 1032 return nil 1033 } 1034 m, ok := (*i.state).(map[string]interface{}) 1035 if !ok { 1036 return nil 1037 } 1038 return m[key] 1039} 1040 1041func (i *Interp) includePaths() []string { 1042 pathsAny, _ := i.lookupState("include_paths").([]interface{}) 1043 var paths []string 1044 for _, pathAny := range pathsAny { 1045 paths = append(paths, pathAny.(string)) 1046 } 1047 return paths 1048} 1049 1050func (i *Interp) variables() map[string]interface{} { 1051 variablesAny, _ := i.lookupState("variables").(map[string]interface{}) 1052 return variablesAny 1053} 1054 1055func (i *Interp) Options(v interface{}) Options { 1056 var opts Options 1057 _ = mapstructure.Decode(v, &opts) 1058 opts.ArrayTruncate = num.MaxInt(0, opts.ArrayTruncate) 1059 opts.Depth = num.MaxInt(0, opts.Depth) 1060 opts.AddrBase = num.ClampInt(2, 36, opts.AddrBase) 1061 opts.SizeBase = num.ClampInt(2, 36, opts.SizeBase) 1062 opts.LineBytes = num.MaxInt(0, opts.LineBytes) 1063 opts.DisplayBytes = num.MaxInt(0, opts.DisplayBytes) 1064 opts.Decorator = decoratorFromOptions(opts) 1065 opts.BitsFormatFn = bitsFormatFnFromOptions(opts) 1066 1067 return opts 1068} 1069 1070func (i *Interp) NewColorJSON(opts Options) (*colorjson.Encoder, error) { 1071 indent := 2 1072 if opts.Compact { 1073 indent = 0 1074 } 1075 1076 return colorjson.NewEncoder( 1077 opts.Color, 1078 false, 1079 indent, 1080 func(v interface{}) interface{} { 1081 if v, ok := toValue(func() Options { return opts }, v); ok { 1082 return v 1083 } 1084 panic(fmt.Sprintf("toValue not a JQValue value: %#v", v)) 1085 }, 1086 colorjson.Colors{ 1087 Reset: []byte(ansi.Reset.SetString), 1088 Null: []byte(opts.Decorator.Null.SetString), 1089 False: []byte(opts.Decorator.False.SetString), 1090 True: []byte(opts.Decorator.True.SetString), 1091 Number: []byte(opts.Decorator.Number.SetString), 1092 String: []byte(opts.Decorator.String.SetString), 1093 ObjectKey: []byte(opts.Decorator.ObjectKey.SetString), 1094 Array: []byte(opts.Decorator.Array.SetString), 1095 Object: []byte(opts.Decorator.Object.SetString), 1096 }, 1097 ), nil 1098} 1099