1// Copyright 2017 Google Inc. All Rights Reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15//go:generate ./COMPILE-PROTOS.sh 16 17// Gnostic is a tool for building better REST APIs through knowledge. 18// 19// Gnostic reads declarative descriptions of REST APIs that conform 20// to the OpenAPI Specification, reports errors, resolves internal 21// dependencies, and puts the results in a binary form that can 22// be used in any language that is supported by the Protocol Buffer 23// tools. 24// 25// Gnostic models are validated and typed. This allows API tool 26// developers to focus on their product and not worry about input 27// validation and type checking. 28// 29// Gnostic calls plugins that implement a variety of API implementation 30// and support features including generation of client and server 31// support code. 32package main 33 34import ( 35 "bytes" 36 "errors" 37 "fmt" 38 "io" 39 "os" 40 "os/exec" 41 "path/filepath" 42 "regexp" 43 "strings" 44 "time" 45 46 "github.com/golang/protobuf/proto" 47 "github.com/googleapis/gnostic/OpenAPIv2" 48 "github.com/googleapis/gnostic/OpenAPIv3" 49 "github.com/googleapis/gnostic/compiler" 50 "github.com/googleapis/gnostic/discovery" 51 "github.com/googleapis/gnostic/jsonwriter" 52 plugins "github.com/googleapis/gnostic/plugins" 53 surface "github.com/googleapis/gnostic/surface" 54 "gopkg.in/yaml.v2" 55) 56 57const ( // Source Format 58 SourceFormatUnknown = 0 59 SourceFormatOpenAPI2 = 2 60 SourceFormatOpenAPI3 = 3 61 SourceFormatDiscovery = 4 62) 63 64// Determine the version of an OpenAPI description read from JSON or YAML. 65func getOpenAPIVersionFromInfo(info interface{}) int { 66 m, ok := compiler.UnpackMap(info) 67 if !ok { 68 return SourceFormatUnknown 69 } 70 swagger, ok := compiler.MapValueForKey(m, "swagger").(string) 71 if ok && strings.HasPrefix(swagger, "2.0") { 72 return SourceFormatOpenAPI2 73 } 74 openapi, ok := compiler.MapValueForKey(m, "openapi").(string) 75 if ok && strings.HasPrefix(openapi, "3.0") { 76 return SourceFormatOpenAPI3 77 } 78 kind, ok := compiler.MapValueForKey(m, "kind").(string) 79 if ok && kind == "discovery#restDescription" { 80 return SourceFormatDiscovery 81 } 82 return SourceFormatUnknown 83} 84 85const ( 86 pluginPrefix = "gnostic-" 87 extensionPrefix = "gnostic-x-" 88) 89 90type pluginCall struct { 91 Name string 92 Invocation string 93} 94 95// Invokes a plugin. 96func (p *pluginCall) perform(document proto.Message, sourceFormat int, sourceName string, timePlugins bool) ([]*plugins.Message, error) { 97 if p.Name != "" { 98 request := &plugins.Request{} 99 100 // Infer the name of the executable by adding the prefix. 101 executableName := pluginPrefix + p.Name 102 103 // Validate invocation string with regular expression. 104 invocation := p.Invocation 105 106 // 107 // Plugin invocations must consist of 108 // zero or more comma-separated key=value pairs followed by a path. 109 // If pairs are present, a colon separates them from the path. 110 // Keys and values must be alphanumeric strings and may contain 111 // dashes, underscores, periods, or forward slashes. 112 // A path can contain any characters other than the separators ',', ':', and '='. 113 // 114 invocationRegex := regexp.MustCompile(`^([\w-_\/\.]+=[\w-_\/\.]+(,[\w-_\/\.]+=[\w-_\/\.]+)*:)?[^,:=]+$`) 115 if !invocationRegex.Match([]byte(p.Invocation)) { 116 return nil, fmt.Errorf("Invalid invocation of %s: %s", executableName, invocation) 117 } 118 119 invocationParts := strings.Split(p.Invocation, ":") 120 var outputLocation string 121 switch len(invocationParts) { 122 case 1: 123 outputLocation = invocationParts[0] 124 case 2: 125 parameters := strings.Split(invocationParts[0], ",") 126 for _, keyvalue := range parameters { 127 pair := strings.Split(keyvalue, "=") 128 if len(pair) == 2 { 129 request.Parameters = append(request.Parameters, &plugins.Parameter{Name: pair[0], Value: pair[1]}) 130 } 131 } 132 outputLocation = invocationParts[1] 133 default: 134 // badly-formed request 135 outputLocation = invocationParts[len(invocationParts)-1] 136 } 137 138 version := &plugins.Version{} 139 version.Major = 0 140 version.Minor = 1 141 version.Patch = 0 142 request.CompilerVersion = version 143 144 request.OutputPath = outputLocation 145 146 request.SourceName = sourceName 147 switch sourceFormat { 148 case SourceFormatOpenAPI2: 149 request.AddModel("openapi.v2.Document", document) 150 // include experimental API surface model 151 surfaceModel, err := surface.NewModelFromOpenAPI2(document.(*openapi_v2.Document), sourceName) 152 if err == nil { 153 request.AddModel("surface.v1.Model", surfaceModel) 154 } 155 case SourceFormatOpenAPI3: 156 request.AddModel("openapi.v3.Document", document) 157 // include experimental API surface model 158 surfaceModel, err := surface.NewModelFromOpenAPI3(document.(*openapi_v3.Document), sourceName) 159 if err == nil { 160 request.AddModel("surface.v1.Model", surfaceModel) 161 } 162 case SourceFormatDiscovery: 163 request.AddModel("discovery.v1.Document", document) 164 default: 165 } 166 167 requestBytes, _ := proto.Marshal(request) 168 169 cmd := exec.Command(executableName, "-plugin") 170 cmd.Stdin = bytes.NewReader(requestBytes) 171 cmd.Stderr = os.Stderr 172 pluginStartTime := time.Now() 173 output, err := cmd.Output() 174 pluginElapsedTime := time.Since(pluginStartTime) 175 if timePlugins { 176 fmt.Printf("> %s (%s)\n", executableName, pluginElapsedTime) 177 } 178 if err != nil { 179 return nil, err 180 } 181 response := &plugins.Response{} 182 err = proto.Unmarshal(output, response) 183 if err != nil { 184 // Gnostic expects plugins to only write the 185 // response message to stdout. Be sure that 186 // any logging messages are written to stderr only. 187 return nil, errors.New("Invalid plugin response (plugins must write log messages to stderr, not stdout).") 188 } 189 190 err = plugins.HandleResponse(response, outputLocation) 191 192 return response.Messages, err 193 } 194 return nil, nil 195} 196 197func isFile(path string) bool { 198 fileInfo, err := os.Stat(path) 199 if err != nil { 200 return false 201 } 202 return !fileInfo.IsDir() 203} 204 205func isDirectory(path string) bool { 206 fileInfo, err := os.Stat(path) 207 if err != nil { 208 return false 209 } 210 return fileInfo.IsDir() 211} 212 213// Write bytes to a named file. 214// Certain names have special meaning: 215// ! writes nothing 216// - writes to stdout 217// = writes to stderr 218// If a directory name is given, the file is written there with 219// a name derived from the source and extension arguments. 220func writeFile(name string, bytes []byte, source string, extension string) { 221 var writer io.Writer 222 if name == "!" { 223 return 224 } else if name == "-" { 225 writer = os.Stdout 226 } else if name == "=" { 227 writer = os.Stderr 228 } else if isDirectory(name) { 229 base := filepath.Base(source) 230 // Remove the original source extension. 231 base = base[0 : len(base)-len(filepath.Ext(base))] 232 // Build the path that puts the result in the passed-in directory. 233 filename := name + "/" + base + "." + extension 234 file, _ := os.Create(filename) 235 defer file.Close() 236 writer = file 237 } else { 238 file, _ := os.Create(name) 239 defer file.Close() 240 writer = file 241 } 242 writer.Write(bytes) 243 if name == "-" || name == "=" { 244 writer.Write([]byte("\n")) 245 } 246} 247 248// The Gnostic structure holds global state information for gnostic. 249type Gnostic struct { 250 usage string 251 sourceName string 252 binaryOutputPath string 253 textOutputPath string 254 yamlOutputPath string 255 jsonOutputPath string 256 errorOutputPath string 257 messageOutputPath string 258 resolveReferences bool 259 pluginCalls []*pluginCall 260 extensionHandlers []compiler.ExtensionHandler 261 sourceFormat int 262 timePlugins bool 263} 264 265// Initialize a structure to store global application state. 266func newGnostic() *Gnostic { 267 g := &Gnostic{} 268 // Option fields initialize to their default values. 269 g.usage = ` 270Usage: gnostic SOURCE [OPTIONS] 271 SOURCE is the filename or URL of an API description. 272Options: 273 --pb-out=PATH Write a binary proto to the specified location. 274 --text-out=PATH Write a text proto to the specified location. 275 --json-out=PATH Write a json API description to the specified location. 276 --yaml-out=PATH Write a yaml API description to the specified location. 277 --errors-out=PATH Write compilation errors to the specified location. 278 --messages-out=PATH Write messages generated by plugins to the specified 279 location. Messages from all plugin invocations are 280 written to a single common file. 281 --PLUGIN-out=PATH Run the plugin named gnostic-PLUGIN and write results 282 to the specified location. 283 --PLUGIN Run the plugin named gnostic-PLUGIN but don't write any 284 results. Used for plugins that return messages only. 285 PLUGIN must not match any other gnostic option. 286 --x-EXTENSION Use the extension named gnostic-x-EXTENSION 287 to process OpenAPI specification extensions. 288 --resolve-refs Explicitly resolve $ref references. 289 This could have problems with recursive definitions. 290 --time-plugins Report plugin runtimes. 291` 292 // Initialize internal structures. 293 g.pluginCalls = make([]*pluginCall, 0) 294 g.extensionHandlers = make([]compiler.ExtensionHandler, 0) 295 return g 296} 297 298// Parse command-line options. 299func (g *Gnostic) readOptions() { 300 // plugin processing matches patterns of the form "--PLUGIN-out=PATH" and "--PLUGIN_out=PATH" 301 pluginRegex := regexp.MustCompile("--(.+)[-_]out=(.+)") 302 303 // extension processing matches patterns of the form "--x-EXTENSION" 304 extensionRegex := regexp.MustCompile("--x-(.+)") 305 306 for i, arg := range os.Args { 307 if i == 0 { 308 continue // skip the tool name 309 } 310 var m [][]byte 311 if m = pluginRegex.FindSubmatch([]byte(arg)); m != nil { 312 pluginName := string(m[1]) 313 invocation := string(m[2]) 314 switch pluginName { 315 case "pb": 316 g.binaryOutputPath = invocation 317 case "text": 318 g.textOutputPath = invocation 319 case "json": 320 g.jsonOutputPath = invocation 321 case "yaml": 322 g.yamlOutputPath = invocation 323 case "errors": 324 g.errorOutputPath = invocation 325 case "messages": 326 g.messageOutputPath = invocation 327 default: 328 p := &pluginCall{Name: pluginName, Invocation: invocation} 329 g.pluginCalls = append(g.pluginCalls, p) 330 } 331 } else if m = extensionRegex.FindSubmatch([]byte(arg)); m != nil { 332 extensionName := string(m[1]) 333 extensionHandler := compiler.ExtensionHandler{Name: extensionPrefix + extensionName} 334 g.extensionHandlers = append(g.extensionHandlers, extensionHandler) 335 } else if arg == "--resolve-refs" { 336 g.resolveReferences = true 337 } else if arg == "--time-plugins" { 338 g.timePlugins = true 339 } else if arg[0] == '-' && arg[1] == '-' { 340 // try letting the option specify a plugin with no output files (or unwanted output files) 341 // this is useful for calling plugins like linters that only return messages 342 p := &pluginCall{Name: arg[2:len(arg)], Invocation: "!"} 343 g.pluginCalls = append(g.pluginCalls, p) 344 } else if arg[0] == '-' { 345 fmt.Fprintf(os.Stderr, "Unknown option: %s.\n%s\n", arg, g.usage) 346 os.Exit(-1) 347 } else { 348 g.sourceName = arg 349 } 350 } 351} 352 353// Validate command-line options. 354func (g *Gnostic) validateOptions() { 355 if g.binaryOutputPath == "" && 356 g.textOutputPath == "" && 357 g.yamlOutputPath == "" && 358 g.jsonOutputPath == "" && 359 g.errorOutputPath == "" && 360 len(g.pluginCalls) == 0 { 361 fmt.Fprintf(os.Stderr, "Missing output directives.\n%s\n", g.usage) 362 os.Exit(-1) 363 } 364 if g.sourceName == "" { 365 fmt.Fprintf(os.Stderr, "No input specified.\n%s\n", g.usage) 366 os.Exit(-1) 367 } 368 // If we get here and the error output is unspecified, write errors to stderr. 369 if g.errorOutputPath == "" { 370 g.errorOutputPath = "=" 371 } 372} 373 374// Generate an error message to be written to stderr or a file. 375func (g *Gnostic) errorBytes(err error) []byte { 376 return []byte("Errors reading " + g.sourceName + "\n" + err.Error()) 377} 378 379// Read an OpenAPI description from YAML or JSON. 380func (g *Gnostic) readOpenAPIText(bytes []byte) (message proto.Message, err error) { 381 info, err := compiler.ReadInfoFromBytes(g.sourceName, bytes) 382 if err != nil { 383 return nil, err 384 } 385 // Determine the OpenAPI version. 386 g.sourceFormat = getOpenAPIVersionFromInfo(info) 387 if g.sourceFormat == SourceFormatUnknown { 388 return nil, errors.New("unable to identify OpenAPI version") 389 } 390 // Compile to the proto model. 391 if g.sourceFormat == SourceFormatOpenAPI2 { 392 document, err := openapi_v2.NewDocument(info, compiler.NewContextWithExtensions("$root", nil, &g.extensionHandlers)) 393 if err != nil { 394 return nil, err 395 } 396 message = document 397 } else if g.sourceFormat == SourceFormatOpenAPI3 { 398 document, err := openapi_v3.NewDocument(info, compiler.NewContextWithExtensions("$root", nil, &g.extensionHandlers)) 399 if err != nil { 400 return nil, err 401 } 402 message = document 403 } else { 404 document, err := discovery_v1.NewDocument(info, compiler.NewContextWithExtensions("$root", nil, &g.extensionHandlers)) 405 if err != nil { 406 return nil, err 407 } 408 message = document 409 } 410 return message, err 411} 412 413// Read an OpenAPI binary file. 414func (g *Gnostic) readOpenAPIBinary(data []byte) (message proto.Message, err error) { 415 // try to read an OpenAPI v3 document 416 documentV3 := &openapi_v3.Document{} 417 err = proto.Unmarshal(data, documentV3) 418 if err == nil && strings.HasPrefix(documentV3.Openapi, "3.0") { 419 g.sourceFormat = SourceFormatOpenAPI3 420 return documentV3, nil 421 } 422 // if that failed, try to read an OpenAPI v2 document 423 documentV2 := &openapi_v2.Document{} 424 err = proto.Unmarshal(data, documentV2) 425 if err == nil && strings.HasPrefix(documentV2.Swagger, "2.0") { 426 g.sourceFormat = SourceFormatOpenAPI2 427 return documentV2, nil 428 } 429 // if that failed, try to read a Discovery Format document 430 discoveryDocument := &discovery_v1.Document{} 431 err = proto.Unmarshal(data, discoveryDocument) 432 if err == nil { // && strings.HasPrefix(documentV2.Swagger, "2.0") { 433 g.sourceFormat = SourceFormatDiscovery 434 return discoveryDocument, nil 435 } 436 return nil, err 437} 438 439// Write a binary pb representation. 440func (g *Gnostic) writeBinaryOutput(message proto.Message) { 441 protoBytes, err := proto.Marshal(message) 442 if err != nil { 443 writeFile(g.errorOutputPath, g.errorBytes(err), g.sourceName, "errors") 444 defer os.Exit(-1) 445 } else { 446 writeFile(g.binaryOutputPath, protoBytes, g.sourceName, "pb") 447 } 448} 449 450// Write a text pb representation. 451func (g *Gnostic) writeTextOutput(message proto.Message) { 452 bytes := []byte(proto.MarshalTextString(message)) 453 writeFile(g.textOutputPath, bytes, g.sourceName, "text") 454} 455 456// Write JSON/YAML OpenAPI representations. 457func (g *Gnostic) writeJSONYAMLOutput(message proto.Message) { 458 // Convert the OpenAPI document into an exportable MapSlice. 459 var rawInfo yaml.MapSlice 460 var ok bool 461 var err error 462 if g.sourceFormat == SourceFormatOpenAPI2 { 463 document := message.(*openapi_v2.Document) 464 rawInfo, ok = document.ToRawInfo().(yaml.MapSlice) 465 if !ok { 466 rawInfo = nil 467 } 468 } else if g.sourceFormat == SourceFormatOpenAPI3 { 469 document := message.(*openapi_v3.Document) 470 rawInfo, ok = document.ToRawInfo().(yaml.MapSlice) 471 if !ok { 472 rawInfo = nil 473 } 474 } else if g.sourceFormat == SourceFormatDiscovery { 475 document := message.(*discovery_v1.Document) 476 rawInfo, ok = document.ToRawInfo().(yaml.MapSlice) 477 if !ok { 478 rawInfo = nil 479 } 480 } 481 // Optionally write description in yaml format. 482 if g.yamlOutputPath != "" { 483 var bytes []byte 484 if rawInfo != nil { 485 bytes, err = yaml.Marshal(rawInfo) 486 if err != nil { 487 fmt.Fprintf(os.Stderr, "Error generating yaml output %s\n", err.Error()) 488 } 489 writeFile(g.yamlOutputPath, bytes, g.sourceName, "yaml") 490 } else { 491 fmt.Fprintf(os.Stderr, "No yaml output available.\n") 492 } 493 } 494 // Optionally write description in json format. 495 if g.jsonOutputPath != "" { 496 var bytes []byte 497 if rawInfo != nil { 498 bytes, _ = jsonwriter.Marshal(rawInfo) 499 if err != nil { 500 fmt.Fprintf(os.Stderr, "Error generating json output %s\n", err.Error()) 501 } 502 writeFile(g.jsonOutputPath, bytes, g.sourceName, "json") 503 } else { 504 fmt.Fprintf(os.Stderr, "No json output available.\n") 505 } 506 } 507} 508 509// Write messages. 510func (g *Gnostic) writeMessagesOutput(message proto.Message) { 511 protoBytes, err := proto.Marshal(message) 512 if err != nil { 513 writeFile(g.messageOutputPath, g.errorBytes(err), g.sourceName, "errors") 514 defer os.Exit(-1) 515 } else { 516 writeFile(g.messageOutputPath, protoBytes, g.sourceName, "messages.pb") 517 } 518} 519 520// Perform all actions specified in the command-line options. 521func (g *Gnostic) performActions(message proto.Message) (err error) { 522 // Optionally resolve internal references. 523 if g.resolveReferences { 524 if g.sourceFormat == SourceFormatOpenAPI2 { 525 document := message.(*openapi_v2.Document) 526 _, err = document.ResolveReferences(g.sourceName) 527 } else if g.sourceFormat == SourceFormatOpenAPI3 { 528 document := message.(*openapi_v3.Document) 529 _, err = document.ResolveReferences(g.sourceName) 530 } 531 if err != nil { 532 return err 533 } 534 } 535 // Optionally write proto in binary format. 536 if g.binaryOutputPath != "" { 537 g.writeBinaryOutput(message) 538 } 539 // Optionally write proto in text format. 540 if g.textOutputPath != "" { 541 g.writeTextOutput(message) 542 } 543 // Optionally write document in yaml and/or json formats. 544 if g.yamlOutputPath != "" || g.jsonOutputPath != "" { 545 g.writeJSONYAMLOutput(message) 546 } 547 // Call all specified plugins. 548 messages := make([]*plugins.Message, 0) 549 for _, p := range g.pluginCalls { 550 pluginMessages, err := p.perform(message, g.sourceFormat, g.sourceName, g.timePlugins) 551 if err != nil { 552 writeFile(g.errorOutputPath, g.errorBytes(err), g.sourceName, "errors") 553 defer os.Exit(-1) // run all plugins, even when some have errors 554 } 555 messages = append(messages, pluginMessages...) 556 } 557 if g.messageOutputPath != "" { 558 g.writeMessagesOutput(&plugins.Messages{Messages: messages}) 559 } else { 560 // Print any messages from the plugins 561 if len(messages) > 0 { 562 for _, message := range messages { 563 fmt.Printf("%+v\n", message) 564 } 565 } 566 } 567 return nil 568} 569 570func (g *Gnostic) main() { 571 var err error 572 g.readOptions() 573 g.validateOptions() 574 // Read the OpenAPI source. 575 bytes, err := compiler.ReadBytesForFile(g.sourceName) 576 if err != nil { 577 writeFile(g.errorOutputPath, g.errorBytes(err), g.sourceName, "errors") 578 os.Exit(-1) 579 } 580 extension := strings.ToLower(filepath.Ext(g.sourceName)) 581 var message proto.Message 582 if extension == ".json" || extension == ".yaml" { 583 // Try to read the source as JSON/YAML. 584 message, err = g.readOpenAPIText(bytes) 585 if err != nil { 586 writeFile(g.errorOutputPath, g.errorBytes(err), g.sourceName, "errors") 587 os.Exit(-1) 588 } 589 } else if extension == ".pb" { 590 // Try to read the source as a binary protocol buffer. 591 message, err = g.readOpenAPIBinary(bytes) 592 if err != nil { 593 writeFile(g.errorOutputPath, g.errorBytes(err), g.sourceName, "errors") 594 os.Exit(-1) 595 } 596 } else { 597 err = errors.New("unknown file extension. 'json', 'yaml', and 'pb' are accepted") 598 writeFile(g.errorOutputPath, g.errorBytes(err), g.sourceName, "errors") 599 os.Exit(-1) 600 } 601 // Perform actions specified by command options. 602 err = g.performActions(message) 603 if err != nil { 604 writeFile(g.errorOutputPath, g.errorBytes(err), g.sourceName, "errors") 605 os.Exit(-1) 606 } 607} 608 609func main() { 610 g := newGnostic() 611 g.main() 612} 613