1// Copyright 2019 Google LLC 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package generator 16 17import ( 18 "context" 19 "errors" 20 "fmt" 21 "io/ioutil" 22 "log" 23 "path/filepath" 24 "regexp" 25 "strconv" 26 "strings" 27 28 "golang.org/x/sync/errgroup" 29) 30 31var goPkgOptRe = regexp.MustCompile(`(?m)^option go_package = (.*);`) 32 33// denylist is a set of clients to NOT generate. 34var denylist = map[string]bool{ 35 // TODO(codyoss): re-enable after issue is resolve -- https://github.com/googleapis/go-genproto/issues/357 36 "google.golang.org/genproto/googleapis/cloud/recommendationengine/v1beta1": true, 37 38 // These two container APIs are currently frozen. They should not be updated 39 // due to manual layer built on top of them. 40 "google.golang.org/genproto/googleapis/grafeas/v1": true, 41 "google.golang.org/genproto/googleapis/devtools/containeranalysis/v1": true, 42} 43 44// GenprotoGenerator is used to generate code for googleapis/go-genproto. 45type GenprotoGenerator struct { 46 genprotoDir string 47 googleapisDir string 48 protoSrcDir string 49} 50 51// NewGenprotoGenerator creates a new GenprotoGenerator. 52func NewGenprotoGenerator(genprotoDir, googleapisDir, protoDir string) *GenprotoGenerator { 53 return &GenprotoGenerator{ 54 genprotoDir: genprotoDir, 55 googleapisDir: googleapisDir, 56 protoSrcDir: filepath.Join(protoDir, "/src"), 57 } 58} 59 60var skipPrefixes = []string{ 61 "google.golang.org/genproto/googleapis/ads", 62} 63 64func hasPrefix(s string, prefixes []string) bool { 65 for _, prefix := range prefixes { 66 if strings.HasPrefix(s, prefix) { 67 return true 68 } 69 } 70 return false 71} 72 73// Regen regenerates the genproto repository. 74// regenGenproto regenerates the genproto repository. 75// 76// regenGenproto recursively walks through each directory named by given 77// arguments, looking for all .proto files. (Symlinks are not followed.) Any 78// proto file without `go_package` option or whose option does not begin with 79// the genproto prefix is ignored. 80// 81// If multiple roots contain files with the same name, eg "root1/path/to/file" 82// and "root2/path/to/file", only the first file is processed; the rest are 83// ignored. 84// 85// Protoc is executed on remaining files, one invocation per set of files 86// declaring the same Go package. 87func (g *GenprotoGenerator) Regen(ctx context.Context) error { 88 log.Println("regenerating genproto") 89 90 // Create space to put generated .pb.go's. 91 c := command("mkdir", "generated") 92 c.Dir = g.genprotoDir 93 if err := c.Run(); err != nil { 94 return err 95 } 96 97 // Get the last processed googleapis hash. 98 lastHash, err := ioutil.ReadFile(filepath.Join(g.genprotoDir, "regen.txt")) 99 if err != nil { 100 return err 101 } 102 103 pkgFiles, err := g.getUpdatedPackages(string(lastHash)) 104 if err != nil { 105 return err 106 } 107 if len(pkgFiles) == 0 { 108 return errors.New("couldn't find any pkgfiles") 109 } 110 111 log.Println("generating from protos") 112 grp, _ := errgroup.WithContext(ctx) 113 for pkg, fileNames := range pkgFiles { 114 if !strings.HasPrefix(pkg, "google.golang.org/genproto") || denylist[pkg] || hasPrefix(pkg, skipPrefixes) { 115 continue 116 } 117 pk := pkg 118 fn := fileNames 119 grp.Go(func() error { 120 log.Println("running protoc on", pk) 121 return g.protoc(fn) 122 }) 123 } 124 if err := grp.Wait(); err != nil { 125 return err 126 } 127 128 if err := g.moveAndCleanupGeneratedSrc(); err != nil { 129 return err 130 } 131 132 if err := vet(g.genprotoDir); err != nil { 133 return err 134 } 135 136 if err := build(g.genprotoDir); err != nil { 137 return err 138 } 139 140 return nil 141} 142 143// goPkg reports the import path declared in the given file's `go_package` 144// option. If the option is missing, goPkg returns empty string. 145func goPkg(fileName string) (string, error) { 146 content, err := ioutil.ReadFile(fileName) 147 if err != nil { 148 return "", err 149 } 150 151 var pkgName string 152 if match := goPkgOptRe.FindSubmatch(content); len(match) > 0 { 153 pn, err := strconv.Unquote(string(match[1])) 154 if err != nil { 155 return "", err 156 } 157 pkgName = pn 158 } 159 if p := strings.IndexRune(pkgName, ';'); p > 0 { 160 pkgName = pkgName[:p] 161 } 162 return pkgName, nil 163} 164 165// protoc executes the "protoc" command on files named in fileNames, and outputs 166// to "<genprotoDir>/generated". 167func (g *GenprotoGenerator) protoc(fileNames []string) error { 168 args := []string{"--experimental_allow_proto3_optional", fmt.Sprintf("--go_out=plugins=grpc:%s/generated", g.genprotoDir), "-I", g.googleapisDir, "-I", g.protoSrcDir} 169 args = append(args, fileNames...) 170 c := command("protoc", args...) 171 c.Dir = g.genprotoDir 172 return c.Run() 173} 174 175// getUpdatedPackages parses all of the new commits to find what packages need 176// to be regenerated. 177func (g *GenprotoGenerator) getUpdatedPackages(googleapisHash string) (map[string][]string, error) { 178 files, err := UpdateFilesSinceHash(g.googleapisDir, googleapisHash) 179 if err != nil { 180 181 } 182 pkgFiles := make(map[string][]string) 183 for _, v := range files { 184 if !strings.HasSuffix(v, ".proto") { 185 continue 186 } 187 path := filepath.Join(g.googleapisDir, v) 188 pkg, err := goPkg(path) 189 if err != nil { 190 return nil, err 191 } 192 pkgFiles[pkg] = append(pkgFiles[pkg], path) 193 } 194 return pkgFiles, nil 195} 196 197// moveAndCleanupGeneratedSrc moves all generated src to their correct locations 198// in the repository, because protoc puts it in a folder called `generated/``. 199func (g *GenprotoGenerator) moveAndCleanupGeneratedSrc() error { 200 log.Println("moving generated code") 201 // The period at the end is analogous to * (copy everything in this dir). 202 c := command("cp", "-R", filepath.Join(g.genprotoDir, "generated", "google.golang.org", "genproto", "googleapis"), g.genprotoDir) 203 if err := c.Run(); err != nil { 204 return err 205 } 206 207 c = command("rm", "-rf", "generated") 208 c.Dir = g.genprotoDir 209 if err := c.Run(); err != nil { 210 return err 211 } 212 213 return nil 214} 215