1// Copyright 2019 Google LLC 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package generator 16 17import ( 18 "context" 19 "encoding/json" 20 "fmt" 21 "log" 22 "os" 23 "path/filepath" 24 "strings" 25 26 "gopkg.in/yaml.v2" 27) 28 29// GapicGenerator is used to regenerate gapic libraries. 30type GapicGenerator struct { 31 googleapisDir string 32 protoDir string 33 googleCloudDir string 34 genprotoDir string 35 gapicToGenerate string 36} 37 38// NewGapicGenerator creates a GapicGenerator. 39func NewGapicGenerator(googleapisDir, protoDir, googleCloudDir, genprotoDir string, gapicToGenerate string) *GapicGenerator { 40 return &GapicGenerator{ 41 googleapisDir: googleapisDir, 42 protoDir: protoDir, 43 googleCloudDir: googleCloudDir, 44 genprotoDir: genprotoDir, 45 gapicToGenerate: gapicToGenerate, 46 } 47} 48 49// Regen generates gapics. 50func (g *GapicGenerator) Regen(ctx context.Context) error { 51 log.Println("regenerating gapics") 52 for _, c := range microgenGapicConfigs { 53 // Skip generation if generating all of the gapics and the associated 54 // config has a block on it. Or if generating a single gapic and it does 55 // not match the specified import path. 56 if (c.stopGeneration && g.gapicToGenerate == "") || 57 (g.gapicToGenerate != "" && g.gapicToGenerate != c.importPath) { 58 continue 59 } 60 if err := g.microgen(c); err != nil { 61 return err 62 } 63 } 64 65 if err := g.copyMicrogenFiles(); err != nil { 66 return err 67 } 68 69 if err := g.manifest(microgenGapicConfigs); err != nil { 70 return err 71 } 72 73 if err := g.setVersion(); err != nil { 74 return err 75 } 76 77 if err := g.addModReplaceGenproto(); err != nil { 78 return err 79 } 80 81 if err := vet(g.googleCloudDir); err != nil { 82 return err 83 } 84 85 if err := build(g.googleCloudDir); err != nil { 86 return err 87 } 88 89 if err := g.dropModReplaceGenproto(); err != nil { 90 return err 91 } 92 93 return nil 94} 95 96// addModReplaceGenproto adds a genproto replace statement that points genproto 97// to the local copy. This is necessary since the remote genproto may not have 98// changes that are necessary for the in-flight regen. 99func (g *GapicGenerator) addModReplaceGenproto() error { 100 log.Println("adding temporary genproto replace statement") 101 c := command("bash", "-c", ` 102set -ex 103 104GENPROTO_VERSION=$(cat go.mod | cat go.mod | grep genproto | awk '{print $2}') 105go mod edit -replace "google.golang.org/genproto@$GENPROTO_VERSION=$GENPROTO_DIR" 106`) 107 c.Dir = g.googleCloudDir 108 c.Env = []string{ 109 "GENPROTO_DIR=" + g.genprotoDir, 110 fmt.Sprintf("PATH=%s", os.Getenv("PATH")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands. 111 fmt.Sprintf("HOME=%s", os.Getenv("HOME")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands. 112 } 113 return c.Run() 114} 115 116// dropModReplaceGenproto drops the genproto replace statement. It is intended 117// to be run after addModReplaceGenproto. 118func (g *GapicGenerator) dropModReplaceGenproto() error { 119 log.Println("removing genproto replace statement") 120 c := command("bash", "-c", ` 121set -ex 122 123GENPROTO_VERSION=$(cat go.mod | cat go.mod | grep genproto | grep -v replace | awk '{print $2}') 124go mod edit -dropreplace "google.golang.org/genproto@$GENPROTO_VERSION" 125`) 126 c.Dir = g.googleCloudDir 127 c.Env = []string{ 128 fmt.Sprintf("PATH=%s", os.Getenv("PATH")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands. 129 fmt.Sprintf("HOME=%s", os.Getenv("HOME")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands. 130 } 131 return c.Run() 132} 133 134// setVersion updates the versionClient constant in all .go files. It may create 135// .backup files on certain systems (darwin), and so should be followed by a 136// clean-up of .backup files. 137func (g *GapicGenerator) setVersion() error { 138 log.Println("updating client version") 139 // TODO(deklerk): Migrate this all to Go instead of using bash. 140 141 c := command("bash", "-c", ` 142ver=$(date +%Y%m%d) 143git ls-files -mo | while read modified; do 144 dir=${modified%/*.*} 145 find . -path "*/$dir/doc.go" -exec sed -i.backup -e "s/^const versionClient.*/const versionClient = \"$ver\"/" '{}' +; 146done 147find . -name '*.backup' -delete 148`) 149 c.Dir = g.googleCloudDir 150 return c.Run() 151} 152 153// microgen runs the microgenerator on a single microgen config. 154func (g *GapicGenerator) microgen(conf *microgenConfig) error { 155 log.Println("microgen generating", conf.pkg) 156 157 var protoFiles []string 158 if err := filepath.Walk(g.googleapisDir+"/"+conf.inputDirectoryPath, func(path string, info os.FileInfo, err error) error { 159 if err != nil { 160 return err 161 } 162 if strings.Contains(info.Name(), ".proto") { 163 protoFiles = append(protoFiles, path) 164 } 165 return nil 166 }); err != nil { 167 return err 168 } 169 170 args := []string{"-I", g.googleapisDir, 171 "--experimental_allow_proto3_optional", 172 "-I", g.protoDir, 173 "--go_gapic_out", g.googleCloudDir, 174 "--go_gapic_opt", fmt.Sprintf("go-gapic-package=%s;%s", conf.importPath, conf.pkg), 175 "--go_gapic_opt", fmt.Sprintf("gapic-service-config=%s", conf.apiServiceConfigPath), 176 "--go_gapic_opt", fmt.Sprintf("release-level=%s", conf.releaseLevel)} 177 178 if conf.gRPCServiceConfigPath != "" { 179 args = append(args, "--go_gapic_opt", fmt.Sprintf("grpc-service-config=%s", conf.gRPCServiceConfigPath)) 180 } 181 if !conf.disableMetadata { 182 args = append(args, "--go_gapic_opt", "metadata") 183 } 184 args = append(args, protoFiles...) 185 c := command("protoc", args...) 186 c.Dir = g.googleapisDir 187 return c.Run() 188} 189 190// manifestEntry is used for JSON marshaling in manifest. 191type manifestEntry struct { 192 DistributionName string `json:"distribution_name"` 193 Description string `json:"description"` 194 Language string `json:"language"` 195 ClientLibraryType string `json:"client_library_type"` 196 DocsURL string `json:"docs_url"` 197 ReleaseLevel string `json:"release_level"` 198} 199 200// TODO: consider getting Description from the gapic, if there is one. 201var manualEntries = []manifestEntry{ 202 // Pure manual clients. 203 { 204 DistributionName: "cloud.google.com/go/bigquery", 205 Description: "BigQuery", 206 Language: "Go", 207 ClientLibraryType: "manual", 208 DocsURL: "https://pkg.go.dev/cloud.google.com/go/bigquery", 209 ReleaseLevel: "ga", 210 }, 211 { 212 DistributionName: "cloud.google.com/go/bigtable", 213 Description: "Cloud BigTable", 214 Language: "Go", 215 ClientLibraryType: "manual", 216 DocsURL: "https://pkg.go.dev/cloud.google.com/go/bigtable", 217 ReleaseLevel: "ga", 218 }, 219 { 220 DistributionName: "cloud.google.com/go/datastore", 221 Description: "Cloud Datastore", 222 Language: "Go", 223 ClientLibraryType: "manual", 224 DocsURL: "https://pkg.go.dev/cloud.google.com/go/datastore", 225 ReleaseLevel: "ga", 226 }, 227 { 228 DistributionName: "cloud.google.com/go/iam", 229 Description: "Cloud IAM", 230 Language: "Go", 231 ClientLibraryType: "manual", 232 DocsURL: "https://pkg.go.dev/cloud.google.com/go/iam", 233 ReleaseLevel: "ga", 234 }, 235 { 236 DistributionName: "cloud.google.com/go/storage", 237 Description: "Cloud Storage (GCS)", 238 Language: "Go", 239 ClientLibraryType: "manual", 240 DocsURL: "https://pkg.go.dev/cloud.google.com/go/storage", 241 ReleaseLevel: "ga", 242 }, 243 { 244 DistributionName: "cloud.google.com/go/rpcreplay", 245 Description: "RPC Replay", 246 Language: "Go", 247 ClientLibraryType: "manual", 248 DocsURL: "https://pkg.go.dev/cloud.google.com/go/rpcreplay", 249 ReleaseLevel: "ga", 250 }, 251 { 252 DistributionName: "cloud.google.com/go/profiler", 253 Description: "Cloud Profiler", 254 Language: "Go", 255 ClientLibraryType: "manual", 256 DocsURL: "https://pkg.go.dev/cloud.google.com/go/profiler", 257 ReleaseLevel: "ga", 258 }, 259 // Manuals with a GAPIC. 260 { 261 DistributionName: "cloud.google.com/go/errorreporting", 262 Description: "Cloud Error Reporting API", 263 Language: "Go", 264 ClientLibraryType: "manual", 265 DocsURL: "https://pkg.go.dev/cloud.google.com/go/errorreporting", 266 ReleaseLevel: "beta", 267 }, 268 { 269 DistributionName: "cloud.google.com/go/firestore", 270 Description: "Cloud Firestore API", 271 Language: "Go", 272 ClientLibraryType: "manual", 273 DocsURL: "https://pkg.go.dev/cloud.google.com/go/firestore", 274 ReleaseLevel: "ga", 275 }, 276 { 277 DistributionName: "cloud.google.com/go/logging", 278 Description: "Cloud Logging API", 279 Language: "Go", 280 ClientLibraryType: "manual", 281 DocsURL: "https://pkg.go.dev/cloud.google.com/go/logging", 282 ReleaseLevel: "ga", 283 }, 284 { 285 DistributionName: "cloud.google.com/go/pubsub", 286 Description: "Cloud PubSub", 287 Language: "Go", 288 ClientLibraryType: "manual", 289 DocsURL: "https://pkg.go.dev/cloud.google.com/go/pubsub", 290 ReleaseLevel: "ga", 291 }, 292 { 293 DistributionName: "cloud.google.com/go/spanner", 294 Description: "Cloud Spanner", 295 Language: "Go", 296 ClientLibraryType: "manual", 297 DocsURL: "https://pkg.go.dev/cloud.google.com/go/spanner", 298 ReleaseLevel: "ga", 299 }, 300} 301 302// manifest writes a manifest file with info about all of the confs. 303func (g *GapicGenerator) manifest(confs []*microgenConfig) error { 304 log.Println("updating gapic manifest") 305 entries := map[string]manifestEntry{} // Key is the package name. 306 f, err := os.Create(filepath.Join(g.googleCloudDir, "internal", ".repo-metadata-full.json")) 307 if err != nil { 308 return err 309 } 310 defer f.Close() 311 for _, manual := range manualEntries { 312 entries[manual.DistributionName] = manual 313 } 314 for _, conf := range confs { 315 yamlPath := filepath.Join(g.googleapisDir, conf.apiServiceConfigPath) 316 yamlFile, err := os.Open(yamlPath) 317 if err != nil { 318 return err 319 } 320 yamlConfig := struct { 321 Title string `yaml:"title"` // We only need the title field. 322 }{} 323 if err := yaml.NewDecoder(yamlFile).Decode(&yamlConfig); err != nil { 324 return fmt.Errorf("Decode: %v", err) 325 } 326 entry := manifestEntry{ 327 DistributionName: conf.importPath, 328 Description: yamlConfig.Title, 329 Language: "Go", 330 ClientLibraryType: "generated", 331 DocsURL: "https://pkg.go.dev/" + conf.importPath, 332 ReleaseLevel: conf.releaseLevel, 333 } 334 entries[conf.importPath] = entry 335 } 336 enc := json.NewEncoder(f) 337 enc.SetIndent("", " ") 338 return enc.Encode(entries) 339} 340 341// copyMicrogenFiles takes microgen files from gocloudDir/cloud.google.com/go 342// and places them in gocloudDir. 343func (g *GapicGenerator) copyMicrogenFiles() error { 344 // The period at the end is analagous to * (copy everything in this dir). 345 c := command("cp", "-R", g.googleCloudDir+"/cloud.google.com/go/.", ".") 346 c.Dir = g.googleCloudDir 347 if err := c.Run(); err != nil { 348 return err 349 } 350 351 c = command("rm", "-rf", "cloud.google.com") 352 c.Dir = g.googleCloudDir 353 return c.Run() 354} 355