1// Copyright 2019 Google LLC
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//      http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package generator
16
17import (
18	"context"
19	"encoding/json"
20	"fmt"
21	"log"
22	"os"
23	"path/filepath"
24	"strings"
25
26	"gopkg.in/yaml.v2"
27)
28
29// GapicGenerator is used to regenerate gapic libraries.
30type GapicGenerator struct {
31	googleapisDir   string
32	protoDir        string
33	googleCloudDir  string
34	genprotoDir     string
35	gapicToGenerate string
36}
37
38// NewGapicGenerator creates a GapicGenerator.
39func NewGapicGenerator(googleapisDir, protoDir, googleCloudDir, genprotoDir string, gapicToGenerate string) *GapicGenerator {
40	return &GapicGenerator{
41		googleapisDir:   googleapisDir,
42		protoDir:        protoDir,
43		googleCloudDir:  googleCloudDir,
44		genprotoDir:     genprotoDir,
45		gapicToGenerate: gapicToGenerate,
46	}
47}
48
49// Regen generates gapics.
50func (g *GapicGenerator) Regen(ctx context.Context) error {
51	log.Println("regenerating gapics")
52	for _, c := range microgenGapicConfigs {
53		// Skip generation if generating all of the gapics and the associated
54		// config has a block on it. Or if generating a single gapic and it does
55		// not match the specified import path.
56		if (c.stopGeneration && g.gapicToGenerate == "") ||
57			(g.gapicToGenerate != "" && g.gapicToGenerate != c.importPath) {
58			continue
59		}
60		if err := g.microgen(c); err != nil {
61			return err
62		}
63	}
64
65	if err := g.copyMicrogenFiles(); err != nil {
66		return err
67	}
68
69	if err := g.manifest(microgenGapicConfigs); err != nil {
70		return err
71	}
72
73	if err := g.setVersion(); err != nil {
74		return err
75	}
76
77	if err := g.addModReplaceGenproto(); err != nil {
78		return err
79	}
80
81	if err := vet(g.googleCloudDir); err != nil {
82		return err
83	}
84
85	if err := build(g.googleCloudDir); err != nil {
86		return err
87	}
88
89	if err := g.dropModReplaceGenproto(); err != nil {
90		return err
91	}
92
93	return nil
94}
95
96// addModReplaceGenproto adds a genproto replace statement that points genproto
97// to the local copy. This is necessary since the remote genproto may not have
98// changes that are necessary for the in-flight regen.
99func (g *GapicGenerator) addModReplaceGenproto() error {
100	log.Println("adding temporary genproto replace statement")
101	c := command("bash", "-c", `
102set -ex
103
104GENPROTO_VERSION=$(cat go.mod | cat go.mod | grep genproto | awk '{print $2}')
105go mod edit -replace "google.golang.org/genproto@$GENPROTO_VERSION=$GENPROTO_DIR"
106`)
107	c.Dir = g.googleCloudDir
108	c.Env = []string{
109		"GENPROTO_DIR=" + g.genprotoDir,
110		fmt.Sprintf("PATH=%s", os.Getenv("PATH")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
111		fmt.Sprintf("HOME=%s", os.Getenv("HOME")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
112	}
113	return c.Run()
114}
115
116// dropModReplaceGenproto drops the genproto replace statement. It is intended
117// to be run after addModReplaceGenproto.
118func (g *GapicGenerator) dropModReplaceGenproto() error {
119	log.Println("removing genproto replace statement")
120	c := command("bash", "-c", `
121set -ex
122
123GENPROTO_VERSION=$(cat go.mod | cat go.mod | grep genproto | grep -v replace | awk '{print $2}')
124go mod edit -dropreplace "google.golang.org/genproto@$GENPROTO_VERSION"
125`)
126	c.Dir = g.googleCloudDir
127	c.Env = []string{
128		fmt.Sprintf("PATH=%s", os.Getenv("PATH")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
129		fmt.Sprintf("HOME=%s", os.Getenv("HOME")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
130	}
131	return c.Run()
132}
133
134// setVersion updates the versionClient constant in all .go files. It may create
135// .backup files on certain systems (darwin), and so should be followed by a
136// clean-up of .backup files.
137func (g *GapicGenerator) setVersion() error {
138	log.Println("updating client version")
139	// TODO(deklerk): Migrate this all to Go instead of using bash.
140
141	c := command("bash", "-c", `
142ver=$(date +%Y%m%d)
143git ls-files -mo | while read modified; do
144	dir=${modified%/*.*}
145	find . -path "*/$dir/doc.go" -exec sed -i.backup -e "s/^const versionClient.*/const versionClient = \"$ver\"/" '{}' +;
146done
147find . -name '*.backup' -delete
148`)
149	c.Dir = g.googleCloudDir
150	return c.Run()
151}
152
153// microgen runs the microgenerator on a single microgen config.
154func (g *GapicGenerator) microgen(conf *microgenConfig) error {
155	log.Println("microgen generating", conf.pkg)
156
157	var protoFiles []string
158	if err := filepath.Walk(g.googleapisDir+"/"+conf.inputDirectoryPath, func(path string, info os.FileInfo, err error) error {
159		if err != nil {
160			return err
161		}
162		if strings.Contains(info.Name(), ".proto") {
163			protoFiles = append(protoFiles, path)
164		}
165		return nil
166	}); err != nil {
167		return err
168	}
169
170	args := []string{"-I", g.googleapisDir,
171		"--experimental_allow_proto3_optional",
172		"-I", g.protoDir,
173		"--go_gapic_out", g.googleCloudDir,
174		"--go_gapic_opt", fmt.Sprintf("go-gapic-package=%s;%s", conf.importPath, conf.pkg),
175		"--go_gapic_opt", fmt.Sprintf("gapic-service-config=%s", conf.apiServiceConfigPath),
176		"--go_gapic_opt", fmt.Sprintf("release-level=%s", conf.releaseLevel)}
177
178	if conf.gRPCServiceConfigPath != "" {
179		args = append(args, "--go_gapic_opt", fmt.Sprintf("grpc-service-config=%s", conf.gRPCServiceConfigPath))
180	}
181	if !conf.disableMetadata {
182		args = append(args, "--go_gapic_opt", "metadata")
183	}
184	args = append(args, protoFiles...)
185	c := command("protoc", args...)
186	c.Dir = g.googleapisDir
187	return c.Run()
188}
189
190// manifestEntry is used for JSON marshaling in manifest.
191type manifestEntry struct {
192	DistributionName  string `json:"distribution_name"`
193	Description       string `json:"description"`
194	Language          string `json:"language"`
195	ClientLibraryType string `json:"client_library_type"`
196	DocsURL           string `json:"docs_url"`
197	ReleaseLevel      string `json:"release_level"`
198}
199
200// TODO: consider getting Description from the gapic, if there is one.
201var manualEntries = []manifestEntry{
202	// Pure manual clients.
203	{
204		DistributionName:  "cloud.google.com/go/bigquery",
205		Description:       "BigQuery",
206		Language:          "Go",
207		ClientLibraryType: "manual",
208		DocsURL:           "https://pkg.go.dev/cloud.google.com/go/bigquery",
209		ReleaseLevel:      "ga",
210	},
211	{
212		DistributionName:  "cloud.google.com/go/bigtable",
213		Description:       "Cloud BigTable",
214		Language:          "Go",
215		ClientLibraryType: "manual",
216		DocsURL:           "https://pkg.go.dev/cloud.google.com/go/bigtable",
217		ReleaseLevel:      "ga",
218	},
219	{
220		DistributionName:  "cloud.google.com/go/datastore",
221		Description:       "Cloud Datastore",
222		Language:          "Go",
223		ClientLibraryType: "manual",
224		DocsURL:           "https://pkg.go.dev/cloud.google.com/go/datastore",
225		ReleaseLevel:      "ga",
226	},
227	{
228		DistributionName:  "cloud.google.com/go/iam",
229		Description:       "Cloud IAM",
230		Language:          "Go",
231		ClientLibraryType: "manual",
232		DocsURL:           "https://pkg.go.dev/cloud.google.com/go/iam",
233		ReleaseLevel:      "ga",
234	},
235	{
236		DistributionName:  "cloud.google.com/go/storage",
237		Description:       "Cloud Storage (GCS)",
238		Language:          "Go",
239		ClientLibraryType: "manual",
240		DocsURL:           "https://pkg.go.dev/cloud.google.com/go/storage",
241		ReleaseLevel:      "ga",
242	},
243	{
244		DistributionName:  "cloud.google.com/go/rpcreplay",
245		Description:       "RPC Replay",
246		Language:          "Go",
247		ClientLibraryType: "manual",
248		DocsURL:           "https://pkg.go.dev/cloud.google.com/go/rpcreplay",
249		ReleaseLevel:      "ga",
250	},
251	{
252		DistributionName:  "cloud.google.com/go/profiler",
253		Description:       "Cloud Profiler",
254		Language:          "Go",
255		ClientLibraryType: "manual",
256		DocsURL:           "https://pkg.go.dev/cloud.google.com/go/profiler",
257		ReleaseLevel:      "ga",
258	},
259	// Manuals with a GAPIC.
260	{
261		DistributionName:  "cloud.google.com/go/errorreporting",
262		Description:       "Cloud Error Reporting API",
263		Language:          "Go",
264		ClientLibraryType: "manual",
265		DocsURL:           "https://pkg.go.dev/cloud.google.com/go/errorreporting",
266		ReleaseLevel:      "beta",
267	},
268	{
269		DistributionName:  "cloud.google.com/go/firestore",
270		Description:       "Cloud Firestore API",
271		Language:          "Go",
272		ClientLibraryType: "manual",
273		DocsURL:           "https://pkg.go.dev/cloud.google.com/go/firestore",
274		ReleaseLevel:      "ga",
275	},
276	{
277		DistributionName:  "cloud.google.com/go/logging",
278		Description:       "Cloud Logging API",
279		Language:          "Go",
280		ClientLibraryType: "manual",
281		DocsURL:           "https://pkg.go.dev/cloud.google.com/go/logging",
282		ReleaseLevel:      "ga",
283	},
284	{
285		DistributionName:  "cloud.google.com/go/pubsub",
286		Description:       "Cloud PubSub",
287		Language:          "Go",
288		ClientLibraryType: "manual",
289		DocsURL:           "https://pkg.go.dev/cloud.google.com/go/pubsub",
290		ReleaseLevel:      "ga",
291	},
292	{
293		DistributionName:  "cloud.google.com/go/spanner",
294		Description:       "Cloud Spanner",
295		Language:          "Go",
296		ClientLibraryType: "manual",
297		DocsURL:           "https://pkg.go.dev/cloud.google.com/go/spanner",
298		ReleaseLevel:      "ga",
299	},
300}
301
302// manifest writes a manifest file with info about all of the confs.
303func (g *GapicGenerator) manifest(confs []*microgenConfig) error {
304	log.Println("updating gapic manifest")
305	entries := map[string]manifestEntry{} // Key is the package name.
306	f, err := os.Create(filepath.Join(g.googleCloudDir, "internal", ".repo-metadata-full.json"))
307	if err != nil {
308		return err
309	}
310	defer f.Close()
311	for _, manual := range manualEntries {
312		entries[manual.DistributionName] = manual
313	}
314	for _, conf := range confs {
315		yamlPath := filepath.Join(g.googleapisDir, conf.apiServiceConfigPath)
316		yamlFile, err := os.Open(yamlPath)
317		if err != nil {
318			return err
319		}
320		yamlConfig := struct {
321			Title string `yaml:"title"` // We only need the title field.
322		}{}
323		if err := yaml.NewDecoder(yamlFile).Decode(&yamlConfig); err != nil {
324			return fmt.Errorf("Decode: %v", err)
325		}
326		entry := manifestEntry{
327			DistributionName:  conf.importPath,
328			Description:       yamlConfig.Title,
329			Language:          "Go",
330			ClientLibraryType: "generated",
331			DocsURL:           "https://pkg.go.dev/" + conf.importPath,
332			ReleaseLevel:      conf.releaseLevel,
333		}
334		entries[conf.importPath] = entry
335	}
336	enc := json.NewEncoder(f)
337	enc.SetIndent("", "  ")
338	return enc.Encode(entries)
339}
340
341// copyMicrogenFiles takes microgen files from gocloudDir/cloud.google.com/go
342// and places them in gocloudDir.
343func (g *GapicGenerator) copyMicrogenFiles() error {
344	// The period at the end is analagous to * (copy everything in this dir).
345	c := command("cp", "-R", g.googleCloudDir+"/cloud.google.com/go/.", ".")
346	c.Dir = g.googleCloudDir
347	if err := c.Run(); err != nil {
348		return err
349	}
350
351	c = command("rm", "-rf", "cloud.google.com")
352	c.Dir = g.googleCloudDir
353	return c.Run()
354}
355