1// Copyright 2019 Google LLC
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//      http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package generator
16
17import (
18	"context"
19	"errors"
20	"fmt"
21	"io/ioutil"
22	"log"
23	"path/filepath"
24	"regexp"
25	"strconv"
26	"strings"
27
28	"golang.org/x/sync/errgroup"
29)
30
31var goPkgOptRe = regexp.MustCompile(`(?m)^option go_package = (.*);`)
32
33// denylist is a set of clients to NOT generate.
34var denylist = map[string]bool{
35	// TODO(codyoss): re-enable after issue is resolve -- https://github.com/googleapis/go-genproto/issues/357
36	"google.golang.org/genproto/googleapis/cloud/recommendationengine/v1beta1": true,
37
38	// These two container APIs are currently frozen. They should not be updated
39	// due to manual layer built on top of them.
40	"google.golang.org/genproto/googleapis/grafeas/v1":                    true,
41	"google.golang.org/genproto/googleapis/devtools/containeranalysis/v1": true,
42}
43
44// GenprotoGenerator is used to generate code for googleapis/go-genproto.
45type GenprotoGenerator struct {
46	genprotoDir   string
47	googleapisDir string
48	protoSrcDir   string
49}
50
51// NewGenprotoGenerator creates a new GenprotoGenerator.
52func NewGenprotoGenerator(genprotoDir, googleapisDir, protoDir string) *GenprotoGenerator {
53	return &GenprotoGenerator{
54		genprotoDir:   genprotoDir,
55		googleapisDir: googleapisDir,
56		protoSrcDir:   filepath.Join(protoDir, "/src"),
57	}
58}
59
60var skipPrefixes = []string{
61	"google.golang.org/genproto/googleapis/ads",
62}
63
64func hasPrefix(s string, prefixes []string) bool {
65	for _, prefix := range prefixes {
66		if strings.HasPrefix(s, prefix) {
67			return true
68		}
69	}
70	return false
71}
72
73// Regen regenerates the genproto repository.
74// regenGenproto regenerates the genproto repository.
75//
76// regenGenproto recursively walks through each directory named by given
77// arguments, looking for all .proto files. (Symlinks are not followed.) Any
78// proto file without `go_package` option or whose option does not begin with
79// the genproto prefix is ignored.
80//
81// If multiple roots contain files with the same name, eg "root1/path/to/file"
82// and "root2/path/to/file", only the first file is processed; the rest are
83// ignored.
84//
85// Protoc is executed on remaining files, one invocation per set of files
86// declaring the same Go package.
87func (g *GenprotoGenerator) Regen(ctx context.Context) error {
88	log.Println("regenerating genproto")
89
90	// Create space to put generated .pb.go's.
91	c := command("mkdir", "generated")
92	c.Dir = g.genprotoDir
93	if err := c.Run(); err != nil {
94		return err
95	}
96
97	// Get the last processed googleapis hash.
98	lastHash, err := ioutil.ReadFile(filepath.Join(g.genprotoDir, "regen.txt"))
99	if err != nil {
100		return err
101	}
102
103	pkgFiles, err := g.getUpdatedPackages(string(lastHash))
104	if err != nil {
105		return err
106	}
107	if len(pkgFiles) == 0 {
108		return errors.New("couldn't find any pkgfiles")
109	}
110
111	log.Println("generating from protos")
112	grp, _ := errgroup.WithContext(ctx)
113	for pkg, fileNames := range pkgFiles {
114		if !strings.HasPrefix(pkg, "google.golang.org/genproto") || denylist[pkg] || hasPrefix(pkg, skipPrefixes) {
115			continue
116		}
117		pk := pkg
118		fn := fileNames
119		grp.Go(func() error {
120			log.Println("running protoc on", pk)
121			return g.protoc(fn)
122		})
123	}
124	if err := grp.Wait(); err != nil {
125		return err
126	}
127
128	if err := g.moveAndCleanupGeneratedSrc(); err != nil {
129		return err
130	}
131
132	if err := vet(g.genprotoDir); err != nil {
133		return err
134	}
135
136	if err := build(g.genprotoDir); err != nil {
137		return err
138	}
139
140	return nil
141}
142
143// goPkg reports the import path declared in the given file's `go_package`
144// option. If the option is missing, goPkg returns empty string.
145func goPkg(fileName string) (string, error) {
146	content, err := ioutil.ReadFile(fileName)
147	if err != nil {
148		return "", err
149	}
150
151	var pkgName string
152	if match := goPkgOptRe.FindSubmatch(content); len(match) > 0 {
153		pn, err := strconv.Unquote(string(match[1]))
154		if err != nil {
155			return "", err
156		}
157		pkgName = pn
158	}
159	if p := strings.IndexRune(pkgName, ';'); p > 0 {
160		pkgName = pkgName[:p]
161	}
162	return pkgName, nil
163}
164
165// protoc executes the "protoc" command on files named in fileNames, and outputs
166// to "<genprotoDir>/generated".
167func (g *GenprotoGenerator) protoc(fileNames []string) error {
168	args := []string{"--experimental_allow_proto3_optional", fmt.Sprintf("--go_out=plugins=grpc:%s/generated", g.genprotoDir), "-I", g.googleapisDir, "-I", g.protoSrcDir}
169	args = append(args, fileNames...)
170	c := command("protoc", args...)
171	c.Dir = g.genprotoDir
172	return c.Run()
173}
174
175// getUpdatedPackages parses all of the new commits to find what packages need
176// to be regenerated.
177func (g *GenprotoGenerator) getUpdatedPackages(googleapisHash string) (map[string][]string, error) {
178	files, err := UpdateFilesSinceHash(g.googleapisDir, googleapisHash)
179	if err != nil {
180
181	}
182	pkgFiles := make(map[string][]string)
183	for _, v := range files {
184		if !strings.HasSuffix(v, ".proto") {
185			continue
186		}
187		path := filepath.Join(g.googleapisDir, v)
188		pkg, err := goPkg(path)
189		if err != nil {
190			return nil, err
191		}
192		pkgFiles[pkg] = append(pkgFiles[pkg], path)
193	}
194	return pkgFiles, nil
195}
196
197// moveAndCleanupGeneratedSrc moves all generated src to their correct locations
198// in the repository, because protoc puts it in a folder called `generated/``.
199func (g *GenprotoGenerator) moveAndCleanupGeneratedSrc() error {
200	log.Println("moving generated code")
201	// The period at the end is analogous to * (copy everything in this dir).
202	c := command("cp", "-R", filepath.Join(g.genprotoDir, "generated", "google.golang.org", "genproto", "googleapis"), g.genprotoDir)
203	if err := c.Run(); err != nil {
204		return err
205	}
206
207	c = command("rm", "-rf", "generated")
208	c.Dir = g.genprotoDir
209	if err := c.Run(); err != nil {
210		return err
211	}
212
213	return nil
214}
215