1// Copyright (c) Microsoft Corporation. All rights reserved.
2// Licensed under the MIT License. See License.txt in the project root for license information.
3
4package automation
5
6import (
7	"bufio"
8	"fmt"
9	"log"
10	"os"
11	"path/filepath"
12	"sort"
13	"strings"
14
15	"github.com/Azure/azure-sdk-for-go/tools/generator/autorest"
16	"github.com/Azure/azure-sdk-for-go/tools/generator/autorest/model"
17	"github.com/Azure/azure-sdk-for-go/tools/generator/cmd/automation/pipeline"
18	"github.com/Azure/azure-sdk-for-go/tools/generator/common"
19	"github.com/Azure/azure-sdk-for-go/tools/internal/exports"
20	"github.com/Azure/azure-sdk-for-go/tools/internal/packages/track1"
21	"github.com/Azure/azure-sdk-for-go/tools/internal/utils"
22	"github.com/spf13/cobra"
23)
24
25// Command returns the automation command. Note that this command is designed to run in the root directory of
26// azure-sdk-for-go. It does not work if you are running this tool in somewhere else
27func Command() *cobra.Command {
28	automationCmd := &cobra.Command{
29		Use:  "automation <generate input filepath> <generate output filepath>",
30		Args: cobra.ExactArgs(2),
31		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
32			log.SetFlags(0) // remove the time stamp prefix
33			return nil
34		},
35		RunE: func(cmd *cobra.Command, args []string) error {
36			optionPath, err := cmd.Flags().GetString("options")
37			if err != nil {
38				logError(err)
39				return err
40			}
41			if err := execute(args[0], args[1], Flags{
42				OptionPath: optionPath,
43			}); err != nil {
44				logError(err)
45				return err
46			}
47			return nil
48		},
49		SilenceUsage: true, // this command is used for a pipeline, the usage should never show
50	}
51
52	flags := automationCmd.Flags()
53	flags.String("options", common.DefaultOptionPath, "Specify a file with the autorest options")
54
55	return automationCmd
56}
57
58// Flags ...
59type Flags struct {
60	OptionPath string
61}
62
63func execute(inputPath, outputPath string, flags Flags) error {
64	log.Printf("Reading generate input file from '%s'...", inputPath)
65	input, err := pipeline.ReadInput(inputPath)
66	if err != nil {
67		return fmt.Errorf("cannot read generate input: %+v", err)
68	}
69	log.Printf("Generating using the following GenerateInput...\n%s", input.String())
70	cwd, err := os.Getwd()
71	if err != nil {
72		return err
73	}
74	log.Printf("Using current directory as SDK root: %s", cwd)
75
76	ctx := automationContext{
77		sdkRoot:    utils.NormalizePath(cwd),
78		specRoot:   input.SpecFolder,
79		commitHash: input.HeadSha,
80		optionPath: flags.OptionPath,
81	}
82	output, err := ctx.generate(input)
83	if err != nil {
84		return err
85	}
86	log.Printf("Output generated: \n%s", output.String())
87	log.Printf("Writing output to file '%s'...", outputPath)
88	if err := pipeline.WriteOutput(outputPath, output); err != nil {
89		return fmt.Errorf("cannot write generate output: %+v", err)
90	}
91	return nil
92}
93
94type automationContext struct {
95	sdkRoot    string
96	specRoot   string
97	commitHash string
98	optionPath string
99
100	repoContent map[string]exports.Content
101
102	sdkVersion string
103
104	existingPackages existingPackageMap
105
106	defaultOptions    model.Options
107	additionalOptions []model.Option
108}
109
110func (ctx *automationContext) categorizePackages() error {
111	ctx.existingPackages = existingPackageMap{}
112
113	serviceRoot := filepath.Join(ctx.sdkRoot, "services")
114	m, err := autorest.CollectGenerationMetadata(serviceRoot)
115	if err != nil {
116		return err
117	}
118
119	for path, metadata := range m {
120		// the path in the metadata map is the absolute path
121		relPath, err := filepath.Rel(ctx.sdkRoot, path)
122		if err != nil {
123			return err
124		}
125		ctx.existingPackages.add(utils.NormalizePath(relPath), metadata)
126	}
127
128	return nil
129}
130
131func (ctx *automationContext) readDefaultOptions() error {
132	log.Printf("Reading defaultOptions from file '%s'...", ctx.optionPath)
133	optionFile, err := os.Open(ctx.optionPath)
134	if err != nil {
135		return err
136	}
137
138	generateOptions, err := model.NewGenerateOptionsFrom(optionFile)
139	if err != nil {
140		return err
141	}
142
143	// parsing the default options
144	defaultOptions, err := model.ParseOptions(generateOptions.AutorestArguments)
145	if err != nil {
146		return fmt.Errorf("cannot parse default options from %v: %+v", generateOptions.AutorestArguments, err)
147	}
148
149	// remove the `--multiapi` in default options
150	var options []model.Option
151	for _, o := range defaultOptions.Arguments() {
152		if v, ok := o.(model.FlagOption); ok && v.Flag() == "multiapi" {
153			continue
154		}
155		options = append(options, o)
156	}
157
158	ctx.defaultOptions = model.NewOptions(options...)
159	log.Printf("Autorest defaultOptions: \n%+v", ctx.defaultOptions.Arguments())
160
161	// parsing the additional options
162	additionalOptions, err := model.ParseOptions(generateOptions.AdditionalOptions)
163	if err != nil {
164		return fmt.Errorf("cannot parse additional options from %v: %+v", generateOptions.AdditionalOptions, err)
165	}
166	ctx.additionalOptions = additionalOptions.Arguments()
167
168	return nil
169}
170
171// TODO -- support dry run
172func (ctx *automationContext) generate(input *pipeline.GenerateInput) (*pipeline.GenerateOutput, error) {
173	if input.DryRun {
174		return nil, fmt.Errorf("dry run not supported yet")
175	}
176
177	log.Printf("Reading packages in azure-sdk-for-go...")
178	if err := ctx.readRepoContent(); err != nil {
179		return nil, err
180	}
181
182	log.Printf("Reading metadata information in azure-sdk-for-go...")
183	if err := ctx.categorizePackages(); err != nil {
184		return nil, err
185	}
186
187	log.Printf("Reading default options...")
188	if err := ctx.readDefaultOptions(); err != nil {
189		return nil, err
190	}
191
192	log.Printf("Reading version number...")
193	if err := ctx.readVersion(); err != nil {
194		return nil, err
195	}
196
197	// iterate over all the readme
198	results := make([]pipeline.PackageResult, 0)
199	errorBuilder := generateErrorBuilder{}
200	for _, readme := range input.RelatedReadmeMdFiles {
201		generateCtx := generateContext{
202			sdkRoot:          ctx.sdkRoot,
203			specRoot:         ctx.specRoot,
204			commitHash:       ctx.commitHash,
205			repoContent:      ctx.repoContent,
206			existingPackages: ctx.existingPackages[readme],
207			defaultOptions:   ctx.defaultOptions,
208		}
209
210		packageResults, errors := generateCtx.generate(readme)
211		if len(errors) != 0 {
212			errorBuilder.add(errors...)
213			continue
214		}
215
216		// iterate over the changed packages
217		set := packageResultSet{}
218		for _, p := range packageResults {
219			log.Printf("Getting package result for package '%s'", p.Package.PackageName)
220			content := p.Package.Changelog.ToCompactMarkdown()
221			breaking := p.Package.Changelog.HasBreakingChanges()
222			breakingChangeItems := p.Package.Changelog.GetBreakingChangeItems()
223			set.add(pipeline.PackageResult{
224				Version:     ctx.sdkVersion,
225				PackageName: getPackageIdentifier(p.Package.PackageName),
226				Path:        []string{p.Package.PackageName},
227				ReadmeMd:    []string{readme},
228				Changelog: &pipeline.Changelog{
229					Content:             &content,
230					HasBreakingChange:   &breaking,
231					BreakingChangeItems: &breakingChangeItems,
232				},
233			})
234		}
235		results = append(results, set.toSlice()...)
236	}
237
238	// validate the sdk structure
239	log.Printf("Validating services directory structure...")
240	exceptions, err := loadExceptions(filepath.Join(ctx.sdkRoot, "tools/pkgchk/exceptions.txt"))
241	if err != nil {
242		return nil, err
243	}
244	if err := track1.VerifyWithDefaultVerifiers(filepath.Join(ctx.sdkRoot, "services"), exceptions); err != nil {
245		return nil, err
246	}
247
248	return &pipeline.GenerateOutput{
249		Packages: squashResults(results),
250	}, errorBuilder.build()
251}
252
253// squashResults squashes the package results by appending all of the `path`s in the following items to the first item
254// By doing this, the SDK automation pipeline will only create one PR that contains all of the generation results
255// instead of creating one PR for each generation result.
256// This is to reduce the resource cost on GitHub
257func squashResults(packages []pipeline.PackageResult) []pipeline.PackageResult {
258	if len(packages) == 0 {
259		return packages
260	}
261	for i := 1; i < len(packages); i++ {
262		// append the path of the i-th item to the first
263		packages[0].Path = append(packages[0].Path, packages[i].Path...)
264		// erase the path on the i-th item
265		packages[i].Path = make([]string, 0)
266	}
267
268	return packages
269}
270
271func (ctx *automationContext) readRepoContent() error {
272	ctx.repoContent = make(map[string]exports.Content)
273	pkgs, err := track1.List(filepath.Join(ctx.sdkRoot, "services"))
274	if err != nil {
275		return fmt.Errorf("failed to list track 1 packages: %+v", err)
276	}
277
278	for _, pkg := range pkgs {
279		relativePath, err := filepath.Rel(ctx.sdkRoot, pkg.FullPath())
280		if err != nil {
281			return err
282		}
283		relativePath = utils.NormalizePath(relativePath)
284		if _, ok := ctx.repoContent[relativePath]; ok {
285			return fmt.Errorf("duplicate package: %s", pkg.Path())
286		}
287		exp, err := exports.Get(pkg.FullPath())
288		if err != nil {
289			return err
290		}
291		ctx.repoContent[relativePath] = exp
292	}
293
294	return nil
295}
296
297func (ctx *automationContext) readVersion() error {
298	v, err := ReadVersion(filepath.Join(ctx.sdkRoot, "version"))
299	if err != nil {
300		return err
301	}
302	ctx.sdkVersion = v
303	return nil
304}
305
306func contains(array []autorest.GenerateResult, item string) bool {
307	for _, r := range array {
308		if utils.NormalizePath(r.Package.PackageName) == utils.NormalizePath(item) {
309			return true
310		}
311	}
312	return false
313}
314
315type generateErrorBuilder struct {
316	errors []error
317}
318
319func (b *generateErrorBuilder) add(err ...error) {
320	b.errors = append(b.errors, err...)
321}
322
323func (b *generateErrorBuilder) build() error {
324	if len(b.errors) == 0 {
325		return nil
326	}
327	var messages []string
328	for _, err := range b.errors {
329		messages = append(messages, err.Error())
330	}
331	return fmt.Errorf("total %d error(s): \n%s", len(b.errors), strings.Join(messages, "\n"))
332}
333
334type packageResultSet map[string]pipeline.PackageResult
335
336func (s packageResultSet) contains(r pipeline.PackageResult) bool {
337	_, ok := s[r.PackageName]
338	return ok
339}
340
341func (s packageResultSet) add(r pipeline.PackageResult) {
342	if s.contains(r) {
343		log.Printf("[WARNING] The result set already contains key %s with value %+v, but we are still trying to insert a new value %+v on the same key", r.PackageName, s[r.PackageName], r)
344	}
345	s[r.PackageName] = r
346}
347
348func (s packageResultSet) toSlice() []pipeline.PackageResult {
349	results := make([]pipeline.PackageResult, 0)
350	for _, r := range s {
351		results = append(results, r)
352	}
353	// sort the results
354	sort.SliceStable(results, func(i, j int) bool {
355		// we first clip the preview segment and then sort by string literal
356		pI := strings.Replace(results[i].PackageName, "preview/", "/", 1)
357		pJ := strings.Replace(results[j].PackageName, "preview/", "/", 1)
358		return pI > pJ
359	})
360	return results
361}
362
363func getPackageIdentifier(pkg string) string {
364	return strings.TrimPrefix(utils.NormalizePath(pkg), "services/")
365}
366
367func loadExceptions(exceptFile string) (map[string]bool, error) {
368	if exceptFile == "" {
369		return nil, nil
370	}
371	f, err := os.Open(exceptFile)
372	if err != nil {
373		return nil, err
374	}
375	defer f.Close()
376
377	exceptions := make(map[string]bool)
378	scanner := bufio.NewScanner(f)
379	for scanner.Scan() {
380		exceptions[scanner.Text()] = true
381	}
382	if err = scanner.Err(); err != nil {
383		return nil, err
384	}
385
386	return exceptions, nil
387}
388
389func logError(err error) {
390	for _, line := range strings.Split(err.Error(), "\n") {
391		if l := strings.TrimSpace(line); l != "" {
392			log.Printf("[ERROR] %s", l)
393		}
394	}
395}
396