1// Copyright (c) Microsoft Corporation. All rights reserved. 2// Licensed under the MIT License. See License.txt in the project root for license information. 3 4package automation 5 6import ( 7 "bufio" 8 "fmt" 9 "log" 10 "os" 11 "path/filepath" 12 "sort" 13 "strings" 14 15 "github.com/Azure/azure-sdk-for-go/tools/generator/autorest" 16 "github.com/Azure/azure-sdk-for-go/tools/generator/autorest/model" 17 "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/automation/pipeline" 18 "github.com/Azure/azure-sdk-for-go/tools/generator/common" 19 "github.com/Azure/azure-sdk-for-go/tools/internal/exports" 20 "github.com/Azure/azure-sdk-for-go/tools/internal/packages/track1" 21 "github.com/Azure/azure-sdk-for-go/tools/internal/utils" 22 "github.com/spf13/cobra" 23) 24 25// Command returns the automation command. Note that this command is designed to run in the root directory of 26// azure-sdk-for-go. It does not work if you are running this tool in somewhere else 27func Command() *cobra.Command { 28 automationCmd := &cobra.Command{ 29 Use: "automation <generate input filepath> <generate output filepath>", 30 Args: cobra.ExactArgs(2), 31 PersistentPreRunE: func(cmd *cobra.Command, args []string) error { 32 log.SetFlags(0) // remove the time stamp prefix 33 return nil 34 }, 35 RunE: func(cmd *cobra.Command, args []string) error { 36 optionPath, err := cmd.Flags().GetString("options") 37 if err != nil { 38 logError(err) 39 return err 40 } 41 if err := execute(args[0], args[1], Flags{ 42 OptionPath: optionPath, 43 }); err != nil { 44 logError(err) 45 return err 46 } 47 return nil 48 }, 49 SilenceUsage: true, // this command is used for a pipeline, the usage should never show 50 } 51 52 flags := automationCmd.Flags() 53 flags.String("options", common.DefaultOptionPath, "Specify a file with the autorest options") 54 55 return automationCmd 56} 57 58// Flags ... 59type Flags struct { 60 OptionPath string 61} 62 63func execute(inputPath, outputPath string, flags Flags) error { 64 log.Printf("Reading generate input file from '%s'...", inputPath) 65 input, err := pipeline.ReadInput(inputPath) 66 if err != nil { 67 return fmt.Errorf("cannot read generate input: %+v", err) 68 } 69 log.Printf("Generating using the following GenerateInput...\n%s", input.String()) 70 cwd, err := os.Getwd() 71 if err != nil { 72 return err 73 } 74 log.Printf("Using current directory as SDK root: %s", cwd) 75 76 ctx := automationContext{ 77 sdkRoot: utils.NormalizePath(cwd), 78 specRoot: input.SpecFolder, 79 commitHash: input.HeadSha, 80 optionPath: flags.OptionPath, 81 } 82 output, err := ctx.generate(input) 83 if err != nil { 84 return err 85 } 86 log.Printf("Output generated: \n%s", output.String()) 87 log.Printf("Writing output to file '%s'...", outputPath) 88 if err := pipeline.WriteOutput(outputPath, output); err != nil { 89 return fmt.Errorf("cannot write generate output: %+v", err) 90 } 91 return nil 92} 93 94type automationContext struct { 95 sdkRoot string 96 specRoot string 97 commitHash string 98 optionPath string 99 100 repoContent map[string]exports.Content 101 102 sdkVersion string 103 104 existingPackages existingPackageMap 105 106 defaultOptions model.Options 107 additionalOptions []model.Option 108} 109 110func (ctx *automationContext) categorizePackages() error { 111 ctx.existingPackages = existingPackageMap{} 112 113 serviceRoot := filepath.Join(ctx.sdkRoot, "services") 114 m, err := autorest.CollectGenerationMetadata(serviceRoot) 115 if err != nil { 116 return err 117 } 118 119 for path, metadata := range m { 120 // the path in the metadata map is the absolute path 121 relPath, err := filepath.Rel(ctx.sdkRoot, path) 122 if err != nil { 123 return err 124 } 125 ctx.existingPackages.add(utils.NormalizePath(relPath), metadata) 126 } 127 128 return nil 129} 130 131func (ctx *automationContext) readDefaultOptions() error { 132 log.Printf("Reading defaultOptions from file '%s'...", ctx.optionPath) 133 optionFile, err := os.Open(ctx.optionPath) 134 if err != nil { 135 return err 136 } 137 138 generateOptions, err := model.NewGenerateOptionsFrom(optionFile) 139 if err != nil { 140 return err 141 } 142 143 // parsing the default options 144 defaultOptions, err := model.ParseOptions(generateOptions.AutorestArguments) 145 if err != nil { 146 return fmt.Errorf("cannot parse default options from %v: %+v", generateOptions.AutorestArguments, err) 147 } 148 149 // remove the `--multiapi` in default options 150 var options []model.Option 151 for _, o := range defaultOptions.Arguments() { 152 if v, ok := o.(model.FlagOption); ok && v.Flag() == "multiapi" { 153 continue 154 } 155 options = append(options, o) 156 } 157 158 ctx.defaultOptions = model.NewOptions(options...) 159 log.Printf("Autorest defaultOptions: \n%+v", ctx.defaultOptions.Arguments()) 160 161 // parsing the additional options 162 additionalOptions, err := model.ParseOptions(generateOptions.AdditionalOptions) 163 if err != nil { 164 return fmt.Errorf("cannot parse additional options from %v: %+v", generateOptions.AdditionalOptions, err) 165 } 166 ctx.additionalOptions = additionalOptions.Arguments() 167 168 return nil 169} 170 171// TODO -- support dry run 172func (ctx *automationContext) generate(input *pipeline.GenerateInput) (*pipeline.GenerateOutput, error) { 173 if input.DryRun { 174 return nil, fmt.Errorf("dry run not supported yet") 175 } 176 177 log.Printf("Reading packages in azure-sdk-for-go...") 178 if err := ctx.readRepoContent(); err != nil { 179 return nil, err 180 } 181 182 log.Printf("Reading metadata information in azure-sdk-for-go...") 183 if err := ctx.categorizePackages(); err != nil { 184 return nil, err 185 } 186 187 log.Printf("Reading default options...") 188 if err := ctx.readDefaultOptions(); err != nil { 189 return nil, err 190 } 191 192 log.Printf("Reading version number...") 193 if err := ctx.readVersion(); err != nil { 194 return nil, err 195 } 196 197 // iterate over all the readme 198 results := make([]pipeline.PackageResult, 0) 199 errorBuilder := generateErrorBuilder{} 200 for _, readme := range input.RelatedReadmeMdFiles { 201 generateCtx := generateContext{ 202 sdkRoot: ctx.sdkRoot, 203 specRoot: ctx.specRoot, 204 commitHash: ctx.commitHash, 205 repoContent: ctx.repoContent, 206 existingPackages: ctx.existingPackages[readme], 207 defaultOptions: ctx.defaultOptions, 208 } 209 210 packageResults, errors := generateCtx.generate(readme) 211 if len(errors) != 0 { 212 errorBuilder.add(errors...) 213 continue 214 } 215 216 // iterate over the changed packages 217 set := packageResultSet{} 218 for _, p := range packageResults { 219 log.Printf("Getting package result for package '%s'", p.Package.PackageName) 220 content := p.Package.Changelog.ToCompactMarkdown() 221 breaking := p.Package.Changelog.HasBreakingChanges() 222 breakingChangeItems := p.Package.Changelog.GetBreakingChangeItems() 223 set.add(pipeline.PackageResult{ 224 Version: ctx.sdkVersion, 225 PackageName: getPackageIdentifier(p.Package.PackageName), 226 Path: []string{p.Package.PackageName}, 227 ReadmeMd: []string{readme}, 228 Changelog: &pipeline.Changelog{ 229 Content: &content, 230 HasBreakingChange: &breaking, 231 BreakingChangeItems: &breakingChangeItems, 232 }, 233 }) 234 } 235 results = append(results, set.toSlice()...) 236 } 237 238 // validate the sdk structure 239 log.Printf("Validating services directory structure...") 240 exceptions, err := loadExceptions(filepath.Join(ctx.sdkRoot, "tools/pkgchk/exceptions.txt")) 241 if err != nil { 242 return nil, err 243 } 244 if err := track1.VerifyWithDefaultVerifiers(filepath.Join(ctx.sdkRoot, "services"), exceptions); err != nil { 245 return nil, err 246 } 247 248 return &pipeline.GenerateOutput{ 249 Packages: squashResults(results), 250 }, errorBuilder.build() 251} 252 253// squashResults squashes the package results by appending all of the `path`s in the following items to the first item 254// By doing this, the SDK automation pipeline will only create one PR that contains all of the generation results 255// instead of creating one PR for each generation result. 256// This is to reduce the resource cost on GitHub 257func squashResults(packages []pipeline.PackageResult) []pipeline.PackageResult { 258 if len(packages) == 0 { 259 return packages 260 } 261 for i := 1; i < len(packages); i++ { 262 // append the path of the i-th item to the first 263 packages[0].Path = append(packages[0].Path, packages[i].Path...) 264 // erase the path on the i-th item 265 packages[i].Path = make([]string, 0) 266 } 267 268 return packages 269} 270 271func (ctx *automationContext) readRepoContent() error { 272 ctx.repoContent = make(map[string]exports.Content) 273 pkgs, err := track1.List(filepath.Join(ctx.sdkRoot, "services")) 274 if err != nil { 275 return fmt.Errorf("failed to list track 1 packages: %+v", err) 276 } 277 278 for _, pkg := range pkgs { 279 relativePath, err := filepath.Rel(ctx.sdkRoot, pkg.FullPath()) 280 if err != nil { 281 return err 282 } 283 relativePath = utils.NormalizePath(relativePath) 284 if _, ok := ctx.repoContent[relativePath]; ok { 285 return fmt.Errorf("duplicate package: %s", pkg.Path()) 286 } 287 exp, err := exports.Get(pkg.FullPath()) 288 if err != nil { 289 return err 290 } 291 ctx.repoContent[relativePath] = exp 292 } 293 294 return nil 295} 296 297func (ctx *automationContext) readVersion() error { 298 v, err := ReadVersion(filepath.Join(ctx.sdkRoot, "version")) 299 if err != nil { 300 return err 301 } 302 ctx.sdkVersion = v 303 return nil 304} 305 306func contains(array []autorest.GenerateResult, item string) bool { 307 for _, r := range array { 308 if utils.NormalizePath(r.Package.PackageName) == utils.NormalizePath(item) { 309 return true 310 } 311 } 312 return false 313} 314 315type generateErrorBuilder struct { 316 errors []error 317} 318 319func (b *generateErrorBuilder) add(err ...error) { 320 b.errors = append(b.errors, err...) 321} 322 323func (b *generateErrorBuilder) build() error { 324 if len(b.errors) == 0 { 325 return nil 326 } 327 var messages []string 328 for _, err := range b.errors { 329 messages = append(messages, err.Error()) 330 } 331 return fmt.Errorf("total %d error(s): \n%s", len(b.errors), strings.Join(messages, "\n")) 332} 333 334type packageResultSet map[string]pipeline.PackageResult 335 336func (s packageResultSet) contains(r pipeline.PackageResult) bool { 337 _, ok := s[r.PackageName] 338 return ok 339} 340 341func (s packageResultSet) add(r pipeline.PackageResult) { 342 if s.contains(r) { 343 log.Printf("[WARNING] The result set already contains key %s with value %+v, but we are still trying to insert a new value %+v on the same key", r.PackageName, s[r.PackageName], r) 344 } 345 s[r.PackageName] = r 346} 347 348func (s packageResultSet) toSlice() []pipeline.PackageResult { 349 results := make([]pipeline.PackageResult, 0) 350 for _, r := range s { 351 results = append(results, r) 352 } 353 // sort the results 354 sort.SliceStable(results, func(i, j int) bool { 355 // we first clip the preview segment and then sort by string literal 356 pI := strings.Replace(results[i].PackageName, "preview/", "/", 1) 357 pJ := strings.Replace(results[j].PackageName, "preview/", "/", 1) 358 return pI > pJ 359 }) 360 return results 361} 362 363func getPackageIdentifier(pkg string) string { 364 return strings.TrimPrefix(utils.NormalizePath(pkg), "services/") 365} 366 367func loadExceptions(exceptFile string) (map[string]bool, error) { 368 if exceptFile == "" { 369 return nil, nil 370 } 371 f, err := os.Open(exceptFile) 372 if err != nil { 373 return nil, err 374 } 375 defer f.Close() 376 377 exceptions := make(map[string]bool) 378 scanner := bufio.NewScanner(f) 379 for scanner.Scan() { 380 exceptions[scanner.Text()] = true 381 } 382 if err = scanner.Err(); err != nil { 383 return nil, err 384 } 385 386 return exceptions, nil 387} 388 389func logError(err error) { 390 for _, line := range strings.Split(err.Error(), "\n") { 391 if l := strings.TrimSpace(line); l != "" { 392 log.Printf("[ERROR] %s", l) 393 } 394 } 395} 396