1// Copyright 2015 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5//go:build ignore 6// +build ignore 7 8// gen runs go generate on Unicode- and CLDR-related package in the text 9// repositories, taking into account dependencies and versions. 10package main 11 12import ( 13 "bytes" 14 "flag" 15 "fmt" 16 "go/format" 17 "io/ioutil" 18 "os" 19 "os/exec" 20 "path" 21 "path/filepath" 22 "regexp" 23 "runtime" 24 "strings" 25 "sync" 26 "unicode" 27 28 "golang.org/x/text/collate" 29 "golang.org/x/text/internal/gen" 30 "golang.org/x/text/language" 31) 32 33var ( 34 verbose = flag.Bool("v", false, "verbose output") 35 force = flag.Bool("force", false, "ignore failing dependencies") 36 doCore = flag.Bool("core", false, "force an update to core") 37 skipTest = flag.Bool("skiptest", false, "skip tests") 38 excludeList = flag.String("exclude", "", 39 "comma-separated list of packages to exclude") 40 41 // The user can specify a selection of packages to build on the command line. 42 args []string 43) 44 45func exclude(pkg string) bool { 46 if len(args) > 0 { 47 return !contains(args, pkg) 48 } 49 return contains(strings.Split(*excludeList, ","), pkg) 50} 51 52// TODO: 53// - Better version handling. 54// - Generate tables for the core unicode package? 55// - Add generation for encodings. This requires some retooling here and there. 56// - Running repo-wide "long" tests. 57 58var vprintf = fmt.Printf 59 60func main() { 61 gen.Init() 62 args = flag.Args() 63 if !*verbose { 64 // Set vprintf to a no-op. 65 vprintf = func(string, ...interface{}) (int, error) { return 0, nil } 66 } 67 68 // TODO: create temporary cache directory to load files and create and set 69 // a "cache" option if the user did not specify the UNICODE_DIR environment 70 // variable. This will prevent duplicate downloads and also will enable long 71 // tests, which really need to be run after each generated package. 72 73 updateCore := *doCore 74 if gen.UnicodeVersion() != unicode.Version { 75 fmt.Printf("Requested Unicode version %s; core unicode version is %s.\n", 76 gen.UnicodeVersion(), 77 unicode.Version) 78 c := collate.New(language.Und, collate.Numeric) 79 if c.CompareString(gen.UnicodeVersion(), unicode.Version) < 0 && !*force { 80 os.Exit(2) 81 } 82 updateCore = true 83 goroot := os.Getenv("GOROOT") 84 appendToFile( 85 filepath.Join(goroot, "api", "except.txt"), 86 fmt.Sprintf("pkg unicode, const Version = %q\n", unicode.Version), 87 ) 88 const lines = `pkg unicode, const Version = %q 89// TODO: add a new line of the following form for each new script and property. 90pkg unicode, var <new script or property> *RangeTable 91` 92 appendToFile( 93 filepath.Join(goroot, "api", "next.txt"), 94 fmt.Sprintf(lines, gen.UnicodeVersion()), 95 ) 96 } 97 98 var unicode = &dependency{} 99 if updateCore { 100 fmt.Printf("Updating core to version %s...\n", gen.UnicodeVersion()) 101 unicodeInternal := generate("./internal/export/unicode") 102 unicode = generate("unicode", unicodeInternal) 103 104 // Test some users of the unicode packages, especially the ones that 105 // keep a mirrored table. These may need to be corrected by hand. 106 generate("regexp", unicode) 107 generate("strconv", unicode) // mimics Unicode table 108 generate("strings", unicode) 109 generate("testing", unicode) // mimics Unicode table 110 } 111 112 var ( 113 cldr = generate("./unicode/cldr", unicode) 114 intlang = generate("./internal/language", cldr) 115 compact = generate("./internal/language/compact", intlang, cldr) 116 language = generate("./language", cldr, compact) 117 internal = generate("./internal", unicode, language) 118 norm = generate("./unicode/norm", unicode) 119 rangetable = generate("./unicode/rangetable", unicode) 120 cases = generate("./cases", unicode, norm, language, rangetable) 121 width = generate("./width", unicode) 122 bidi = generate("./unicode/bidi", unicode, norm, rangetable) 123 mib = generate("./encoding/internal/identifier", unicode) 124 number = generate("./internal/number", unicode, cldr, language, internal) 125 cldrtree = generate("./internal/cldrtree", language, internal) 126 _ = generate("./unicode/runenames", unicode) 127 _ = generate("./encoding/htmlindex", unicode, language, mib) 128 _ = generate("./encoding/ianaindex", unicode, language, mib) 129 _ = generate("./secure/precis", unicode, norm, rangetable, cases, width, bidi) 130 _ = generate("./currency", unicode, cldr, language, internal, number) 131 _ = generate("./feature/plural", unicode, cldr, language, internal, number) 132 _ = generate("./internal/export/idna", unicode, bidi, norm) 133 _ = generate("./language/display", unicode, cldr, language, internal, number) 134 _ = generate("./collate", unicode, norm, cldr, language, rangetable) 135 _ = generate("./search", unicode, norm, cldr, language, rangetable) 136 _ = generate("./date", cldr, language, cldrtree) 137 ) 138 all.Wait() 139 140 // Copy exported packages to the destination golang.org repo. 141 copyExported("golang.org/x/net/idna") 142 143 if hasErrors { 144 fmt.Println("FAIL") 145 os.Exit(1) 146 } 147 vprintf("SUCCESS\n") 148} 149 150func appendToFile(file, text string) { 151 fmt.Println("Augmenting", file) 152 w, err := os.OpenFile(file, os.O_APPEND|os.O_WRONLY, 0600) 153 if err != nil { 154 fmt.Println("Failed to open file:", err) 155 os.Exit(1) 156 } 157 defer w.Close() 158 if _, err := w.WriteString(text); err != nil { 159 fmt.Println("Failed to write to file:", err) 160 os.Exit(1) 161 } 162} 163 164var ( 165 all sync.WaitGroup 166 hasErrors bool 167) 168 169type dependency struct { 170 sync.WaitGroup 171 hasErrors bool 172} 173 174func generate(pkg string, deps ...*dependency) *dependency { 175 var wg dependency 176 if exclude(pkg) { 177 return &wg 178 } 179 wg.Add(1) 180 all.Add(1) 181 go func() { 182 defer wg.Done() 183 defer all.Done() 184 // Wait for dependencies to finish. 185 for _, d := range deps { 186 d.Wait() 187 if d.hasErrors && !*force { 188 fmt.Printf("--- ABORT: %s\n", pkg) 189 wg.hasErrors = true 190 return 191 } 192 } 193 vprintf("=== GENERATE %s\n", pkg) 194 args := []string{"generate"} 195 if *verbose { 196 args = append(args, "-v") 197 } 198 args = append(args, pkg) 199 cmd := exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), args...) 200 w := &bytes.Buffer{} 201 cmd.Stderr = w 202 cmd.Stdout = w 203 if err := cmd.Run(); err != nil { 204 fmt.Printf("--- FAIL: %s:\n\t%v\n\tError: %v\n", pkg, indent(w), err) 205 hasErrors = true 206 wg.hasErrors = true 207 return 208 } 209 210 if *skipTest { 211 return 212 } 213 214 vprintf("=== TEST %s\n", pkg) 215 args[0] = "test" 216 cmd = exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), args...) 217 wt := &bytes.Buffer{} 218 cmd.Stderr = wt 219 cmd.Stdout = wt 220 if err := cmd.Run(); err != nil { 221 fmt.Printf("--- FAIL: %s:\n\t%v\n\tError: %v\n", pkg, indent(wt), err) 222 hasErrors = true 223 wg.hasErrors = true 224 return 225 } 226 vprintf("--- SUCCESS: %s\n\t%v\n", pkg, indent(w)) 227 fmt.Print(wt.String()) 228 }() 229 return &wg 230} 231 232// copyExported copies a package in x/text/internal/export to the 233// destination repository. 234func copyExported(p string) { 235 copyPackage( 236 filepath.Join("internal", "export", path.Base(p)), 237 filepath.Join("..", filepath.FromSlash(p[len("golang.org/x"):])), 238 "golang.org/x/text/internal/export/"+path.Base(p), 239 p) 240} 241 242// goGenRE is used to remove go:generate lines. 243var goGenRE = regexp.MustCompile("//go:generate[^\n]*\n") 244 245// copyPackage copies relevant files from a directory in x/text to the 246// destination package directory. The destination package is assumed to have 247// the same name. For each copied file go:generate lines are removed and 248// package comments are rewritten to the new path. 249func copyPackage(dirSrc, dirDst, search, replace string) { 250 err := filepath.Walk(dirSrc, func(file string, info os.FileInfo, err error) error { 251 base := filepath.Base(file) 252 if err != nil || info.IsDir() || 253 !strings.HasSuffix(base, ".go") || 254 strings.HasSuffix(base, "_test.go") || 255 // Don't process subdirectories. 256 filepath.Dir(file) != dirSrc { 257 return nil 258 } 259 if strings.HasPrefix(base, "tables") { 260 if !strings.HasSuffix(base, gen.UnicodeVersion()+".go") { 261 return nil 262 } 263 base = "tables.go" 264 } 265 b, err := ioutil.ReadFile(file) 266 if err != nil || bytes.Contains(b, []byte("\n// +build ignore")) { 267 return err 268 } 269 // Fix paths. 270 b = bytes.Replace(b, []byte(search), []byte(replace), -1) 271 b = bytes.Replace(b, []byte("internal/export"), []byte(""), -1) 272 // Remove go:generate lines. 273 b = goGenRE.ReplaceAllLiteral(b, nil) 274 comment := "// Code generated by running \"go generate\" in golang.org/x/text. DO NOT EDIT.\n\n" 275 if !bytes.HasPrefix(b, []byte(comment)) { 276 b = append([]byte(comment), b...) 277 } 278 if b, err = format.Source(b); err != nil { 279 fmt.Println("Failed to format file:", err) 280 os.Exit(1) 281 } 282 file = filepath.Join(dirDst, base) 283 vprintf("=== COPY %s\n", file) 284 return ioutil.WriteFile(file, b, 0666) 285 }) 286 if err != nil { 287 fmt.Println("Copying exported files failed:", err) 288 os.Exit(1) 289 } 290} 291 292func contains(a []string, s string) bool { 293 for _, e := range a { 294 if s == e { 295 return true 296 } 297 } 298 return false 299} 300 301func indent(b *bytes.Buffer) string { 302 return strings.Replace(strings.TrimSpace(b.String()), "\n", "\n\t", -1) 303} 304