1// Copyright 2015 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5// +build ignore 6 7// gen runs go generate on Unicode- and CLDR-related package in the text 8// repositories, taking into account dependencies and versions. 9package main 10 11import ( 12 "bytes" 13 "flag" 14 "fmt" 15 "go/format" 16 "io/ioutil" 17 "os" 18 "os/exec" 19 "path" 20 "path/filepath" 21 "regexp" 22 "runtime" 23 "strings" 24 "sync" 25 "unicode" 26 27 "golang.org/x/text/collate" 28 "golang.org/x/text/internal/gen" 29 "golang.org/x/text/language" 30) 31 32var ( 33 verbose = flag.Bool("v", false, "verbose output") 34 force = flag.Bool("force", false, "ignore failing dependencies") 35 doCore = flag.Bool("core", false, "force an update to core") 36 skipTest = flag.Bool("skiptest", false, "skip tests") 37 excludeList = flag.String("exclude", "", 38 "comma-separated list of packages to exclude") 39 40 // The user can specify a selection of packages to build on the command line. 41 args []string 42) 43 44func exclude(pkg string) bool { 45 if len(args) > 0 { 46 return !contains(args, pkg) 47 } 48 return contains(strings.Split(*excludeList, ","), pkg) 49} 50 51// TODO: 52// - Better version handling. 53// - Generate tables for the core unicode package? 54// - Add generation for encodings. This requires some retooling here and there. 55// - Running repo-wide "long" tests. 56 57var vprintf = fmt.Printf 58 59func main() { 60 gen.Init() 61 args = flag.Args() 62 if !*verbose { 63 // Set vprintf to a no-op. 64 vprintf = func(string, ...interface{}) (int, error) { return 0, nil } 65 } 66 67 // TODO: create temporary cache directory to load files and create and set 68 // a "cache" option if the user did not specify the UNICODE_DIR environment 69 // variable. This will prevent duplicate downloads and also will enable long 70 // tests, which really need to be run after each generated package. 71 72 updateCore := *doCore 73 if gen.UnicodeVersion() != unicode.Version { 74 fmt.Printf("Requested Unicode version %s; core unicode version is %s.\n", 75 gen.UnicodeVersion(), 76 unicode.Version) 77 c := collate.New(language.Und, collate.Numeric) 78 if c.CompareString(gen.UnicodeVersion(), unicode.Version) < 0 && !*force { 79 os.Exit(2) 80 } 81 updateCore = true 82 goroot := os.Getenv("GOROOT") 83 appendToFile( 84 filepath.Join(goroot, "api", "except.txt"), 85 fmt.Sprintf("pkg unicode, const Version = %q\n", unicode.Version), 86 ) 87 const lines = `pkg unicode, const Version = %q 88// TODO: add a new line of the following form for each new script and property. 89pkg unicode, var <new script or property> *RangeTable 90` 91 appendToFile( 92 filepath.Join(goroot, "api", "next.txt"), 93 fmt.Sprintf(lines, gen.UnicodeVersion()), 94 ) 95 } 96 97 var unicode = &dependency{} 98 if updateCore { 99 fmt.Printf("Updating core to version %s...\n", gen.UnicodeVersion()) 100 unicodeInternal := generate("./internal/export/unicode") 101 unicode = generate("unicode", unicodeInternal) 102 103 // Test some users of the unicode packages, especially the ones that 104 // keep a mirrored table. These may need to be corrected by hand. 105 generate("regexp", unicode) 106 generate("strconv", unicode) // mimics Unicode table 107 generate("strings", unicode) 108 generate("testing", unicode) // mimics Unicode table 109 } 110 111 var ( 112 cldr = generate("./unicode/cldr", unicode) 113 compact = generate("./internal/language/compact", cldr) 114 language = generate("./language", cldr, compact) 115 internal = generate("./internal", unicode, language) 116 norm = generate("./unicode/norm", unicode) 117 rangetable = generate("./unicode/rangetable", unicode) 118 cases = generate("./cases", unicode, norm, language, rangetable) 119 width = generate("./width", unicode) 120 bidi = generate("./unicode/bidi", unicode, norm, rangetable) 121 mib = generate("./encoding/internal/identifier", unicode) 122 number = generate("./internal/number", unicode, cldr, language, internal) 123 cldrtree = generate("./internal/cldrtree", language, internal) 124 _ = generate("./unicode/runenames", unicode) 125 _ = generate("./encoding/htmlindex", unicode, language, mib) 126 _ = generate("./encoding/ianaindex", unicode, language, mib) 127 _ = generate("./secure/precis", unicode, norm, rangetable, cases, width, bidi) 128 _ = generate("./currency", unicode, cldr, language, internal, number) 129 _ = generate("./feature/plural", unicode, cldr, language, internal, number) 130 _ = generate("./internal/export/idna", unicode, bidi, norm) 131 _ = generate("./language/display", unicode, cldr, language, internal, number) 132 _ = generate("./collate", unicode, norm, cldr, language, rangetable) 133 _ = generate("./search", unicode, norm, cldr, language, rangetable) 134 _ = generate("./date", cldr, language, cldrtree) 135 ) 136 all.Wait() 137 138 // Copy exported packages to the destination golang.org repo. 139 copyExported("golang.org/x/net/idna") 140 141 if hasErrors { 142 fmt.Println("FAIL") 143 os.Exit(1) 144 } 145 vprintf("SUCCESS\n") 146} 147 148func appendToFile(file, text string) { 149 fmt.Println("Augmenting", file) 150 w, err := os.OpenFile(file, os.O_APPEND|os.O_WRONLY, 0600) 151 if err != nil { 152 fmt.Println("Failed to open file:", err) 153 os.Exit(1) 154 } 155 defer w.Close() 156 if _, err := w.WriteString(text); err != nil { 157 fmt.Println("Failed to write to file:", err) 158 os.Exit(1) 159 } 160} 161 162var ( 163 all sync.WaitGroup 164 hasErrors bool 165) 166 167type dependency struct { 168 sync.WaitGroup 169 hasErrors bool 170} 171 172func generate(pkg string, deps ...*dependency) *dependency { 173 var wg dependency 174 if exclude(pkg) { 175 return &wg 176 } 177 wg.Add(1) 178 all.Add(1) 179 go func() { 180 defer wg.Done() 181 defer all.Done() 182 // Wait for dependencies to finish. 183 for _, d := range deps { 184 d.Wait() 185 if d.hasErrors && !*force { 186 fmt.Printf("--- ABORT: %s\n", pkg) 187 wg.hasErrors = true 188 return 189 } 190 } 191 vprintf("=== GENERATE %s\n", pkg) 192 args := []string{"generate"} 193 if *verbose { 194 args = append(args, "-v") 195 } 196 args = append(args, pkg) 197 cmd := exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), args...) 198 w := &bytes.Buffer{} 199 cmd.Stderr = w 200 cmd.Stdout = w 201 if err := cmd.Run(); err != nil { 202 fmt.Printf("--- FAIL: %s:\n\t%v\n\tError: %v\n", pkg, indent(w), err) 203 hasErrors = true 204 wg.hasErrors = true 205 return 206 } 207 208 if *skipTest { 209 return 210 } 211 212 vprintf("=== TEST %s\n", pkg) 213 args[0] = "test" 214 cmd = exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), args...) 215 wt := &bytes.Buffer{} 216 cmd.Stderr = wt 217 cmd.Stdout = wt 218 if err := cmd.Run(); err != nil { 219 fmt.Printf("--- FAIL: %s:\n\t%v\n\tError: %v\n", pkg, indent(wt), err) 220 hasErrors = true 221 wg.hasErrors = true 222 return 223 } 224 vprintf("--- SUCCESS: %s\n\t%v\n", pkg, indent(w)) 225 fmt.Print(wt.String()) 226 }() 227 return &wg 228} 229 230// copyExported copies a package in x/text/internal/export to the 231// destination repository. 232func copyExported(p string) { 233 copyPackage( 234 filepath.Join("internal", "export", path.Base(p)), 235 filepath.Join("..", filepath.FromSlash(p[len("golang.org/x"):])), 236 "golang.org/x/text/internal/export/"+path.Base(p), 237 p) 238} 239 240// goGenRE is used to remove go:generate lines. 241var goGenRE = regexp.MustCompile("//go:generate[^\n]*\n") 242 243// copyPackage copies relevant files from a directory in x/text to the 244// destination package directory. The destination package is assumed to have 245// the same name. For each copied file go:generate lines are removed and 246// package comments are rewritten to the new path. 247func copyPackage(dirSrc, dirDst, search, replace string) { 248 err := filepath.Walk(dirSrc, func(file string, info os.FileInfo, err error) error { 249 base := filepath.Base(file) 250 if err != nil || info.IsDir() || 251 !strings.HasSuffix(base, ".go") || 252 strings.HasSuffix(base, "_test.go") || 253 // Don't process subdirectories. 254 filepath.Dir(file) != dirSrc { 255 return nil 256 } 257 if strings.HasPrefix(base, "tables") { 258 if !strings.HasSuffix(base, gen.UnicodeVersion()+".go") { 259 return nil 260 } 261 base = "tables.go" 262 } 263 b, err := ioutil.ReadFile(file) 264 if err != nil || bytes.Contains(b, []byte("\n// +build ignore")) { 265 return err 266 } 267 // Fix paths. 268 b = bytes.Replace(b, []byte(search), []byte(replace), -1) 269 b = bytes.Replace(b, []byte("internal/export"), []byte(""), -1) 270 // Remove go:generate lines. 271 b = goGenRE.ReplaceAllLiteral(b, nil) 272 comment := "// Code generated by running \"go generate\" in golang.org/x/text. DO NOT EDIT.\n\n" 273 if !bytes.HasPrefix(b, []byte(comment)) { 274 b = append([]byte(comment), b...) 275 } 276 if b, err = format.Source(b); err != nil { 277 fmt.Println("Failed to format file:", err) 278 os.Exit(1) 279 } 280 file = filepath.Join(dirDst, base) 281 vprintf("=== COPY %s\n", file) 282 return ioutil.WriteFile(file, b, 0666) 283 }) 284 if err != nil { 285 fmt.Println("Copying exported files failed:", err) 286 os.Exit(1) 287 } 288} 289 290func contains(a []string, s string) bool { 291 for _, e := range a { 292 if s == e { 293 return true 294 } 295 } 296 return false 297} 298 299func indent(b *bytes.Buffer) string { 300 return strings.Replace(strings.TrimSpace(b.String()), "\n", "\n\t", -1) 301} 302