1// Copyright 2015 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5// +build ignore 6 7// gen runs go generate on Unicode- and CLDR-related package in the text 8// repositories, taking into account dependencies and versions. 9package main 10 11import ( 12 "bytes" 13 "flag" 14 "fmt" 15 "go/build" 16 "go/format" 17 "io/ioutil" 18 "os" 19 "os/exec" 20 "path" 21 "path/filepath" 22 "regexp" 23 "runtime" 24 "strings" 25 "sync" 26 "unicode" 27 28 "golang.org/x/text/collate" 29 "golang.org/x/text/internal/gen" 30 "golang.org/x/text/language" 31) 32 33var ( 34 verbose = flag.Bool("v", false, "verbose output") 35 force = flag.Bool("force", false, "ignore failing dependencies") 36 doCore = flag.Bool("core", false, "force an update to core") 37 excludeList = flag.String("exclude", "", 38 "comma-separated list of packages to exclude") 39 40 // The user can specify a selection of packages to build on the command line. 41 args []string 42) 43 44func exclude(pkg string) bool { 45 if len(args) > 0 { 46 return !contains(args, pkg) 47 } 48 return contains(strings.Split(*excludeList, ","), pkg) 49} 50 51// TODO: 52// - Better version handling. 53// - Generate tables for the core unicode package? 54// - Add generation for encodings. This requires some retooling here and there. 55// - Running repo-wide "long" tests. 56 57var vprintf = fmt.Printf 58 59func main() { 60 gen.Init() 61 args = flag.Args() 62 if !*verbose { 63 // Set vprintf to a no-op. 64 vprintf = func(string, ...interface{}) (int, error) { return 0, nil } 65 } 66 67 // TODO: create temporary cache directory to load files and create and set 68 // a "cache" option if the user did not specify the UNICODE_DIR environment 69 // variable. This will prevent duplicate downloads and also will enable long 70 // tests, which really need to be run after each generated package. 71 72 updateCore := *doCore 73 if gen.UnicodeVersion() != unicode.Version { 74 fmt.Printf("Requested Unicode version %s; core unicode version is %s.\n", 75 gen.UnicodeVersion(), 76 unicode.Version) 77 c := collate.New(language.Und, collate.Numeric) 78 if c.CompareString(gen.UnicodeVersion(), unicode.Version) < 0 && !*force { 79 os.Exit(2) 80 } 81 updateCore = true 82 goroot := os.Getenv("GOROOT") 83 appendToFile( 84 filepath.Join(goroot, "api", "except.txt"), 85 fmt.Sprintf("pkg unicode, const Version = %q\n", unicode.Version), 86 ) 87 const lines = `pkg unicode, const Version = %q 88// TODO: add a new line of the following form for each new script and property. 89pkg unicode, var <new script or property> *RangeTable 90` 91 appendToFile( 92 filepath.Join(goroot, "api", "next.txt"), 93 fmt.Sprintf(lines, gen.UnicodeVersion()), 94 ) 95 } 96 97 var unicode = &dependency{} 98 if updateCore { 99 fmt.Printf("Updating core to version %s...\n", gen.UnicodeVersion()) 100 unicode = generate("unicode") 101 102 // Test some users of the unicode packages, especially the ones that 103 // keep a mirrored table. These may need to be corrected by hand. 104 generate("regexp", unicode) 105 generate("strconv", unicode) // mimics Unicode table 106 generate("strings", unicode) 107 generate("testing", unicode) // mimics Unicode table 108 } 109 110 var ( 111 cldr = generate("./unicode/cldr", unicode) 112 language = generate("./language", cldr) 113 internal = generate("./internal", unicode, language) 114 norm = generate("./unicode/norm", unicode) 115 rangetable = generate("./unicode/rangetable", unicode) 116 cases = generate("./cases", unicode, norm, language, rangetable) 117 width = generate("./width", unicode) 118 bidi = generate("./unicode/bidi", unicode, norm, rangetable) 119 mib = generate("./encoding/internal/identifier", unicode) 120 number = generate("./internal/number", unicode, cldr, language, internal) 121 _ = generate("./encoding/htmlindex", unicode, language, mib) 122 _ = generate("./encoding/ianaindex", unicode, language, mib) 123 _ = generate("./secure/precis", unicode, norm, rangetable, cases, width, bidi) 124 _ = generate("./internal/cldrtree", language) 125 _ = generate("./currency", unicode, cldr, language, internal, number) 126 _ = generate("./feature/plural", unicode, cldr, language, internal, number) 127 _ = generate("./internal/export/idna", unicode, bidi, norm) 128 _ = generate("./language/display", unicode, cldr, language, internal, number) 129 _ = generate("./collate", unicode, norm, cldr, language, rangetable) 130 _ = generate("./search", unicode, norm, cldr, language, rangetable) 131 ) 132 all.Wait() 133 134 // Copy exported packages to the destination golang.org repo. 135 copyExported("golang.org/x/net/idna") 136 137 if updateCore { 138 copyVendored() 139 } 140 141 if hasErrors { 142 fmt.Println("FAIL") 143 os.Exit(1) 144 } 145 vprintf("SUCCESS\n") 146} 147 148func appendToFile(file, text string) { 149 fmt.Println("Augmenting", file) 150 w, err := os.OpenFile(file, os.O_APPEND|os.O_WRONLY, 0600) 151 if err != nil { 152 fmt.Println("Failed to open file:", err) 153 os.Exit(1) 154 } 155 defer w.Close() 156 if _, err := w.WriteString(text); err != nil { 157 fmt.Println("Failed to write to file:", err) 158 os.Exit(1) 159 } 160} 161 162var ( 163 all sync.WaitGroup 164 hasErrors bool 165) 166 167type dependency struct { 168 sync.WaitGroup 169 hasErrors bool 170} 171 172func generate(pkg string, deps ...*dependency) *dependency { 173 var wg dependency 174 if exclude(pkg) { 175 return &wg 176 } 177 wg.Add(1) 178 all.Add(1) 179 go func() { 180 defer wg.Done() 181 defer all.Done() 182 // Wait for dependencies to finish. 183 for _, d := range deps { 184 d.Wait() 185 if d.hasErrors && !*force { 186 fmt.Printf("--- ABORT: %s\n", pkg) 187 wg.hasErrors = true 188 return 189 } 190 } 191 vprintf("=== GENERATE %s\n", pkg) 192 args := []string{"generate"} 193 if *verbose { 194 args = append(args, "-v") 195 } 196 args = append(args, pkg) 197 cmd := exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), args...) 198 w := &bytes.Buffer{} 199 cmd.Stderr = w 200 cmd.Stdout = w 201 if err := cmd.Run(); err != nil { 202 fmt.Printf("--- FAIL: %s:\n\t%v\n\tError: %v\n", pkg, indent(w), err) 203 hasErrors = true 204 wg.hasErrors = true 205 return 206 } 207 208 vprintf("=== TEST %s\n", pkg) 209 args[0] = "test" 210 cmd = exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), args...) 211 wt := &bytes.Buffer{} 212 cmd.Stderr = wt 213 cmd.Stdout = wt 214 if err := cmd.Run(); err != nil { 215 fmt.Printf("--- FAIL: %s:\n\t%v\n\tError: %v\n", pkg, indent(wt), err) 216 hasErrors = true 217 wg.hasErrors = true 218 return 219 } 220 vprintf("--- SUCCESS: %s\n\t%v\n", pkg, indent(w)) 221 fmt.Print(wt.String()) 222 }() 223 return &wg 224} 225 226// copyExported copies a package in x/text/internal/export to the 227// destination repository. 228func copyExported(p string) { 229 copyPackage( 230 filepath.Join("internal", "export", path.Base(p)), 231 filepath.Join("..", filepath.FromSlash(p[len("golang.org/x"):])), 232 "golang.org/x/text/internal/export/"+path.Base(p), 233 p) 234} 235 236// copyVendored copies packages used by Go core into the vendored directory. 237func copyVendored() { 238 root := filepath.Join(build.Default.GOROOT, filepath.FromSlash("src/vendor/golang_org/x")) 239 240 err := filepath.Walk(root, func(dir string, info os.FileInfo, err error) error { 241 if err != nil || !info.IsDir() || root == dir { 242 return err 243 } 244 src := dir[len(root)+1:] 245 const slash = string(filepath.Separator) 246 if c := strings.Split(src, slash); c[0] == "text" { 247 // Copy a text repo package from its normal location. 248 src = strings.Join(c[1:], slash) 249 } else { 250 // Copy the vendored package if it exists in the export directory. 251 src = filepath.Join("internal", "export", filepath.Base(src)) 252 } 253 copyPackage(src, dir, "golang.org", "golang_org") 254 return nil 255 }) 256 if err != nil { 257 fmt.Printf("Seeding directory %s has failed %v:", root, err) 258 os.Exit(1) 259 } 260} 261 262// goGenRE is used to remove go:generate lines. 263var goGenRE = regexp.MustCompile("//go:generate[^\n]*\n") 264 265// copyPackage copies relevant files from a directory in x/text to the 266// destination package directory. The destination package is assumed to have 267// the same name. For each copied file go:generate lines are removed and 268// and package comments are rewritten to the new path. 269func copyPackage(dirSrc, dirDst, search, replace string) { 270 err := filepath.Walk(dirSrc, func(file string, info os.FileInfo, err error) error { 271 base := filepath.Base(file) 272 if err != nil || info.IsDir() || 273 !strings.HasSuffix(base, ".go") || 274 strings.HasSuffix(base, "_test.go") || 275 // Don't process subdirectories. 276 filepath.Dir(file) != dirSrc { 277 return nil 278 } 279 b, err := ioutil.ReadFile(file) 280 if err != nil || bytes.Contains(b, []byte("\n// +build ignore")) { 281 return err 282 } 283 // Fix paths. 284 b = bytes.Replace(b, []byte(search), []byte(replace), -1) 285 // Remove go:generate lines. 286 b = goGenRE.ReplaceAllLiteral(b, nil) 287 comment := "// Code generated by running \"go generate\" in golang.org/x/text. DO NOT EDIT.\n\n" 288 if *doCore { 289 comment = "// Code generated by running \"go run gen.go -core\" in golang.org/x/text. DO NOT EDIT.\n\n" 290 } 291 if !bytes.HasPrefix(b, []byte(comment)) { 292 b = append([]byte(comment), b...) 293 } 294 if b, err = format.Source(b); err != nil { 295 fmt.Println("Failed to format file:", err) 296 os.Exit(1) 297 } 298 file = filepath.Join(dirDst, base) 299 vprintf("=== COPY %s\n", file) 300 return ioutil.WriteFile(file, b, 0666) 301 }) 302 if err != nil { 303 fmt.Println("Copying exported files failed:", err) 304 os.Exit(1) 305 } 306} 307 308func contains(a []string, s string) bool { 309 for _, e := range a { 310 if s == e { 311 return true 312 } 313 } 314 return false 315} 316 317func indent(b *bytes.Buffer) string { 318 return strings.Replace(strings.TrimSpace(b.String()), "\n", "\n\t", -1) 319} 320