1// Copyright 2015 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5// Package gen contains common code for the various code generation tools in the 6// text repository. Its usage ensures consistency between tools. 7// 8// This package defines command line flags that are common to most generation 9// tools. The flags allow for specifying specific Unicode and CLDR versions 10// in the public Unicode data repository (https://www.unicode.org/Public). 11// 12// A local Unicode data mirror can be set through the flag -local or the 13// environment variable UNICODE_DIR. The former takes precedence. The local 14// directory should follow the same structure as the public repository. 15// 16// IANA data can also optionally be mirrored by putting it in the iana directory 17// rooted at the top of the local mirror. Beware, though, that IANA data is not 18// versioned. So it is up to the developer to use the right version. 19package gen // import "golang.org/x/text/internal/gen" 20 21import ( 22 "bytes" 23 "flag" 24 "fmt" 25 "go/build" 26 "go/format" 27 "io" 28 "io/ioutil" 29 "log" 30 "net/http" 31 "os" 32 "path" 33 "path/filepath" 34 "regexp" 35 "strings" 36 "sync" 37 "unicode" 38 39 "golang.org/x/text/unicode/cldr" 40) 41 42var ( 43 url = flag.String("url", 44 "https://www.unicode.org/Public", 45 "URL of Unicode database directory") 46 iana = flag.String("iana", 47 "http://www.iana.org", 48 "URL of the IANA repository") 49 unicodeVersion = flag.String("unicode", 50 getEnv("UNICODE_VERSION", unicode.Version), 51 "unicode version to use") 52 cldrVersion = flag.String("cldr", 53 getEnv("CLDR_VERSION", cldr.Version), 54 "cldr version to use") 55) 56 57func getEnv(name, def string) string { 58 if v := os.Getenv(name); v != "" { 59 return v 60 } 61 return def 62} 63 64// Init performs common initialization for a gen command. It parses the flags 65// and sets up the standard logging parameters. 66func Init() { 67 log.SetPrefix("") 68 log.SetFlags(log.Lshortfile) 69 flag.Parse() 70} 71 72const header = `// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. 73 74` 75 76// UnicodeVersion reports the requested Unicode version. 77func UnicodeVersion() string { 78 return *unicodeVersion 79} 80 81// CLDRVersion reports the requested CLDR version. 82func CLDRVersion() string { 83 return *cldrVersion 84} 85 86var tags = []struct{ version, buildTags string }{ 87 {"9.0.0", "!go1.10"}, 88 {"10.0.0", "go1.10,!go1.13"}, 89 {"11.0.0", "go1.13,!go1.14"}, 90 {"12.0.0", "go1.14,!go1.16"}, 91 {"13.0.0", "go1.16"}, 92} 93 94// buildTags reports the build tags used for the current Unicode version. 95func buildTags() string { 96 v := UnicodeVersion() 97 for _, e := range tags { 98 if e.version == v { 99 return e.buildTags 100 } 101 } 102 log.Fatalf("Unknown build tags for Unicode version %q.", v) 103 return "" 104} 105 106// IsLocal reports whether data files are available locally. 107func IsLocal() bool { 108 dir, err := localReadmeFile() 109 if err != nil { 110 return false 111 } 112 if _, err = os.Stat(dir); err != nil { 113 return false 114 } 115 return true 116} 117 118// OpenUCDFile opens the requested UCD file. The file is specified relative to 119// the public Unicode root directory. It will call log.Fatal if there are any 120// errors. 121func OpenUCDFile(file string) io.ReadCloser { 122 return openUnicode(path.Join(*unicodeVersion, "ucd", file)) 123} 124 125// OpenCLDRCoreZip opens the CLDR core zip file. It will call log.Fatal if there 126// are any errors. 127func OpenCLDRCoreZip() io.ReadCloser { 128 return OpenUnicodeFile("cldr", *cldrVersion, "core.zip") 129} 130 131// OpenUnicodeFile opens the requested file of the requested category from the 132// root of the Unicode data archive. The file is specified relative to the 133// public Unicode root directory. If version is "", it will use the default 134// Unicode version. It will call log.Fatal if there are any errors. 135func OpenUnicodeFile(category, version, file string) io.ReadCloser { 136 if version == "" { 137 version = UnicodeVersion() 138 } 139 return openUnicode(path.Join(category, version, file)) 140} 141 142// OpenIANAFile opens the requested IANA file. The file is specified relative 143// to the IANA root, which is typically either http://www.iana.org or the 144// iana directory in the local mirror. It will call log.Fatal if there are any 145// errors. 146func OpenIANAFile(path string) io.ReadCloser { 147 return Open(*iana, "iana", path) 148} 149 150var ( 151 dirMutex sync.Mutex 152 localDir string 153) 154 155const permissions = 0755 156 157func localReadmeFile() (string, error) { 158 p, err := build.Import("golang.org/x/text", "", build.FindOnly) 159 if err != nil { 160 return "", fmt.Errorf("Could not locate package: %v", err) 161 } 162 return filepath.Join(p.Dir, "DATA", "README"), nil 163} 164 165func getLocalDir() string { 166 dirMutex.Lock() 167 defer dirMutex.Unlock() 168 169 readme, err := localReadmeFile() 170 if err != nil { 171 log.Fatal(err) 172 } 173 dir := filepath.Dir(readme) 174 if _, err := os.Stat(readme); err != nil { 175 if err := os.MkdirAll(dir, permissions); err != nil { 176 log.Fatalf("Could not create directory: %v", err) 177 } 178 ioutil.WriteFile(readme, []byte(readmeTxt), permissions) 179 } 180 return dir 181} 182 183const readmeTxt = `Generated by golang.org/x/text/internal/gen. DO NOT EDIT. 184 185This directory contains downloaded files used to generate the various tables 186in the golang.org/x/text subrepo. 187 188Note that the language subtag repo (iana/assignments/language-subtag-registry) 189and all other times in the iana subdirectory are not versioned and will need 190to be periodically manually updated. The easiest way to do this is to remove 191the entire iana directory. This is mostly of concern when updating the language 192package. 193` 194 195// Open opens subdir/path if a local directory is specified and the file exists, 196// where subdir is a directory relative to the local root, or fetches it from 197// urlRoot/path otherwise. It will call log.Fatal if there are any errors. 198func Open(urlRoot, subdir, path string) io.ReadCloser { 199 file := filepath.Join(getLocalDir(), subdir, filepath.FromSlash(path)) 200 return open(file, urlRoot, path) 201} 202 203func openUnicode(path string) io.ReadCloser { 204 file := filepath.Join(getLocalDir(), filepath.FromSlash(path)) 205 return open(file, *url, path) 206} 207 208// TODO: automatically periodically update non-versioned files. 209 210func open(file, urlRoot, path string) io.ReadCloser { 211 if f, err := os.Open(file); err == nil { 212 return f 213 } 214 r := get(urlRoot, path) 215 defer r.Close() 216 b, err := ioutil.ReadAll(r) 217 if err != nil { 218 log.Fatalf("Could not download file: %v", err) 219 } 220 os.MkdirAll(filepath.Dir(file), permissions) 221 if err := ioutil.WriteFile(file, b, permissions); err != nil { 222 log.Fatalf("Could not create file: %v", err) 223 } 224 return ioutil.NopCloser(bytes.NewReader(b)) 225} 226 227func get(root, path string) io.ReadCloser { 228 url := root + "/" + path 229 fmt.Printf("Fetching %s...", url) 230 defer fmt.Println(" done.") 231 resp, err := http.Get(url) 232 if err != nil { 233 log.Fatalf("HTTP GET: %v", err) 234 } 235 if resp.StatusCode != 200 { 236 log.Fatalf("Bad GET status for %q: %q", url, resp.Status) 237 } 238 return resp.Body 239} 240 241// TODO: use Write*Version in all applicable packages. 242 243// WriteUnicodeVersion writes a constant for the Unicode version from which the 244// tables are generated. 245func WriteUnicodeVersion(w io.Writer) { 246 fmt.Fprintf(w, "// UnicodeVersion is the Unicode version from which the tables in this package are derived.\n") 247 fmt.Fprintf(w, "const UnicodeVersion = %q\n\n", UnicodeVersion()) 248} 249 250// WriteCLDRVersion writes a constant for the CLDR version from which the 251// tables are generated. 252func WriteCLDRVersion(w io.Writer) { 253 fmt.Fprintf(w, "// CLDRVersion is the CLDR version from which the tables in this package are derived.\n") 254 fmt.Fprintf(w, "const CLDRVersion = %q\n\n", CLDRVersion()) 255} 256 257// WriteGoFile prepends a standard file comment and package statement to the 258// given bytes, applies gofmt, and writes them to a file with the given name. 259// It will call log.Fatal if there are any errors. 260func WriteGoFile(filename, pkg string, b []byte) { 261 w, err := os.Create(filename) 262 if err != nil { 263 log.Fatalf("Could not create file %s: %v", filename, err) 264 } 265 defer w.Close() 266 if _, err = WriteGo(w, pkg, "", b); err != nil { 267 log.Fatalf("Error writing file %s: %v", filename, err) 268 } 269} 270 271func fileToPattern(filename string) string { 272 suffix := ".go" 273 if strings.HasSuffix(filename, "_test.go") { 274 suffix = "_test.go" 275 } 276 prefix := filename[:len(filename)-len(suffix)] 277 return fmt.Sprint(prefix, "%s", suffix) 278} 279 280func updateBuildTags(pattern string) { 281 for _, t := range tags { 282 oldFile := fmt.Sprintf(pattern, t.version) 283 b, err := ioutil.ReadFile(oldFile) 284 if err != nil { 285 continue 286 } 287 build := fmt.Sprintf("// +build %s", t.buildTags) 288 b = regexp.MustCompile(`// \+build .*`).ReplaceAll(b, []byte(build)) 289 err = ioutil.WriteFile(oldFile, b, 0644) 290 if err != nil { 291 log.Fatal(err) 292 } 293 } 294} 295 296// WriteVersionedGoFile prepends a standard file comment, adds build tags to 297// version the file for the current Unicode version, and package statement to 298// the given bytes, applies gofmt, and writes them to a file with the given 299// name. It will call log.Fatal if there are any errors. 300func WriteVersionedGoFile(filename, pkg string, b []byte) { 301 pattern := fileToPattern(filename) 302 updateBuildTags(pattern) 303 filename = fmt.Sprintf(pattern, UnicodeVersion()) 304 305 w, err := os.Create(filename) 306 if err != nil { 307 log.Fatalf("Could not create file %s: %v", filename, err) 308 } 309 defer w.Close() 310 if _, err = WriteGo(w, pkg, buildTags(), b); err != nil { 311 log.Fatalf("Error writing file %s: %v", filename, err) 312 } 313} 314 315// WriteGo prepends a standard file comment and package statement to the given 316// bytes, applies gofmt, and writes them to w. 317func WriteGo(w io.Writer, pkg, tags string, b []byte) (n int, err error) { 318 src := []byte(header) 319 if tags != "" { 320 src = append(src, fmt.Sprintf("// +build %s\n\n", tags)...) 321 } 322 src = append(src, fmt.Sprintf("package %s\n\n", pkg)...) 323 src = append(src, b...) 324 formatted, err := format.Source(src) 325 if err != nil { 326 // Print the generated code even in case of an error so that the 327 // returned error can be meaningfully interpreted. 328 n, _ = w.Write(src) 329 return n, err 330 } 331 return w.Write(formatted) 332} 333 334// Repackage rewrites a Go file from belonging to package main to belonging to 335// the given package. 336func Repackage(inFile, outFile, pkg string) { 337 src, err := ioutil.ReadFile(inFile) 338 if err != nil { 339 log.Fatalf("reading %s: %v", inFile, err) 340 } 341 const toDelete = "package main\n\n" 342 i := bytes.Index(src, []byte(toDelete)) 343 if i < 0 { 344 log.Fatalf("Could not find %q in %s.", toDelete, inFile) 345 } 346 w := &bytes.Buffer{} 347 w.Write(src[i+len(toDelete):]) 348 WriteGoFile(outFile, pkg, w.Bytes()) 349} 350