1// Copyright 2015 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5// Package gen contains common code for the various code generation tools in the 6// text repository. Its usage ensures consistency between tools. 7// 8// This package defines command line flags that are common to most generation 9// tools. The flags allow for specifying specific Unicode and CLDR versions 10// in the public Unicode data repository (http://www.unicode.org/Public). 11// 12// A local Unicode data mirror can be set through the flag -local or the 13// environment variable UNICODE_DIR. The former takes precedence. The local 14// directory should follow the same structure as the public repository. 15// 16// IANA data can also optionally be mirrored by putting it in the iana directory 17// rooted at the top of the local mirror. Beware, though, that IANA data is not 18// versioned. So it is up to the developer to use the right version. 19package gen // import "golang.org/x/text/internal/gen" 20 21import ( 22 "bytes" 23 "flag" 24 "fmt" 25 "go/build" 26 "go/format" 27 "io" 28 "io/ioutil" 29 "log" 30 "net/http" 31 "os" 32 "path" 33 "path/filepath" 34 "strings" 35 "sync" 36 "unicode" 37 38 "golang.org/x/text/unicode/cldr" 39) 40 41var ( 42 url = flag.String("url", 43 "http://www.unicode.org/Public", 44 "URL of Unicode database directory") 45 iana = flag.String("iana", 46 "http://www.iana.org", 47 "URL of the IANA repository") 48 unicodeVersion = flag.String("unicode", 49 getEnv("UNICODE_VERSION", unicode.Version), 50 "unicode version to use") 51 cldrVersion = flag.String("cldr", 52 getEnv("CLDR_VERSION", cldr.Version), 53 "cldr version to use") 54) 55 56func getEnv(name, def string) string { 57 if v := os.Getenv(name); v != "" { 58 return v 59 } 60 return def 61} 62 63// Init performs common initialization for a gen command. It parses the flags 64// and sets up the standard logging parameters. 65func Init() { 66 log.SetPrefix("") 67 log.SetFlags(log.Lshortfile) 68 flag.Parse() 69} 70 71const header = `// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. 72 73` 74 75// UnicodeVersion reports the requested Unicode version. 76func UnicodeVersion() string { 77 return *unicodeVersion 78} 79 80// CLDRVersion reports the requested CLDR version. 81func CLDRVersion() string { 82 return *cldrVersion 83} 84 85var tags = []struct{ version, buildTags string }{ 86 {"10.0.0", "go1.10"}, 87 {"", "!go1.10"}, 88} 89 90// buildTags reports the build tags used for the current Unicode version. 91func buildTags() string { 92 v := UnicodeVersion() 93 for _, x := range tags { 94 // We should do a numeric comparison, but including the collate package 95 // would create an import cycle. We approximate it by assuming that 96 // longer version strings are later. 97 if len(x.version) <= len(v) { 98 return x.buildTags 99 } 100 if len(x.version) == len(v) && x.version <= v { 101 return x.buildTags 102 } 103 } 104 return tags[0].buildTags 105} 106 107// IsLocal reports whether data files are available locally. 108func IsLocal() bool { 109 dir, err := localReadmeFile() 110 if err != nil { 111 return false 112 } 113 if _, err = os.Stat(dir); err != nil { 114 return false 115 } 116 return true 117} 118 119// OpenUCDFile opens the requested UCD file. The file is specified relative to 120// the public Unicode root directory. It will call log.Fatal if there are any 121// errors. 122func OpenUCDFile(file string) io.ReadCloser { 123 return openUnicode(path.Join(*unicodeVersion, "ucd", file)) 124} 125 126// OpenCLDRCoreZip opens the CLDR core zip file. It will call log.Fatal if there 127// are any errors. 128func OpenCLDRCoreZip() io.ReadCloser { 129 return OpenUnicodeFile("cldr", *cldrVersion, "core.zip") 130} 131 132// OpenUnicodeFile opens the requested file of the requested category from the 133// root of the Unicode data archive. The file is specified relative to the 134// public Unicode root directory. If version is "", it will use the default 135// Unicode version. It will call log.Fatal if there are any errors. 136func OpenUnicodeFile(category, version, file string) io.ReadCloser { 137 if version == "" { 138 version = UnicodeVersion() 139 } 140 return openUnicode(path.Join(category, version, file)) 141} 142 143// OpenIANAFile opens the requested IANA file. The file is specified relative 144// to the IANA root, which is typically either http://www.iana.org or the 145// iana directory in the local mirror. It will call log.Fatal if there are any 146// errors. 147func OpenIANAFile(path string) io.ReadCloser { 148 return Open(*iana, "iana", path) 149} 150 151var ( 152 dirMutex sync.Mutex 153 localDir string 154) 155 156const permissions = 0755 157 158func localReadmeFile() (string, error) { 159 p, err := build.Import("golang.org/x/text", "", build.FindOnly) 160 if err != nil { 161 return "", fmt.Errorf("Could not locate package: %v", err) 162 } 163 return filepath.Join(p.Dir, "DATA", "README"), nil 164} 165 166func getLocalDir() string { 167 dirMutex.Lock() 168 defer dirMutex.Unlock() 169 170 readme, err := localReadmeFile() 171 if err != nil { 172 log.Fatal(err) 173 } 174 dir := filepath.Dir(readme) 175 if _, err := os.Stat(readme); err != nil { 176 if err := os.MkdirAll(dir, permissions); err != nil { 177 log.Fatalf("Could not create directory: %v", err) 178 } 179 ioutil.WriteFile(readme, []byte(readmeTxt), permissions) 180 } 181 return dir 182} 183 184const readmeTxt = `Generated by golang.org/x/text/internal/gen. DO NOT EDIT. 185 186This directory contains downloaded files used to generate the various tables 187in the golang.org/x/text subrepo. 188 189Note that the language subtag repo (iana/assignments/language-subtag-registry) 190and all other times in the iana subdirectory are not versioned and will need 191to be periodically manually updated. The easiest way to do this is to remove 192the entire iana directory. This is mostly of concern when updating the language 193package. 194` 195 196// Open opens subdir/path if a local directory is specified and the file exists, 197// where subdir is a directory relative to the local root, or fetches it from 198// urlRoot/path otherwise. It will call log.Fatal if there are any errors. 199func Open(urlRoot, subdir, path string) io.ReadCloser { 200 file := filepath.Join(getLocalDir(), subdir, filepath.FromSlash(path)) 201 return open(file, urlRoot, path) 202} 203 204func openUnicode(path string) io.ReadCloser { 205 file := filepath.Join(getLocalDir(), filepath.FromSlash(path)) 206 return open(file, *url, path) 207} 208 209// TODO: automatically periodically update non-versioned files. 210 211func open(file, urlRoot, path string) io.ReadCloser { 212 if f, err := os.Open(file); err == nil { 213 return f 214 } 215 r := get(urlRoot, path) 216 defer r.Close() 217 b, err := ioutil.ReadAll(r) 218 if err != nil { 219 log.Fatalf("Could not download file: %v", err) 220 } 221 os.MkdirAll(filepath.Dir(file), permissions) 222 if err := ioutil.WriteFile(file, b, permissions); err != nil { 223 log.Fatalf("Could not create file: %v", err) 224 } 225 return ioutil.NopCloser(bytes.NewReader(b)) 226} 227 228func get(root, path string) io.ReadCloser { 229 url := root + "/" + path 230 fmt.Printf("Fetching %s...", url) 231 defer fmt.Println(" done.") 232 resp, err := http.Get(url) 233 if err != nil { 234 log.Fatalf("HTTP GET: %v", err) 235 } 236 if resp.StatusCode != 200 { 237 log.Fatalf("Bad GET status for %q: %q", url, resp.Status) 238 } 239 return resp.Body 240} 241 242// TODO: use Write*Version in all applicable packages. 243 244// WriteUnicodeVersion writes a constant for the Unicode version from which the 245// tables are generated. 246func WriteUnicodeVersion(w io.Writer) { 247 fmt.Fprintf(w, "// UnicodeVersion is the Unicode version from which the tables in this package are derived.\n") 248 fmt.Fprintf(w, "const UnicodeVersion = %q\n\n", UnicodeVersion()) 249} 250 251// WriteCLDRVersion writes a constant for the CLDR version from which the 252// tables are generated. 253func WriteCLDRVersion(w io.Writer) { 254 fmt.Fprintf(w, "// CLDRVersion is the CLDR version from which the tables in this package are derived.\n") 255 fmt.Fprintf(w, "const CLDRVersion = %q\n\n", CLDRVersion()) 256} 257 258// WriteGoFile prepends a standard file comment and package statement to the 259// given bytes, applies gofmt, and writes them to a file with the given name. 260// It will call log.Fatal if there are any errors. 261func WriteGoFile(filename, pkg string, b []byte) { 262 w, err := os.Create(filename) 263 if err != nil { 264 log.Fatalf("Could not create file %s: %v", filename, err) 265 } 266 defer w.Close() 267 if _, err = WriteGo(w, pkg, "", b); err != nil { 268 log.Fatalf("Error writing file %s: %v", filename, err) 269 } 270} 271 272func insertVersion(filename, version string) string { 273 suffix := ".go" 274 if strings.HasSuffix(filename, "_test.go") { 275 suffix = "_test.go" 276 } 277 return fmt.Sprint(filename[:len(filename)-len(suffix)], version, suffix) 278} 279 280// WriteVersionedGoFile prepends a standard file comment, adds build tags to 281// version the file for the current Unicode version, and package statement to 282// the given bytes, applies gofmt, and writes them to a file with the given 283// name. It will call log.Fatal if there are any errors. 284func WriteVersionedGoFile(filename, pkg string, b []byte) { 285 tags := buildTags() 286 if tags != "" { 287 filename = insertVersion(filename, UnicodeVersion()) 288 } 289 w, err := os.Create(filename) 290 if err != nil { 291 log.Fatalf("Could not create file %s: %v", filename, err) 292 } 293 defer w.Close() 294 if _, err = WriteGo(w, pkg, tags, b); err != nil { 295 log.Fatalf("Error writing file %s: %v", filename, err) 296 } 297} 298 299// WriteGo prepends a standard file comment and package statement to the given 300// bytes, applies gofmt, and writes them to w. 301func WriteGo(w io.Writer, pkg, tags string, b []byte) (n int, err error) { 302 src := []byte(header) 303 if tags != "" { 304 src = append(src, fmt.Sprintf("// +build %s\n\n", tags)...) 305 } 306 src = append(src, fmt.Sprintf("package %s\n\n", pkg)...) 307 src = append(src, b...) 308 formatted, err := format.Source(src) 309 if err != nil { 310 // Print the generated code even in case of an error so that the 311 // returned error can be meaningfully interpreted. 312 n, _ = w.Write(src) 313 return n, err 314 } 315 return w.Write(formatted) 316} 317 318// Repackage rewrites a Go file from belonging to package main to belonging to 319// the given package. 320func Repackage(inFile, outFile, pkg string) { 321 src, err := ioutil.ReadFile(inFile) 322 if err != nil { 323 log.Fatalf("reading %s: %v", inFile, err) 324 } 325 const toDelete = "package main\n\n" 326 i := bytes.Index(src, []byte(toDelete)) 327 if i < 0 { 328 log.Fatalf("Could not find %q in %s.", toDelete, inFile) 329 } 330 w := &bytes.Buffer{} 331 w.Write(src[i+len(toDelete):]) 332 WriteGoFile(outFile, pkg, w.Bytes()) 333} 334