1// Copyright 2015 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5//go:build ignore
6// +build ignore
7
8// gen runs go generate on Unicode- and CLDR-related package in the text
9// repositories, taking into account dependencies and versions.
10package main
11
12import (
13	"bytes"
14	"flag"
15	"fmt"
16	"go/format"
17	"io/ioutil"
18	"os"
19	"os/exec"
20	"path"
21	"path/filepath"
22	"regexp"
23	"runtime"
24	"strings"
25	"sync"
26	"unicode"
27
28	"golang.org/x/text/collate"
29	"golang.org/x/text/internal/gen"
30	"golang.org/x/text/language"
31)
32
33var (
34	verbose     = flag.Bool("v", false, "verbose output")
35	force       = flag.Bool("force", false, "ignore failing dependencies")
36	doCore      = flag.Bool("core", false, "force an update to core")
37	skipTest    = flag.Bool("skiptest", false, "skip tests")
38	excludeList = flag.String("exclude", "",
39		"comma-separated list of packages to exclude")
40
41	// The user can specify a selection of packages to build on the command line.
42	args []string
43)
44
45func exclude(pkg string) bool {
46	if len(args) > 0 {
47		return !contains(args, pkg)
48	}
49	return contains(strings.Split(*excludeList, ","), pkg)
50}
51
52// TODO:
53// - Better version handling.
54// - Generate tables for the core unicode package?
55// - Add generation for encodings. This requires some retooling here and there.
56// - Running repo-wide "long" tests.
57
58var vprintf = fmt.Printf
59
60func main() {
61	gen.Init()
62	args = flag.Args()
63	if !*verbose {
64		// Set vprintf to a no-op.
65		vprintf = func(string, ...interface{}) (int, error) { return 0, nil }
66	}
67
68	// TODO: create temporary cache directory to load files and create and set
69	// a "cache" option if the user did not specify the UNICODE_DIR environment
70	// variable. This will prevent duplicate downloads and also will enable long
71	// tests, which really need to be run after each generated package.
72
73	updateCore := *doCore
74	if gen.UnicodeVersion() != unicode.Version {
75		fmt.Printf("Requested Unicode version %s; core unicode version is %s.\n",
76			gen.UnicodeVersion(),
77			unicode.Version)
78		c := collate.New(language.Und, collate.Numeric)
79		if c.CompareString(gen.UnicodeVersion(), unicode.Version) < 0 && !*force {
80			os.Exit(2)
81		}
82		updateCore = true
83		goroot := os.Getenv("GOROOT")
84		appendToFile(
85			filepath.Join(goroot, "api", "except.txt"),
86			fmt.Sprintf("pkg unicode, const Version = %q\n", unicode.Version),
87		)
88		const lines = `pkg unicode, const Version = %q
89// TODO: add a new line of the following form for each new script and property.
90pkg unicode, var <new script or property> *RangeTable
91`
92		appendToFile(
93			filepath.Join(goroot, "api", "next.txt"),
94			fmt.Sprintf(lines, gen.UnicodeVersion()),
95		)
96	}
97
98	var unicode = &dependency{}
99	if updateCore {
100		fmt.Printf("Updating core to version %s...\n", gen.UnicodeVersion())
101		unicodeInternal := generate("./internal/export/unicode")
102		unicode = generate("unicode", unicodeInternal)
103
104		// Test some users of the unicode packages, especially the ones that
105		// keep a mirrored table. These may need to be corrected by hand.
106		generate("regexp", unicode)
107		generate("strconv", unicode) // mimics Unicode table
108		generate("strings", unicode)
109		generate("testing", unicode) // mimics Unicode table
110	}
111
112	var (
113		cldr       = generate("./unicode/cldr", unicode)
114		intlang    = generate("./internal/language", cldr)
115		compact    = generate("./internal/language/compact", intlang, cldr)
116		language   = generate("./language", cldr, compact)
117		internal   = generate("./internal", unicode, language)
118		norm       = generate("./unicode/norm", unicode)
119		rangetable = generate("./unicode/rangetable", unicode)
120		cases      = generate("./cases", unicode, norm, language, rangetable)
121		width      = generate("./width", unicode)
122		bidi       = generate("./unicode/bidi", unicode, norm, rangetable)
123		mib        = generate("./encoding/internal/identifier", unicode)
124		number     = generate("./internal/number", unicode, cldr, language, internal)
125		cldrtree   = generate("./internal/cldrtree", language, internal)
126		_          = generate("./unicode/runenames", unicode)
127		_          = generate("./encoding/htmlindex", unicode, language, mib)
128		_          = generate("./encoding/ianaindex", unicode, language, mib)
129		_          = generate("./secure/precis", unicode, norm, rangetable, cases, width, bidi)
130		_          = generate("./currency", unicode, cldr, language, internal, number)
131		_          = generate("./feature/plural", unicode, cldr, language, internal, number)
132		_          = generate("./internal/export/idna", unicode, bidi, norm)
133		_          = generate("./language/display", unicode, cldr, language, internal, number)
134		_          = generate("./collate", unicode, norm, cldr, language, rangetable)
135		_          = generate("./search", unicode, norm, cldr, language, rangetable)
136		_          = generate("./date", cldr, language, cldrtree)
137	)
138	all.Wait()
139
140	// Copy exported packages to the destination golang.org repo.
141	copyExported("golang.org/x/net/idna")
142
143	if hasErrors {
144		fmt.Println("FAIL")
145		os.Exit(1)
146	}
147	vprintf("SUCCESS\n")
148}
149
150func appendToFile(file, text string) {
151	fmt.Println("Augmenting", file)
152	w, err := os.OpenFile(file, os.O_APPEND|os.O_WRONLY, 0600)
153	if err != nil {
154		fmt.Println("Failed to open file:", err)
155		os.Exit(1)
156	}
157	defer w.Close()
158	if _, err := w.WriteString(text); err != nil {
159		fmt.Println("Failed to write to file:", err)
160		os.Exit(1)
161	}
162}
163
164var (
165	all       sync.WaitGroup
166	hasErrors bool
167)
168
169type dependency struct {
170	sync.WaitGroup
171	hasErrors bool
172}
173
174func generate(pkg string, deps ...*dependency) *dependency {
175	var wg dependency
176	if exclude(pkg) {
177		return &wg
178	}
179	wg.Add(1)
180	all.Add(1)
181	go func() {
182		defer wg.Done()
183		defer all.Done()
184		// Wait for dependencies to finish.
185		for _, d := range deps {
186			d.Wait()
187			if d.hasErrors && !*force {
188				fmt.Printf("--- ABORT: %s\n", pkg)
189				wg.hasErrors = true
190				return
191			}
192		}
193		vprintf("=== GENERATE %s\n", pkg)
194		args := []string{"generate"}
195		if *verbose {
196			args = append(args, "-v")
197		}
198		args = append(args, pkg)
199		cmd := exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), args...)
200		w := &bytes.Buffer{}
201		cmd.Stderr = w
202		cmd.Stdout = w
203		if err := cmd.Run(); err != nil {
204			fmt.Printf("--- FAIL: %s:\n\t%v\n\tError: %v\n", pkg, indent(w), err)
205			hasErrors = true
206			wg.hasErrors = true
207			return
208		}
209
210		if *skipTest {
211			return
212		}
213
214		vprintf("=== TEST %s\n", pkg)
215		args[0] = "test"
216		cmd = exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), args...)
217		wt := &bytes.Buffer{}
218		cmd.Stderr = wt
219		cmd.Stdout = wt
220		if err := cmd.Run(); err != nil {
221			fmt.Printf("--- FAIL: %s:\n\t%v\n\tError: %v\n", pkg, indent(wt), err)
222			hasErrors = true
223			wg.hasErrors = true
224			return
225		}
226		vprintf("--- SUCCESS: %s\n\t%v\n", pkg, indent(w))
227		fmt.Print(wt.String())
228	}()
229	return &wg
230}
231
232// copyExported copies a package in x/text/internal/export to the
233// destination repository.
234func copyExported(p string) {
235	copyPackage(
236		filepath.Join("internal", "export", path.Base(p)),
237		filepath.Join("..", filepath.FromSlash(p[len("golang.org/x"):])),
238		"golang.org/x/text/internal/export/"+path.Base(p),
239		p)
240}
241
242// goGenRE is used to remove go:generate lines.
243var goGenRE = regexp.MustCompile("//go:generate[^\n]*\n")
244
245// copyPackage copies relevant files from a directory in x/text to the
246// destination package directory. The destination package is assumed to have
247// the same name. For each copied file go:generate lines are removed and
248// package comments are rewritten to the new path.
249func copyPackage(dirSrc, dirDst, search, replace string) {
250	err := filepath.Walk(dirSrc, func(file string, info os.FileInfo, err error) error {
251		base := filepath.Base(file)
252		if err != nil || info.IsDir() ||
253			!strings.HasSuffix(base, ".go") ||
254			strings.HasSuffix(base, "_test.go") ||
255			// Don't process subdirectories.
256			filepath.Dir(file) != dirSrc {
257			return nil
258		}
259		if strings.HasPrefix(base, "tables") {
260			if !strings.HasSuffix(base, gen.UnicodeVersion()+".go") {
261				return nil
262			}
263			base = "tables.go"
264		}
265		b, err := ioutil.ReadFile(file)
266		if err != nil || bytes.Contains(b, []byte("\n// +build ignore")) {
267			return err
268		}
269		// Fix paths.
270		b = bytes.Replace(b, []byte(search), []byte(replace), -1)
271		b = bytes.Replace(b, []byte("internal/export"), []byte(""), -1)
272		// Remove go:generate lines.
273		b = goGenRE.ReplaceAllLiteral(b, nil)
274		comment := "// Code generated by running \"go generate\" in golang.org/x/text. DO NOT EDIT.\n\n"
275		if !bytes.HasPrefix(b, []byte(comment)) {
276			b = append([]byte(comment), b...)
277		}
278		if b, err = format.Source(b); err != nil {
279			fmt.Println("Failed to format file:", err)
280			os.Exit(1)
281		}
282		file = filepath.Join(dirDst, base)
283		vprintf("=== COPY %s\n", file)
284		return ioutil.WriteFile(file, b, 0666)
285	})
286	if err != nil {
287		fmt.Println("Copying exported files failed:", err)
288		os.Exit(1)
289	}
290}
291
292func contains(a []string, s string) bool {
293	for _, e := range a {
294		if s == e {
295			return true
296		}
297	}
298	return false
299}
300
301func indent(b *bytes.Buffer) string {
302	return strings.Replace(strings.TrimSpace(b.String()), "\n", "\n\t", -1)
303}
304