1// Copyright 2015 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5// +build ignore
6
7// gen runs go generate on Unicode- and CLDR-related package in the text
8// repositories, taking into account dependencies and versions.
9package main
10
11import (
12	"bytes"
13	"flag"
14	"fmt"
15	"go/format"
16	"io/ioutil"
17	"os"
18	"os/exec"
19	"path"
20	"path/filepath"
21	"regexp"
22	"runtime"
23	"strings"
24	"sync"
25	"unicode"
26
27	"golang.org/x/text/collate"
28	"golang.org/x/text/internal/gen"
29	"golang.org/x/text/language"
30)
31
32var (
33	verbose     = flag.Bool("v", false, "verbose output")
34	force       = flag.Bool("force", false, "ignore failing dependencies")
35	doCore      = flag.Bool("core", false, "force an update to core")
36	skipTest    = flag.Bool("skiptest", false, "skip tests")
37	excludeList = flag.String("exclude", "",
38		"comma-separated list of packages to exclude")
39
40	// The user can specify a selection of packages to build on the command line.
41	args []string
42)
43
44func exclude(pkg string) bool {
45	if len(args) > 0 {
46		return !contains(args, pkg)
47	}
48	return contains(strings.Split(*excludeList, ","), pkg)
49}
50
51// TODO:
52// - Better version handling.
53// - Generate tables for the core unicode package?
54// - Add generation for encodings. This requires some retooling here and there.
55// - Running repo-wide "long" tests.
56
57var vprintf = fmt.Printf
58
59func main() {
60	gen.Init()
61	args = flag.Args()
62	if !*verbose {
63		// Set vprintf to a no-op.
64		vprintf = func(string, ...interface{}) (int, error) { return 0, nil }
65	}
66
67	// TODO: create temporary cache directory to load files and create and set
68	// a "cache" option if the user did not specify the UNICODE_DIR environment
69	// variable. This will prevent duplicate downloads and also will enable long
70	// tests, which really need to be run after each generated package.
71
72	updateCore := *doCore
73	if gen.UnicodeVersion() != unicode.Version {
74		fmt.Printf("Requested Unicode version %s; core unicode version is %s.\n",
75			gen.UnicodeVersion(),
76			unicode.Version)
77		c := collate.New(language.Und, collate.Numeric)
78		if c.CompareString(gen.UnicodeVersion(), unicode.Version) < 0 && !*force {
79			os.Exit(2)
80		}
81		updateCore = true
82		goroot := os.Getenv("GOROOT")
83		appendToFile(
84			filepath.Join(goroot, "api", "except.txt"),
85			fmt.Sprintf("pkg unicode, const Version = %q\n", unicode.Version),
86		)
87		const lines = `pkg unicode, const Version = %q
88// TODO: add a new line of the following form for each new script and property.
89pkg unicode, var <new script or property> *RangeTable
90`
91		appendToFile(
92			filepath.Join(goroot, "api", "next.txt"),
93			fmt.Sprintf(lines, gen.UnicodeVersion()),
94		)
95	}
96
97	var unicode = &dependency{}
98	if updateCore {
99		fmt.Printf("Updating core to version %s...\n", gen.UnicodeVersion())
100		unicodeInternal := generate("./internal/export/unicode")
101		unicode = generate("unicode", unicodeInternal)
102
103		// Test some users of the unicode packages, especially the ones that
104		// keep a mirrored table. These may need to be corrected by hand.
105		generate("regexp", unicode)
106		generate("strconv", unicode) // mimics Unicode table
107		generate("strings", unicode)
108		generate("testing", unicode) // mimics Unicode table
109	}
110
111	var (
112		cldr       = generate("./unicode/cldr", unicode)
113		compact    = generate("./internal/language/compact", cldr)
114		language   = generate("./language", cldr, compact)
115		internal   = generate("./internal", unicode, language)
116		norm       = generate("./unicode/norm", unicode)
117		rangetable = generate("./unicode/rangetable", unicode)
118		cases      = generate("./cases", unicode, norm, language, rangetable)
119		width      = generate("./width", unicode)
120		bidi       = generate("./unicode/bidi", unicode, norm, rangetable)
121		mib        = generate("./encoding/internal/identifier", unicode)
122		number     = generate("./internal/number", unicode, cldr, language, internal)
123		cldrtree   = generate("./internal/cldrtree", language, internal)
124		_          = generate("./unicode/runenames", unicode)
125		_          = generate("./encoding/htmlindex", unicode, language, mib)
126		_          = generate("./encoding/ianaindex", unicode, language, mib)
127		_          = generate("./secure/precis", unicode, norm, rangetable, cases, width, bidi)
128		_          = generate("./currency", unicode, cldr, language, internal, number)
129		_          = generate("./feature/plural", unicode, cldr, language, internal, number)
130		_          = generate("./internal/export/idna", unicode, bidi, norm)
131		_          = generate("./language/display", unicode, cldr, language, internal, number)
132		_          = generate("./collate", unicode, norm, cldr, language, rangetable)
133		_          = generate("./search", unicode, norm, cldr, language, rangetable)
134		_          = generate("./date", cldr, language, cldrtree)
135	)
136	all.Wait()
137
138	// Copy exported packages to the destination golang.org repo.
139	copyExported("golang.org/x/net/idna")
140
141	if hasErrors {
142		fmt.Println("FAIL")
143		os.Exit(1)
144	}
145	vprintf("SUCCESS\n")
146}
147
148func appendToFile(file, text string) {
149	fmt.Println("Augmenting", file)
150	w, err := os.OpenFile(file, os.O_APPEND|os.O_WRONLY, 0600)
151	if err != nil {
152		fmt.Println("Failed to open file:", err)
153		os.Exit(1)
154	}
155	defer w.Close()
156	if _, err := w.WriteString(text); err != nil {
157		fmt.Println("Failed to write to file:", err)
158		os.Exit(1)
159	}
160}
161
162var (
163	all       sync.WaitGroup
164	hasErrors bool
165)
166
167type dependency struct {
168	sync.WaitGroup
169	hasErrors bool
170}
171
172func generate(pkg string, deps ...*dependency) *dependency {
173	var wg dependency
174	if exclude(pkg) {
175		return &wg
176	}
177	wg.Add(1)
178	all.Add(1)
179	go func() {
180		defer wg.Done()
181		defer all.Done()
182		// Wait for dependencies to finish.
183		for _, d := range deps {
184			d.Wait()
185			if d.hasErrors && !*force {
186				fmt.Printf("--- ABORT: %s\n", pkg)
187				wg.hasErrors = true
188				return
189			}
190		}
191		vprintf("=== GENERATE %s\n", pkg)
192		args := []string{"generate"}
193		if *verbose {
194			args = append(args, "-v")
195		}
196		args = append(args, pkg)
197		cmd := exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), args...)
198		w := &bytes.Buffer{}
199		cmd.Stderr = w
200		cmd.Stdout = w
201		if err := cmd.Run(); err != nil {
202			fmt.Printf("--- FAIL: %s:\n\t%v\n\tError: %v\n", pkg, indent(w), err)
203			hasErrors = true
204			wg.hasErrors = true
205			return
206		}
207
208		if *skipTest {
209			return
210		}
211
212		vprintf("=== TEST %s\n", pkg)
213		args[0] = "test"
214		cmd = exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), args...)
215		wt := &bytes.Buffer{}
216		cmd.Stderr = wt
217		cmd.Stdout = wt
218		if err := cmd.Run(); err != nil {
219			fmt.Printf("--- FAIL: %s:\n\t%v\n\tError: %v\n", pkg, indent(wt), err)
220			hasErrors = true
221			wg.hasErrors = true
222			return
223		}
224		vprintf("--- SUCCESS: %s\n\t%v\n", pkg, indent(w))
225		fmt.Print(wt.String())
226	}()
227	return &wg
228}
229
230// copyExported copies a package in x/text/internal/export to the
231// destination repository.
232func copyExported(p string) {
233	copyPackage(
234		filepath.Join("internal", "export", path.Base(p)),
235		filepath.Join("..", filepath.FromSlash(p[len("golang.org/x"):])),
236		"golang.org/x/text/internal/export/"+path.Base(p),
237		p)
238}
239
240// goGenRE is used to remove go:generate lines.
241var goGenRE = regexp.MustCompile("//go:generate[^\n]*\n")
242
243// copyPackage copies relevant files from a directory in x/text to the
244// destination package directory. The destination package is assumed to have
245// the same name. For each copied file go:generate lines are removed and
246// package comments are rewritten to the new path.
247func copyPackage(dirSrc, dirDst, search, replace string) {
248	err := filepath.Walk(dirSrc, func(file string, info os.FileInfo, err error) error {
249		base := filepath.Base(file)
250		if err != nil || info.IsDir() ||
251			!strings.HasSuffix(base, ".go") ||
252			strings.HasSuffix(base, "_test.go") ||
253			// Don't process subdirectories.
254			filepath.Dir(file) != dirSrc {
255			return nil
256		}
257		if strings.HasPrefix(base, "tables") {
258			if !strings.HasSuffix(base, gen.UnicodeVersion()+".go") {
259				return nil
260			}
261			base = "tables.go"
262		}
263		b, err := ioutil.ReadFile(file)
264		if err != nil || bytes.Contains(b, []byte("\n// +build ignore")) {
265			return err
266		}
267		// Fix paths.
268		b = bytes.Replace(b, []byte(search), []byte(replace), -1)
269		b = bytes.Replace(b, []byte("internal/export"), []byte(""), -1)
270		// Remove go:generate lines.
271		b = goGenRE.ReplaceAllLiteral(b, nil)
272		comment := "// Code generated by running \"go generate\" in golang.org/x/text. DO NOT EDIT.\n\n"
273		if !bytes.HasPrefix(b, []byte(comment)) {
274			b = append([]byte(comment), b...)
275		}
276		if b, err = format.Source(b); err != nil {
277			fmt.Println("Failed to format file:", err)
278			os.Exit(1)
279		}
280		file = filepath.Join(dirDst, base)
281		vprintf("=== COPY %s\n", file)
282		return ioutil.WriteFile(file, b, 0666)
283	})
284	if err != nil {
285		fmt.Println("Copying exported files failed:", err)
286		os.Exit(1)
287	}
288}
289
290func contains(a []string, s string) bool {
291	for _, e := range a {
292		if s == e {
293			return true
294		}
295	}
296	return false
297}
298
299func indent(b *bytes.Buffer) string {
300	return strings.Replace(strings.TrimSpace(b.String()), "\n", "\n\t", -1)
301}
302