1// Copyright 2015 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5// +build ignore
6
7// gen runs go generate on Unicode- and CLDR-related package in the text
8// repositories, taking into account dependencies and versions.
9package main
10
11import (
12	"bytes"
13	"flag"
14	"fmt"
15	"go/build"
16	"go/format"
17	"io/ioutil"
18	"os"
19	"os/exec"
20	"path"
21	"path/filepath"
22	"regexp"
23	"runtime"
24	"strings"
25	"sync"
26	"unicode"
27
28	"golang.org/x/text/collate"
29	"golang.org/x/text/internal/gen"
30	"golang.org/x/text/language"
31)
32
33var (
34	verbose     = flag.Bool("v", false, "verbose output")
35	force       = flag.Bool("force", false, "ignore failing dependencies")
36	doCore      = flag.Bool("core", false, "force an update to core")
37	excludeList = flag.String("exclude", "",
38		"comma-separated list of packages to exclude")
39
40	// The user can specify a selection of packages to build on the command line.
41	args []string
42)
43
44func exclude(pkg string) bool {
45	if len(args) > 0 {
46		return !contains(args, pkg)
47	}
48	return contains(strings.Split(*excludeList, ","), pkg)
49}
50
51// TODO:
52// - Better version handling.
53// - Generate tables for the core unicode package?
54// - Add generation for encodings. This requires some retooling here and there.
55// - Running repo-wide "long" tests.
56
57var vprintf = fmt.Printf
58
59func main() {
60	gen.Init()
61	args = flag.Args()
62	if !*verbose {
63		// Set vprintf to a no-op.
64		vprintf = func(string, ...interface{}) (int, error) { return 0, nil }
65	}
66
67	// TODO: create temporary cache directory to load files and create and set
68	// a "cache" option if the user did not specify the UNICODE_DIR environment
69	// variable. This will prevent duplicate downloads and also will enable long
70	// tests, which really need to be run after each generated package.
71
72	updateCore := *doCore
73	if gen.UnicodeVersion() != unicode.Version {
74		fmt.Printf("Requested Unicode version %s; core unicode version is %s.\n",
75			gen.UnicodeVersion(),
76			unicode.Version)
77		c := collate.New(language.Und, collate.Numeric)
78		if c.CompareString(gen.UnicodeVersion(), unicode.Version) < 0 && !*force {
79			os.Exit(2)
80		}
81		updateCore = true
82		goroot := os.Getenv("GOROOT")
83		appendToFile(
84			filepath.Join(goroot, "api", "except.txt"),
85			fmt.Sprintf("pkg unicode, const Version = %q\n", unicode.Version),
86		)
87		const lines = `pkg unicode, const Version = %q
88// TODO: add a new line of the following form for each new script and property.
89pkg unicode, var <new script or property> *RangeTable
90`
91		appendToFile(
92			filepath.Join(goroot, "api", "next.txt"),
93			fmt.Sprintf(lines, gen.UnicodeVersion()),
94		)
95	}
96
97	var unicode = &dependency{}
98	if updateCore {
99		fmt.Printf("Updating core to version %s...\n", gen.UnicodeVersion())
100		unicode = generate("unicode")
101
102		// Test some users of the unicode packages, especially the ones that
103		// keep a mirrored table. These may need to be corrected by hand.
104		generate("regexp", unicode)
105		generate("strconv", unicode) // mimics Unicode table
106		generate("strings", unicode)
107		generate("testing", unicode) // mimics Unicode table
108	}
109
110	var (
111		cldr       = generate("./unicode/cldr", unicode)
112		language   = generate("./language", cldr)
113		internal   = generate("./internal", unicode, language)
114		norm       = generate("./unicode/norm", unicode)
115		rangetable = generate("./unicode/rangetable", unicode)
116		cases      = generate("./cases", unicode, norm, language, rangetable)
117		width      = generate("./width", unicode)
118		bidi       = generate("./unicode/bidi", unicode, norm, rangetable)
119		mib        = generate("./encoding/internal/identifier", unicode)
120		number     = generate("./internal/number", unicode, cldr, language, internal)
121		_          = generate("./encoding/htmlindex", unicode, language, mib)
122		_          = generate("./encoding/ianaindex", unicode, language, mib)
123		_          = generate("./secure/precis", unicode, norm, rangetable, cases, width, bidi)
124		_          = generate("./internal/cldrtree", language)
125		_          = generate("./currency", unicode, cldr, language, internal, number)
126		_          = generate("./feature/plural", unicode, cldr, language, internal, number)
127		_          = generate("./internal/export/idna", unicode, bidi, norm)
128		_          = generate("./language/display", unicode, cldr, language, internal, number)
129		_          = generate("./collate", unicode, norm, cldr, language, rangetable)
130		_          = generate("./search", unicode, norm, cldr, language, rangetable)
131	)
132	all.Wait()
133
134	// Copy exported packages to the destination golang.org repo.
135	copyExported("golang.org/x/net/idna")
136
137	if updateCore {
138		copyVendored()
139	}
140
141	if hasErrors {
142		fmt.Println("FAIL")
143		os.Exit(1)
144	}
145	vprintf("SUCCESS\n")
146}
147
148func appendToFile(file, text string) {
149	fmt.Println("Augmenting", file)
150	w, err := os.OpenFile(file, os.O_APPEND|os.O_WRONLY, 0600)
151	if err != nil {
152		fmt.Println("Failed to open file:", err)
153		os.Exit(1)
154	}
155	defer w.Close()
156	if _, err := w.WriteString(text); err != nil {
157		fmt.Println("Failed to write to file:", err)
158		os.Exit(1)
159	}
160}
161
162var (
163	all       sync.WaitGroup
164	hasErrors bool
165)
166
167type dependency struct {
168	sync.WaitGroup
169	hasErrors bool
170}
171
172func generate(pkg string, deps ...*dependency) *dependency {
173	var wg dependency
174	if exclude(pkg) {
175		return &wg
176	}
177	wg.Add(1)
178	all.Add(1)
179	go func() {
180		defer wg.Done()
181		defer all.Done()
182		// Wait for dependencies to finish.
183		for _, d := range deps {
184			d.Wait()
185			if d.hasErrors && !*force {
186				fmt.Printf("--- ABORT: %s\n", pkg)
187				wg.hasErrors = true
188				return
189			}
190		}
191		vprintf("=== GENERATE %s\n", pkg)
192		args := []string{"generate"}
193		if *verbose {
194			args = append(args, "-v")
195		}
196		args = append(args, pkg)
197		cmd := exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), args...)
198		w := &bytes.Buffer{}
199		cmd.Stderr = w
200		cmd.Stdout = w
201		if err := cmd.Run(); err != nil {
202			fmt.Printf("--- FAIL: %s:\n\t%v\n\tError: %v\n", pkg, indent(w), err)
203			hasErrors = true
204			wg.hasErrors = true
205			return
206		}
207
208		vprintf("=== TEST %s\n", pkg)
209		args[0] = "test"
210		cmd = exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), args...)
211		wt := &bytes.Buffer{}
212		cmd.Stderr = wt
213		cmd.Stdout = wt
214		if err := cmd.Run(); err != nil {
215			fmt.Printf("--- FAIL: %s:\n\t%v\n\tError: %v\n", pkg, indent(wt), err)
216			hasErrors = true
217			wg.hasErrors = true
218			return
219		}
220		vprintf("--- SUCCESS: %s\n\t%v\n", pkg, indent(w))
221		fmt.Print(wt.String())
222	}()
223	return &wg
224}
225
226// copyExported copies a package in x/text/internal/export to the
227// destination repository.
228func copyExported(p string) {
229	copyPackage(
230		filepath.Join("internal", "export", path.Base(p)),
231		filepath.Join("..", filepath.FromSlash(p[len("golang.org/x"):])),
232		"golang.org/x/text/internal/export/"+path.Base(p),
233		p)
234}
235
236// copyVendored copies packages used by Go core into the vendored directory.
237func copyVendored() {
238	root := filepath.Join(build.Default.GOROOT, filepath.FromSlash("src/vendor/golang_org/x"))
239
240	err := filepath.Walk(root, func(dir string, info os.FileInfo, err error) error {
241		if err != nil || !info.IsDir() || root == dir {
242			return err
243		}
244		src := dir[len(root)+1:]
245		const slash = string(filepath.Separator)
246		if c := strings.Split(src, slash); c[0] == "text" {
247			// Copy a text repo package from its normal location.
248			src = strings.Join(c[1:], slash)
249		} else {
250			// Copy the vendored package if it exists in the export directory.
251			src = filepath.Join("internal", "export", filepath.Base(src))
252		}
253		copyPackage(src, dir, "golang.org", "golang_org")
254		return nil
255	})
256	if err != nil {
257		fmt.Printf("Seeding directory %s has failed %v:", root, err)
258		os.Exit(1)
259	}
260}
261
262// goGenRE is used to remove go:generate lines.
263var goGenRE = regexp.MustCompile("//go:generate[^\n]*\n")
264
265// copyPackage copies relevant files from a directory in x/text to the
266// destination package directory. The destination package is assumed to have
267// the same name. For each copied file go:generate lines are removed and
268// and package comments are rewritten to the new path.
269func copyPackage(dirSrc, dirDst, search, replace string) {
270	err := filepath.Walk(dirSrc, func(file string, info os.FileInfo, err error) error {
271		base := filepath.Base(file)
272		if err != nil || info.IsDir() ||
273			!strings.HasSuffix(base, ".go") ||
274			strings.HasSuffix(base, "_test.go") ||
275			// Don't process subdirectories.
276			filepath.Dir(file) != dirSrc {
277			return nil
278		}
279		b, err := ioutil.ReadFile(file)
280		if err != nil || bytes.Contains(b, []byte("\n// +build ignore")) {
281			return err
282		}
283		// Fix paths.
284		b = bytes.Replace(b, []byte(search), []byte(replace), -1)
285		// Remove go:generate lines.
286		b = goGenRE.ReplaceAllLiteral(b, nil)
287		comment := "// Code generated by running \"go generate\" in golang.org/x/text. DO NOT EDIT.\n\n"
288		if *doCore {
289			comment = "// Code generated by running \"go run gen.go -core\" in golang.org/x/text. DO NOT EDIT.\n\n"
290		}
291		if !bytes.HasPrefix(b, []byte(comment)) {
292			b = append([]byte(comment), b...)
293		}
294		if b, err = format.Source(b); err != nil {
295			fmt.Println("Failed to format file:", err)
296			os.Exit(1)
297		}
298		file = filepath.Join(dirDst, base)
299		vprintf("=== COPY %s\n", file)
300		return ioutil.WriteFile(file, b, 0666)
301	})
302	if err != nil {
303		fmt.Println("Copying exported files failed:", err)
304		os.Exit(1)
305	}
306}
307
308func contains(a []string, s string) bool {
309	for _, e := range a {
310		if s == e {
311			return true
312		}
313	}
314	return false
315}
316
317func indent(b *bytes.Buffer) string {
318	return strings.Replace(strings.TrimSpace(b.String()), "\n", "\n\t", -1)
319}
320