1// Copyright 2013 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5//go:build ignore 6// +build ignore 7 8package main 9 10// This program generates tables.go: 11// go run maketables.go | gofmt > tables.go 12 13// TODO: Emoji extensions? 14// https://www.unicode.org/faq/emoji_dingbats.html 15// https://www.unicode.org/Public/UNIDATA/EmojiSources.txt 16 17import ( 18 "bufio" 19 "fmt" 20 "log" 21 "net/http" 22 "sort" 23 "strings" 24) 25 26type entry struct { 27 jisCode, table int 28} 29 30func main() { 31 fmt.Printf("// generated by go run maketables.go; DO NOT EDIT\n\n") 32 fmt.Printf("// Package japanese provides Japanese encodings such as EUC-JP and Shift JIS.\n") 33 fmt.Printf(`package japanese // import "golang.org/x/text/encoding/japanese"` + "\n\n") 34 35 reverse := [65536]entry{} 36 for i := range reverse { 37 reverse[i].table = -1 38 } 39 40 tables := []struct { 41 url string 42 name string 43 }{ 44 {"http://encoding.spec.whatwg.org/index-jis0208.txt", "0208"}, 45 {"http://encoding.spec.whatwg.org/index-jis0212.txt", "0212"}, 46 } 47 for i, table := range tables { 48 res, err := http.Get(table.url) 49 if err != nil { 50 log.Fatalf("%q: Get: %v", table.url, err) 51 } 52 defer res.Body.Close() 53 54 mapping := [65536]uint16{} 55 56 scanner := bufio.NewScanner(res.Body) 57 for scanner.Scan() { 58 s := strings.TrimSpace(scanner.Text()) 59 if s == "" || s[0] == '#' { 60 continue 61 } 62 x, y := 0, uint16(0) 63 if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil { 64 log.Fatalf("%q: could not parse %q", table.url, s) 65 } 66 if x < 0 || 120*94 <= x { 67 log.Fatalf("%q: JIS code %d is out of range", table.url, x) 68 } 69 mapping[x] = y 70 if reverse[y].table == -1 { 71 reverse[y] = entry{jisCode: x, table: i} 72 } 73 } 74 if err := scanner.Err(); err != nil { 75 log.Fatalf("%q: scanner error: %v", table.url, err) 76 } 77 78 fmt.Printf("// jis%sDecode is the decoding table from JIS %s code to Unicode.\n// It is defined at %s\n", 79 table.name, table.name, table.url) 80 fmt.Printf("var jis%sDecode = [...]uint16{\n", table.name) 81 for i, m := range mapping { 82 if m != 0 { 83 fmt.Printf("\t%d: 0x%04X,\n", i, m) 84 } 85 } 86 fmt.Printf("}\n\n") 87 } 88 89 // Any run of at least separation continuous zero entries in the reverse map will 90 // be a separate encode table. 91 const separation = 1024 92 93 intervals := []interval(nil) 94 low, high := -1, -1 95 for i, v := range reverse { 96 if v.table == -1 { 97 continue 98 } 99 if low < 0 { 100 low = i 101 } else if i-high >= separation { 102 if high >= 0 { 103 intervals = append(intervals, interval{low, high}) 104 } 105 low = i 106 } 107 high = i + 1 108 } 109 if high >= 0 { 110 intervals = append(intervals, interval{low, high}) 111 } 112 sort.Sort(byDecreasingLength(intervals)) 113 114 fmt.Printf("const (\n") 115 fmt.Printf("\tjis0208 = 1\n") 116 fmt.Printf("\tjis0212 = 2\n") 117 fmt.Printf("\tcodeMask = 0x7f\n") 118 fmt.Printf("\tcodeShift = 7\n") 119 fmt.Printf("\ttableShift = 14\n") 120 fmt.Printf(")\n\n") 121 122 fmt.Printf("const numEncodeTables = %d\n\n", len(intervals)) 123 fmt.Printf("// encodeX are the encoding tables from Unicode to JIS code,\n") 124 fmt.Printf("// sorted by decreasing length.\n") 125 for i, v := range intervals { 126 fmt.Printf("// encode%d: %5d entries for runes in [%5d, %5d).\n", i, v.len(), v.low, v.high) 127 } 128 fmt.Printf("//\n") 129 fmt.Printf("// The high two bits of the value record whether the JIS code comes from the\n") 130 fmt.Printf("// JIS0208 table (high bits == 1) or the JIS0212 table (high bits == 2).\n") 131 fmt.Printf("// The low 14 bits are two 7-bit unsigned integers j1 and j2 that form the\n") 132 fmt.Printf("// JIS code (94*j1 + j2) within that table.\n") 133 fmt.Printf("\n") 134 135 for i, v := range intervals { 136 fmt.Printf("const encode%dLow, encode%dHigh = %d, %d\n\n", i, i, v.low, v.high) 137 fmt.Printf("var encode%d = [...]uint16{\n", i) 138 for j := v.low; j < v.high; j++ { 139 x := reverse[j] 140 if x.table == -1 { 141 continue 142 } 143 fmt.Printf("\t%d - %d: jis%s<<14 | 0x%02X<<7 | 0x%02X,\n", 144 j, v.low, tables[x.table].name, x.jisCode/94, x.jisCode%94) 145 } 146 fmt.Printf("}\n\n") 147 } 148} 149 150// interval is a half-open interval [low, high). 151type interval struct { 152 low, high int 153} 154 155func (i interval) len() int { return i.high - i.low } 156 157// byDecreasingLength sorts intervals by decreasing length. 158type byDecreasingLength []interval 159 160func (b byDecreasingLength) Len() int { return len(b) } 161func (b byDecreasingLength) Less(i, j int) bool { return b[i].len() > b[j].len() } 162func (b byDecreasingLength) Swap(i, j int) { b[i], b[j] = b[j], b[i] } 163