Skip to content

G
go
Commit 94e69155 authored by Rob Pike's avatar Rob Pike
Browse files
Options

unicode tables for all categories 

maketables now has a --test option to verify the data.

R=rsc
DELTA=3087  (1982 added, 1001 deleted, 104 changed)
OCL=33947
CL=33950
parent 5cbc96d9
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 2086 additions and 441 deletions
src/pkg/unicode/Makefile
View file @ 94e69155
......@@ -7,15 +7,22 @@ include $(GOROOT)/src/Make.$(GOARCH)
TARG=unicode
GOFILES=\
digit.go\
digittables.go\
letter.go\
lettertables.go\
tables.go\
include $(GOROOT)/src/Make.pkg
tables:
maketables: maketables.go
$(GC) maketables.go
$(LD) -o maketables maketables.$O
maketables --digits > digittables.go
maketables > lettertables.go
tables: maketables
$(GC) maketables.go
$(LD) -o maketables maketables.$O
./maketables --tables=all > tables.go
rm -f maketables
testtables: maketables
echo '***' Be sure to make tables and make install first
./maketables -test
rm -f maketables
src/pkg/unicode/digit_test.go
View file @ 94e69155
This diff is collapsed.
src/pkg/unicode/digittables.go deleted 100644 → 0
View file @ 5cbc96d9
// Generated by running
// maketables --digits=true --url=http://www.unicode.org/Public/5.1.0/ucd/UnicodeData.txt
// DO NOT EDIT
package unicode
// Digit is the set of Unicode characters with the "decimal digit" (Nd) property.
var Digit = digit
var digit = []Range {
Range{0x0030, 0x0039, 1},
Range{0x0660, 0x0669, 1},
Range{0x06f0, 0x06f9, 1},
Range{0x07c0, 0x07c9, 1},
Range{0x0966, 0x096f, 1},
Range{0x09e6, 0x09ef, 1},
Range{0x0a66, 0x0a6f, 1},
Range{0x0ae6, 0x0aef, 1},
Range{0x0b66, 0x0b6f, 1},
Range{0x0be6, 0x0bef, 1},
Range{0x0c66, 0x0c6f, 1},
Range{0x0ce6, 0x0cef, 1},
Range{0x0d66, 0x0d6f, 1},
Range{0x0e50, 0x0e59, 1},
Range{0x0ed0, 0x0ed9, 1},
Range{0x0f20, 0x0f29, 1},
Range{0x1040, 0x1049, 1},
Range{0x1090, 0x1099, 1},
Range{0x17e0, 0x17e9, 1},
Range{0x1810, 0x1819, 1},
Range{0x1946, 0x194f, 1},
Range{0x19d0, 0x19d9, 1},
Range{0x1b50, 0x1b59, 1},
Range{0x1bb0, 0x1bb9, 1},
Range{0x1c40, 0x1c49, 1},
Range{0x1c50, 0x1c59, 1},
Range{0xa620, 0xa629, 1},
Range{0xa8d0, 0xa8d9, 1},
Range{0xa900, 0xa909, 1},
Range{0xaa50, 0xaa59, 1},
Range{0xff10, 0xff19, 1},
Range{0x104a0, 0x104a9, 1},
Range{0x1d7ce, 0x1d7ff, 1},
}
src/pkg/unicode/maketables.go
View file @ 94e69155
......@@ -16,13 +16,23 @@ import (
"os";
"strconv";
"strings";
"unicode";
)
var url = flag.String("url", "http://www.unicode.org/Public/5.1.0/ucd/UnicodeData.txt", "URL of Unicode database")
var digits = flag.Bool("digits", false, "whether to generate digit tables; default is letter tables");
var url = flag.String("url",
"http://www.unicode.org/Public/5.1.0/ucd/UnicodeData.txt",
"URL of Unicode database")
var tables = flag.String("tables",
"all",
"comma-separated list of which tables to generate; default is all; can be letter");
var test = flag.Bool("test",
false,
"test existing tables; can be used to compare web data with package data");
var die = log.New(os.Stderr, nil, "", log.Lexit|log.Lshortfile);
var category = map[string] bool{ "letter":true } // Nd Lu etc. letter is a special case
// Data has form:
// 0037;DIGIT SEVEN;Nd;0;EN;;7;7;7;N;;;;;
// 007A;LATIN SMALL LETTER Z;Ll;0;L;;;;;N;;;005A;;005A
......@@ -46,7 +56,7 @@ const (
FSimpleTitlecaseMapping;
NumField;
MaxChar = 0xF0000; // anything above this doesn't have useful properties
MaxChar = 0x10FFFF; // anything above this shouldn't exist
)
var fieldName = []string{
......@@ -72,7 +82,6 @@ type Char struct {
field []string; // debugging only; could be deleted if we take out char.dump()
codePoint uint32; // redundant (it's the index in the chars table) but useful
category string;
numValue int;
upperCase uint32;
lowerCase uint32;
titleCase uint32;
......@@ -85,7 +94,7 @@ var lastChar uint32 = 0;
func parse(line string) {
field := strings.Split(line, ";", -1);
if len(field) != NumField {
die.Logf("%.5s...: %d fields (expected %d)\n", line, len(field), NumField);
die.Logf("%5s: %d fields (expected %d)\n", line, len(field), NumField);
}
point, err := strconv.Btoui64(field[FCodePoint], 16);
if err != nil {
......@@ -96,7 +105,6 @@ func parse(line string) {
return // not interesting and we use 0 as unset
}
if point >= MaxChar {
fmt.Fprintf(os.Stderr, "ignoring char U+%04x\n", point);
return;
}
char := &chars[point];
......@@ -106,6 +114,7 @@ func parse(line string) {
}
char.codePoint = lastChar;
char.category = field[FGeneralCategory];
category[char.category] = true;
switch char.category {
case "Nd":
// Decimal digit
......@@ -113,7 +122,6 @@ func parse(line string) {
if err != nil {
die.Log("U+%04x: bad numeric field: %s", point, err);
}
char.numValue = v;
case "Lu":
char.letter(field[FCodePoint], field[FSimpleLowercaseMapping], field[FSimpleTitlecaseMapping]);
case "Ll":
......@@ -151,6 +159,37 @@ func (char *Char) letterValue(s string, cas string) uint32 {
return uint32(v)
}
func allCategories() []string {
a := make([]string, len(category));
i := 0;
for k := range category {
a[i] = k;
i++;
}
return a;
}
// Extract the version number from the URL
func version() string {
// Break on slashes and look for the first numeric field
fields := strings.Split(*url, "/", 0);
for _, f := range fields {
if len(f) > 0 && '0' <= f[0] && f[0] <= '9' {
return f
}
}
die.Log("unknown version");
return "Unknown";
}
func letterOp(code int) bool {
switch chars[code].category {
case "Lu", "Ll", "Lt", "Lm", "Lo":
return true
}
return false
}
func main() {
flag.Parse();
......@@ -158,6 +197,9 @@ func main() {
if err != nil {
die.Log(err);
}
if resp.StatusCode != 200 {
die.Log("bad GET status", resp.StatusCode);
}
input := bufio.NewReader(resp.Body);
for {
line, err := input.ReadLineString('\n', false);
......@@ -170,59 +212,85 @@ func main() {
parse(line);
}
resp.Body.Close();
// Find out which categories to dump
list := strings.Split(*tables, ",", 0);
if *tables == "all" {
list = allCategories();
}
if *test {
fullTest(list);
return
}
fmt.Printf(
"// Generated by running\n"
"// maketables --digits=%t --url=%s\n"
"// maketables --tables=%t --url=%s\n"
"// DO NOT EDIT\n\n"
"package unicode\n",
*digits,
"package unicode\n\n",
*tables,
*url
);
// We generate an UpperCase name to serve as concise documentation and a lowerCase
// name to store the data. This stops godoc dumping all the tables but keeps them
// available to clients.
if *digits {
dumpRange(
"\n// Digit is the set of Unicode characters with the \"decimal digit\" (Nd) property.\n"
"var Digit = digit\n"
"var digit = []Range {\n",
func(code int) bool { return chars[code].category == "Nd" },
"}\n"
);
} else {
dumpRange(
"\n// Letter is the set of Unicode letters.\n"
"var Letter = letter\n"
"var letter = []Range {\n",
func(code int) bool {
switch chars[code].category {
case "Lu", "Ll", "Lt", "Lm", "Lo":
return true
}
return false
},
"}\n"
);
dumpRange(
"\n// Upper is the set of Unicode upper case letters.\n"
"var Upper = upper\n"
"var upper = []Range {\n",
func(code int) bool { return chars[code].category == "Lu" },
"}\n"
);
dumpRange(
"\n// Lower is the set of Unicode lower case letters.\n"
"var Lower = lower\n"
"var lower = []Range {\n",
func(code int) bool { return chars[code].category == "Ll" },
"}\n"
);
fmt.Println("// Version is the Unicode edition from which the tables are derived.");
fmt.Printf("var Version = %q\n\n", version());
if *tables == "all" {
fmt.Println("// Tables is the set of Unicode data tables.");
fmt.Println("var Tables = map[string] []Range {");
for k, _ := range category {
fmt.Printf("\t%q: %s,\n", k, k);
}
fmt.Printf("}\n\n");
}
for _, name := range list {
if _, ok := category[name]; !ok {
die.Log("unknown category", name);
}
// We generate an UpperCase name to serve as concise documentation and an _UnderScored
// name to store the data. This stops godoc dumping all the tables but keeps them
// available to clients.
if name == "letter" { // special case
dumpRange(
"\n// Letter is the set of Unicode letters.\n"
"var Letter = letter\n"
"var letter = []Range {\n",
letterOp,
"}\n"
);
continue;
}
// Cases deserving special comments
switch name {
case "Nd":
fmt.Printf(
"\n// Digit is the set of Unicode characters with the \"decimal digit\" property.\n"
"var Digit = Nd\n\n"
)
case "Lu":
fmt.Printf(
"\n// Upper is the set of Unicode upper case letters.\n"
"var Upper = Lu\n\n"
)
case "Ll":
fmt.Printf(
"\n// Lower is the set of Unicode lower case letters.\n"
"var Lower = Ll\n\n"
)
case "Lt":
fmt.Printf(
"\n// Title is the set of Unicode title case letters.\n"
"var Title = Lt\n\n"
)
}
dumpRange(
"\n// Title is the set of Unicode title case letters.\n"
"var Title = title\n"
"var title = []Range {\n",
func(code int) bool { return chars[code].category == "Lt" },
"}\n"
fmt.Sprintf(
"// %s is the set of Unicode characters in category %s\n"
"var %s = _%s\n"
"var _%s = []Range {\n",
name, name, name, name, name
),
func(code int) bool { return chars[code].category == name },
"}\n\n"
);
}
}
......@@ -279,3 +347,34 @@ func dumpRange(header string, inCategory Op, trailer string) {
}
fmt.Print(trailer);
}
func fullTest(list []string) {
for _, name := range list {
if _, ok := category[name]; !ok {
die.Log("unknown category", name);
}
r, ok := unicode.Tables[name];
if !ok {
die.Log("unknown table", name);
}
if name == "letter" {
verifyRange(name, letterOp, r);
} else {
verifyRange(
name,
func(code int) bool { return chars[code].category == name },
r
);
}
}
}
func verifyRange(name string, inCategory Op, table []unicode.Range) {
for i, c := range chars {
web := inCategory(i);
pkg := unicode.Is(table, i);
if web != pkg {
fmt.Fprintf(os.Stderr, "%s: U+%04X: web=%t pkg=%t\n", name, i, web, pkg);
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7