gen_properties.go

//go:build generate

// This program generates a property file in Go file from Unicode Character
// Database auxiliary data files. The command line arguments are as follows:
//
//  1. The name of the Unicode data file (just the filename, without extension).
//     Can be "-" (to skip) if the emoji flag is included.
//  2. The name of the locally generated Go file.
//  3. The name of the slice mapping code points to properties.
//  4. The name of the generator, for logging purposes.
//  5. (Optional) Flags, comma-separated. The following flags are available:
//     - "emojis=<property>": include the specified emoji properties (e.g.
//     "Extended_Pictographic").
//     - "gencat": include general category properties.
//
//go:generate go run gen_properties.go auxiliary/GraphemeBreakProperty graphemeproperties.go graphemeCodePoints graphemes emojis=Extended_Pictographic
//go:generate go run gen_properties.go auxiliary/WordBreakProperty wordproperties.go workBreakCodePoints words emojis=Extended_Pictographic
//go:generate go run gen_properties.go auxiliary/SentenceBreakProperty sentenceproperties.go sentenceBreakCodePoints sentences
//go:generate go run gen_properties.go LineBreak lineproperties.go lineBreakCodePoints lines gencat
//go:generate go run gen_properties.go EastAsianWidth eastasianwidth.go eastAsianWidth eastasianwidth
//go:generate go run gen_properties.go - emojipresentation.go emojiPresentation emojipresentation emojis=Emoji_Presentation
package main

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"go/format"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"
)

// We want to test against a specific version rather than the latest. When the
// package is upgraded to a new version, change these to generate new tests.
const (
	propertyURL = `https://www.unicode.org/Public/15.0.0/ucd/%s.txt`
	emojiURL    = `https://unicode.org/Public/15.0.0/ucd/emoji/emoji-data.txt`
)

// The regular expression for a line containing a code point range property.
var propertyPattern = regexp.MustCompile(`^([0-9A-F]{4,6})(\.\.([0-9A-F]{4,6}))?\s*;\s*([A-Za-z0-9_]+)\s*#\s(.+)$`)

func main() {
	if len(os.Args) < 5 {
		fmt.Println("Not enough arguments, see code for details")
		os.Exit(1)
	}

	log.SetPrefix("gen_properties (" + os.Args[4] + "): ")
	log.SetFlags(0)

	// Parse flags.
	flags := make(map[string]string)
	if len(os.Args) >= 6 {
		for _, flag := range strings.Split(os.Args[5], ",") {
			flagFields := strings.Split(flag, "=")
			if len(flagFields) == 1 {
				flags[flagFields[0]] = "yes"
			} else {
				flags[flagFields[0]] = flagFields[1]
			}
		}
	}

	// Parse the text file and generate Go source code from it.
	_, includeGeneralCategory := flags["gencat"]
	var mainURL string
	if os.Args[1] != "-" {
		mainURL = fmt.Sprintf(propertyURL, os.Args[1])
	}
	src, err := parse(mainURL, flags["emojis"], includeGeneralCategory)
	if err != nil {
		log.Fatal(err)
	}

	// Format the Go code.
	formatted, err := format.Source([]byte(src))
	if err != nil {
		log.Fatal("gofmt:", err)
	}

	// Save it to the (local) target file.
	log.Print("Writing to ", os.Args[2])
	if err := ioutil.WriteFile(os.Args[2], formatted, 0644); err != nil {
		log.Fatal(err)
	}
}

// parse parses the Unicode Properties text files located at the given URLs and
// returns their equivalent Go source code to be used in the uniseg package. If
// "emojiProperty" is not an empty string, emoji code points for that emoji
// property (e.g. "Extended_Pictographic") will be included. In those cases, you
// may pass an empty "propertyURL" to skip parsing the main properties file. If
// "includeGeneralCategory" is true, the Unicode General Category property will
// be extracted from the comments and included in the output.
func parse(propertyURL, emojiProperty string, includeGeneralCategory bool) (string, error) {
	if propertyURL == "" && emojiProperty == "" {
		return "", errors.New("no properties to parse")
	}

	// Temporary buffer to hold properties.
	var properties [][4]string

	// Open the first URL.
	if propertyURL != "" {
		log.Printf("Parsing %s", propertyURL)
		res, err := http.Get(propertyURL)
		if err != nil {
			return "", err
		}
		in1 := res.Body
		defer in1.Close()

		// Parse it.
		scanner := bufio.NewScanner(in1)
		num := 0
		for scanner.Scan() {
			num++
			line := strings.TrimSpace(scanner.Text())

			// Skip comments and empty lines.
			if strings.HasPrefix(line, "#") || line == "" {
				continue
			}

			// Everything else must be a code point range, a property and a comment.
			from, to, property, comment, err := parseProperty(line)
			if err != nil {
				return "", fmt.Errorf("%s line %d: %v", os.Args[4], num, err)
			}
			properties = append(properties, [4]string{from, to, property, comment})
		}
		if err := scanner.Err(); err != nil {
			return "", err
		}
	}

	// Open the second URL.
	if emojiProperty != "" {
		log.Printf("Parsing %s", emojiURL)
		res, err := http.Get(emojiURL)
		if err != nil {
			return "", err
		}
		in2 := res.Body
		defer in2.Close()

		// Parse it.
		scanner := bufio.NewScanner(in2)
		num := 0
		for scanner.Scan() {
			num++
			line := scanner.Text()

			// Skip comments, empty lines, and everything not containing
			// "Extended_Pictographic".
			if strings.HasPrefix(line, "#") || line == "" || !strings.Contains(line, emojiProperty) {
				continue
			}

			// Everything else must be a code point range, a property and a comment.
			from, to, property, comment, err := parseProperty(line)
			if err != nil {
				return "", fmt.Errorf("emojis line %d: %v", num, err)
			}
			properties = append(properties, [4]string{from, to, property, comment})
		}
		if err := scanner.Err(); err != nil {
			return "", err
		}
	}

	// Avoid overflow during binary search.
	if len(properties) >= 1<<31 {
		return "", errors.New("too many properties")
	}

	// Sort properties.
	sort.Slice(properties, func(i, j int) bool {
		left, _ := strconv.ParseUint(properties[i][0], 16, 64)
		right, _ := strconv.ParseUint(properties[j][0], 16, 64)
		return left < right
	})

	// Header.
	var (
		buf          bytes.Buffer
		emojiComment string
	)
	columns := 3
	if includeGeneralCategory {
		columns = 4
	}
	if emojiURL != "" {
		emojiComment = `
// and
// ` + emojiURL + `
// ("Extended_Pictographic" only)`
	}
	buf.WriteString(`// Code generated via go generate from gen_properties.go. DO NOT EDIT.

package uniseg

// ` + os.Args[3] + ` are taken from
// ` + propertyURL + emojiComment + `
// on ` + time.Now().Format("January 2, 2006") + `. See https://www.unicode.org/license.html for the Unicode
// license agreement.
var ` + os.Args[3] + ` = [][` + strconv.Itoa(columns) + `]int{
	`)

	// Properties.
	for _, prop := range properties {
		if includeGeneralCategory {
			generalCategory := "gc" + prop[3][:2]
			if generalCategory == "gcL&" {
				generalCategory = "gcLC"
			}
			prop[3] = prop[3][3:]
			fmt.Fprintf(&buf, "{0x%s,0x%s,%s,%s}, // %s\n", prop[0], prop[1], translateProperty("pr", prop[2]), generalCategory, prop[3])
		} else {
			fmt.Fprintf(&buf, "{0x%s,0x%s,%s}, // %s\n", prop[0], prop[1], translateProperty("pr", prop[2]), prop[3])
		}
	}

	// Tail.
	buf.WriteString("}")

	return buf.String(), nil
}

// parseProperty parses a line of the Unicode properties text file containing a
// property for a code point range and returns it along with its comment.
func parseProperty(line string) (from, to, property, comment string, err error) {
	fields := propertyPattern.FindStringSubmatch(line)
	if fields == nil {
		err = errors.New("no property found")
		return
	}
	from = fields[1]
	to = fields[3]
	if to == "" {
		to = from
	}
	property = fields[4]
	comment = fields[5]
	return
}

// translateProperty translates a property name as used in the Unicode data file
// to a variable used in the Go code.
func translateProperty(prefix, property string) string {
	return prefix + strings.ReplaceAll(property, "_", "")
}