vendor/github.com/rivo/uniseg/gen_properties.go - voltha-openolt-adapter - Gitiles

 //go:build generate

 // This program generates a property file in Go file from Unicode Character
 // Database auxiliary data files. The command line arguments are as follows:
 //
 //  1. The name of the Unicode data file (just the filename, without extension).
 //     Can be "-" (to skip) if the emoji flag is included.
 //  2. The name of the locally generated Go file.
 //  3. The name of the slice mapping code points to properties.
 //  4. The name of the generator, for logging purposes.
 //  5. (Optional) Flags, comma-separated. The following flags are available:
 //     - "emojis=<property>": include the specified emoji properties (e.g.
 //     "Extended_Pictographic").
 //     - "gencat": include general category properties.
 //
 //go:generate go run gen_properties.go auxiliary/GraphemeBreakProperty graphemeproperties.go graphemeCodePoints graphemes emojis=Extended_Pictographic
 //go:generate go run gen_properties.go auxiliary/WordBreakProperty wordproperties.go workBreakCodePoints words emojis=Extended_Pictographic
 //go:generate go run gen_properties.go auxiliary/SentenceBreakProperty sentenceproperties.go sentenceBreakCodePoints sentences
 //go:generate go run gen_properties.go LineBreak lineproperties.go lineBreakCodePoints lines gencat
 //go:generate go run gen_properties.go EastAsianWidth eastasianwidth.go eastAsianWidth eastasianwidth
 //go:generate go run gen_properties.go - emojipresentation.go emojiPresentation emojipresentation emojis=Emoji_Presentation
 package main

 import (
 	"bufio"
 	"bytes"
 	"errors"
 	"fmt"
 	"go/format"
 	"io/ioutil"
 	"log"
 	"net/http"
 	"os"
 	"regexp"
 	"sort"
 	"strconv"
 	"strings"
 	"time"
 )

 // We want to test against a specific version rather than the latest. When the
 // package is upgraded to a new version, change these to generate new tests.
 const (
 	propertyURL = `https://www.unicode.org/Public/15.0.0/ucd/%s.txt`
 	emojiURL    = `https://unicode.org/Public/15.0.0/ucd/emoji/emoji-data.txt`
 )

 // The regular expression for a line containing a code point range property.
 var propertyPattern = regexp.MustCompile(`^([0-9A-F]{4,6})(\.\.([0-9A-F]{4,6}))?\s*;\s*([A-Za-z0-9_]+)\s*#\s(.+)$`)

 func main() {
 	if len(os.Args) < 5 {
 		fmt.Println("Not enough arguments, see code for details")
 		os.Exit(1)
 	}

 	log.SetPrefix("gen_properties (" + os.Args[4] + "): ")
 	log.SetFlags(0)

 	// Parse flags.
 	flags := make(map[string]string)
 	if len(os.Args) >= 6 {
 		for _, flag := range strings.Split(os.Args[5], ",") {
 			flagFields := strings.Split(flag, "=")
 			if len(flagFields) == 1 {
 				flags[flagFields[0]] = "yes"
 			} else {
 				flags[flagFields[0]] = flagFields[1]
 			}
 		}
 	}

 	// Parse the text file and generate Go source code from it.
 	_, includeGeneralCategory := flags["gencat"]
 	var mainURL string
 	if os.Args[1] != "-" {
 		mainURL = fmt.Sprintf(propertyURL, os.Args[1])
 	}
 	src, err := parse(mainURL, flags["emojis"], includeGeneralCategory)
 	if err != nil {
 		log.Fatal(err)
 	}

 	// Format the Go code.
 	formatted, err := format.Source([]byte(src))
 	if err != nil {
 		log.Fatal("gofmt:", err)
 	}

 	// Save it to the (local) target file.
 	log.Print("Writing to ", os.Args[2])
 	if err := ioutil.WriteFile(os.Args[2], formatted, 0644); err != nil {
 		log.Fatal(err)
 	}
 }

 // parse parses the Unicode Properties text files located at the given URLs and
 // returns their equivalent Go source code to be used in the uniseg package. If
 // "emojiProperty" is not an empty string, emoji code points for that emoji
 // property (e.g. "Extended_Pictographic") will be included. In those cases, you
 // may pass an empty "propertyURL" to skip parsing the main properties file. If
 // "includeGeneralCategory" is true, the Unicode General Category property will
 // be extracted from the comments and included in the output.
 func parse(propertyURL, emojiProperty string, includeGeneralCategory bool) (string, error) {
 	if propertyURL == "" && emojiProperty == "" {
 		return "", errors.New("no properties to parse")
 	}

 	// Temporary buffer to hold properties.
 	var properties [][4]string

 	// Open the first URL.
 	if propertyURL != "" {
 		log.Printf("Parsing %s", propertyURL)
 		res, err := http.Get(propertyURL)
 		if err != nil {
 			return "", err
 		}
 		in1 := res.Body
 		defer in1.Close()

 		// Parse it.
 		scanner := bufio.NewScanner(in1)
 		num := 0
 		for scanner.Scan() {
 			num++
 			line := strings.TrimSpace(scanner.Text())

 			// Skip comments and empty lines.
 			if strings.HasPrefix(line, "#") || line == "" {
 				continue
 			}

 			// Everything else must be a code point range, a property and a comment.
 			from, to, property, comment, err := parseProperty(line)
 			if err != nil {
 				return "", fmt.Errorf("%s line %d: %v", os.Args[4], num, err)
 			}
 			properties = append(properties, [4]string{from, to, property, comment})
 		}
 		if err := scanner.Err(); err != nil {
 			return "", err
 		}
 	}

 	// Open the second URL.
 	if emojiProperty != "" {
 		log.Printf("Parsing %s", emojiURL)
 		res, err := http.Get(emojiURL)
 		if err != nil {
 			return "", err
 		}
 		in2 := res.Body
 		defer in2.Close()

 		// Parse it.
 		scanner := bufio.NewScanner(in2)
 		num := 0
 		for scanner.Scan() {
 			num++
 			line := scanner.Text()

 			// Skip comments, empty lines, and everything not containing
 			// "Extended_Pictographic".
 			if strings.HasPrefix(line, "#") || line == "" || !strings.Contains(line, emojiProperty) {
 				continue
 			}

 			// Everything else must be a code point range, a property and a comment.
 			from, to, property, comment, err := parseProperty(line)
 			if err != nil {
 				return "", fmt.Errorf("emojis line %d: %v", num, err)
 			}
 			properties = append(properties, [4]string{from, to, property, comment})
 		}
 		if err := scanner.Err(); err != nil {
 			return "", err
 		}
 	}

 	// Avoid overflow during binary search.
 	if len(properties) >= 1<<31 {
 		return "", errors.New("too many properties")
 	}

 	// Sort properties.
 	sort.Slice(properties, func(i, j int) bool {
 		left, _ := strconv.ParseUint(properties[i][0], 16, 64)
 		right, _ := strconv.ParseUint(properties[j][0], 16, 64)
 		return left < right
 	})

 	// Header.
 	var (
 		buf          bytes.Buffer
 		emojiComment string
 	)
 	columns := 3
 	if includeGeneralCategory {
 		columns = 4
 	}
 	if emojiURL != "" {
 		emojiComment = `
 // and
 // ` + emojiURL + `
 // ("Extended_Pictographic" only)`
 	}
 	buf.WriteString(`// Code generated via go generate from gen_properties.go. DO NOT EDIT.

 package uniseg

 // ` + os.Args[3] + ` are taken from
 // ` + propertyURL + emojiComment + `
 // on ` + time.Now().Format("January 2, 2006") + `. See https://www.unicode.org/license.html for the Unicode
 // license agreement.
 var ` + os.Args[3] + ` = [][` + strconv.Itoa(columns) + `]int{
 	`)

 	// Properties.
 	for _, prop := range properties {
 		if includeGeneralCategory {
 			generalCategory := "gc" + prop[3][:2]
 			if generalCategory == "gcL&" {
 				generalCategory = "gcLC"
 			}
 			prop[3] = prop[3][3:]
 			fmt.Fprintf(&buf, "{0x%s,0x%s,%s,%s}, // %s\n", prop[0], prop[1], translateProperty("pr", prop[2]), generalCategory, prop[3])
 		} else {
 			fmt.Fprintf(&buf, "{0x%s,0x%s,%s}, // %s\n", prop[0], prop[1], translateProperty("pr", prop[2]), prop[3])
 		}
 	}

 	// Tail.
 	buf.WriteString("}")

 	return buf.String(), nil
 }

 // parseProperty parses a line of the Unicode properties text file containing a
 // property for a code point range and returns it along with its comment.
 func parseProperty(line string) (from, to, property, comment string, err error) {
 	fields := propertyPattern.FindStringSubmatch(line)
 	if fields == nil {
 		err = errors.New("no property found")
 		return
 	}
 	from = fields[1]
 	to = fields[3]
 	if to == "" {
 		to = from
 	}
 	property = fields[4]
 	comment = fields[5]
 	return
 }

 // translateProperty translates a property name as used in the Unicode data file
 // to a variable used in the Go code.
 func translateProperty(prefix, property string) string {
 	return prefix + strings.ReplaceAll(property, "_", "")
 }
	//go:build generate

	// This program generates a property file in Go file from Unicode Character
	// Database auxiliary data files. The command line arguments are as follows:
	//
	// 1. The name of the Unicode data file (just the filename, without extension).
	// Can be "-" (to skip) if the emoji flag is included.
	// 2. The name of the locally generated Go file.
	// 3. The name of the slice mapping code points to properties.
	// 4. The name of the generator, for logging purposes.
	// 5. (Optional) Flags, comma-separated. The following flags are available:
	// - "emojis=<property>": include the specified emoji properties (e.g.
	// "Extended_Pictographic").
	// - "gencat": include general category properties.
	//
	//go:generate go run gen_properties.go auxiliary/GraphemeBreakProperty graphemeproperties.go graphemeCodePoints graphemes emojis=Extended_Pictographic
	//go:generate go run gen_properties.go auxiliary/WordBreakProperty wordproperties.go workBreakCodePoints words emojis=Extended_Pictographic
	//go:generate go run gen_properties.go auxiliary/SentenceBreakProperty sentenceproperties.go sentenceBreakCodePoints sentences
	//go:generate go run gen_properties.go LineBreak lineproperties.go lineBreakCodePoints lines gencat
	//go:generate go run gen_properties.go EastAsianWidth eastasianwidth.go eastAsianWidth eastasianwidth
	//go:generate go run gen_properties.go - emojipresentation.go emojiPresentation emojipresentation emojis=Emoji_Presentation
	package main

	import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"go/format"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"
	)

	// We want to test against a specific version rather than the latest. When the
	// package is upgraded to a new version, change these to generate new tests.
	const (
	propertyURL = `https://www.unicode.org/Public/15.0.0/ucd/%s.txt`
	emojiURL = `https://unicode.org/Public/15.0.0/ucd/emoji/emoji-data.txt`
	)

	// The regular expression for a line containing a code point range property.
	var propertyPattern = regexp.MustCompile(`^([0-9A-F]{4,6})(\.\.([0-9A-F]{4,6}))?\s;\s([A-Za-z0-9_]+)\s*#\s(.+)$`)

	func main() {
	if len(os.Args) < 5 {
	fmt.Println("Not enough arguments, see code for details")
	os.Exit(1)
	}

	log.SetPrefix("gen_properties (" + os.Args[4] + "): ")
	log.SetFlags(0)

	// Parse flags.
	flags := make(map[string]string)
	if len(os.Args) >= 6 {
	for _, flag := range strings.Split(os.Args[5], ",") {
	flagFields := strings.Split(flag, "=")
	if len(flagFields) == 1 {
	flags[flagFields[0]] = "yes"
	} else {
	flags[flagFields[0]] = flagFields[1]
	}
	}
	}

	// Parse the text file and generate Go source code from it.
	_, includeGeneralCategory := flags["gencat"]
	var mainURL string
	if os.Args[1] != "-" {
	mainURL = fmt.Sprintf(propertyURL, os.Args[1])
	}
	src, err := parse(mainURL, flags["emojis"], includeGeneralCategory)
	if err != nil {
	log.Fatal(err)
	}

	// Format the Go code.
	formatted, err := format.Source([]byte(src))
	if err != nil {
	log.Fatal("gofmt:", err)
	}

	// Save it to the (local) target file.
	log.Print("Writing to ", os.Args[2])
	if err := ioutil.WriteFile(os.Args[2], formatted, 0644); err != nil {
	log.Fatal(err)
	}
	}

	// parse parses the Unicode Properties text files located at the given URLs and
	// returns their equivalent Go source code to be used in the uniseg package. If
	// "emojiProperty" is not an empty string, emoji code points for that emoji
	// property (e.g. "Extended_Pictographic") will be included. In those cases, you
	// may pass an empty "propertyURL" to skip parsing the main properties file. If
	// "includeGeneralCategory" is true, the Unicode General Category property will
	// be extracted from the comments and included in the output.
	func parse(propertyURL, emojiProperty string, includeGeneralCategory bool) (string, error) {
	if propertyURL == "" && emojiProperty == "" {
	return "", errors.New("no properties to parse")
	}

	// Temporary buffer to hold properties.
	var properties [][4]string

	// Open the first URL.
	if propertyURL != "" {
	log.Printf("Parsing %s", propertyURL)
	res, err := http.Get(propertyURL)
	if err != nil {
	return "", err
	}
	in1 := res.Body
	defer in1.Close()

	// Parse it.
	scanner := bufio.NewScanner(in1)
	num := 0
	for scanner.Scan() {
	num++
	line := strings.TrimSpace(scanner.Text())

	// Skip comments and empty lines.
	if strings.HasPrefix(line, "#") \|\| line == "" {
	continue
	}

	// Everything else must be a code point range, a property and a comment.
	from, to, property, comment, err := parseProperty(line)
	if err != nil {
	return "", fmt.Errorf("%s line %d: %v", os.Args[4], num, err)
	}
	properties = append(properties, [4]string{from, to, property, comment})
	}
	if err := scanner.Err(); err != nil {
	return "", err
	}
	}

	// Open the second URL.
	if emojiProperty != "" {
	log.Printf("Parsing %s", emojiURL)
	res, err := http.Get(emojiURL)
	if err != nil {
	return "", err
	}
	in2 := res.Body
	defer in2.Close()

	// Parse it.
	scanner := bufio.NewScanner(in2)
	num := 0
	for scanner.Scan() {
	num++
	line := scanner.Text()

	// Skip comments, empty lines, and everything not containing
	// "Extended_Pictographic".
	if strings.HasPrefix(line, "#") \|\| line == "" \|\| !strings.Contains(line, emojiProperty) {
	continue
	}

	// Everything else must be a code point range, a property and a comment.
	from, to, property, comment, err := parseProperty(line)
	if err != nil {
	return "", fmt.Errorf("emojis line %d: %v", num, err)
	}
	properties = append(properties, [4]string{from, to, property, comment})
	}
	if err := scanner.Err(); err != nil {
	return "", err
	}
	}

	// Avoid overflow during binary search.
	if len(properties) >= 1<<31 {
	return "", errors.New("too many properties")
	}

	// Sort properties.
	sort.Slice(properties, func(i, j int) bool {
	left, _ := strconv.ParseUint(properties[i][0], 16, 64)
	right, _ := strconv.ParseUint(properties[j][0], 16, 64)
	return left < right
	})

	// Header.
	var (
	buf bytes.Buffer
	emojiComment string
	)
	columns := 3
	if includeGeneralCategory {
	columns = 4
	}
	if emojiURL != "" {
	emojiComment = `
	// and
	// ` + emojiURL + `
	// ("Extended_Pictographic" only)`
	}
	buf.WriteString(`// Code generated via go generate from gen_properties.go. DO NOT EDIT.

	package uniseg

	// ` + os.Args[3] + ` are taken from
	// ` + propertyURL + emojiComment + `
	// on ` + time.Now().Format("January 2, 2006") + `. See https://www.unicode.org/license.html for the Unicode
	// license agreement.
	var ` + os.Args[3] + ` = [][` + strconv.Itoa(columns) + `]int{
	`)

	// Properties.
	for _, prop := range properties {
	if includeGeneralCategory {
	generalCategory := "gc" + prop[3][:2]
	if generalCategory == "gcL&" {
	generalCategory = "gcLC"
	}
	prop[3] = prop[3][3:]
	fmt.Fprintf(&buf, "{0x%s,0x%s,%s,%s}, // %s\n", prop[0], prop[1], translateProperty("pr", prop[2]), generalCategory, prop[3])
	} else {
	fmt.Fprintf(&buf, "{0x%s,0x%s,%s}, // %s\n", prop[0], prop[1], translateProperty("pr", prop[2]), prop[3])
	}
	}

	// Tail.
	buf.WriteString("}")

	return buf.String(), nil
	}

	// parseProperty parses a line of the Unicode properties text file containing a
	// property for a code point range and returns it along with its comment.
	func parseProperty(line string) (from, to, property, comment string, err error) {
	fields := propertyPattern.FindStringSubmatch(line)
	if fields == nil {
	err = errors.New("no property found")
	return
	}
	from = fields[1]
	to = fields[3]
	if to == "" {
	to = from
	}
	property = fields[4]
	comment = fields[5]
	return
	}

	// translateProperty translates a property name as used in the Unicode data file
	// to a variable used in the Go code.
	func translateProperty(prefix, property string) string {
	return prefix + strings.ReplaceAll(property, "_", "")
	}