Refactor buildmetadata to not require svn export

.gitignore

@@ -10,3 +10,4 @@ geocoding
 buildmetadata
 functions/*
 dist/
+_build

cmd/buildmetadata/main.go

@@ -11,7 +11,6 @@ import (
 	"io/ioutil"
 	"log"
 	"math"
-	"net/http"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -24,28 +23,157 @@ import (
 	"github.com/nyaruka/phonenumbers"
 )
 
+func main() {
+	if err := buildMetadata(); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+func buildMetadata() error {
+	fmt.Print("Cloning upstream repo... ")
+
+	if err := cloneUpstreamRepo("[email protected]:google/libphonenumber.git"); err != nil {
+		return err
+	}
+
+	fmt.Print("OK\nBuilding number metadata...")
+
+	metadata, err := buildNumberMetadata("resources/PhoneNumberMetadata.xml", "metadataData", "metadata_bin.go", false)
+	if err != nil {
+		return err
+	}
+
+	fmt.Print("OK\nBuilding short number metadata...")
+
+	_, err = buildNumberMetadata("resources/ShortNumberMetadata.xml", "shortNumberMetadataData", "shortnumber_metadata_bin.go", true)
+	if err != nil {
+		return err
+	}
+
+	fmt.Print("OK\nBuilding region metadata...")
+
+	if err := buildRegionMetadata(metadata, "regionMapData", "countrycode_to_region_bin.go"); err != nil {
+		return err
+	}
+
+	fmt.Print("OK\nBuilding timezone metadata...")
+
+	if err := buildTimezoneMetadata("resources/timezones/map_data.txt", "timezoneMapData", "prefix_to_timezone_bin.go"); err != nil {
+		return err
+	}
+
+	buildPrefixData(&carrier)
+	buildPrefixData(&geocoding)
+
+	return nil
+}
+
+func cloneUpstreamRepo(url string) error {
+	os.RemoveAll("_build")
+
+	cmd := exec.Command("git", "clone", "--depth=1", url, "_build")
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("error cloning upstream repo: %w", err)
+	}
+
+	return nil
+}
+
+func buildNumberMetadata(srcFile, varName, dstFile string, short bool) (*phonenumbers.PhoneMetadataCollection, error) {
+	body, err := os.ReadFile("_build/" + srcFile)
+	if err != nil {
+		return nil, fmt.Errorf("error reading %s: %w", srcFile, err)
+	}
+
+	collection, err := phonenumbers.BuildPhoneMetadataCollection(body, false, false, short)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing %s: %w", srcFile, err)
+	}
+
+	data, err := proto.Marshal(collection)
+	if err != nil {
+		return nil, fmt.Errorf("error marshaling metadata as protobuf: %w", err)
+	}
+
+	if err := os.WriteFile(dstFile, generateBinFile(varName, data), os.FileMode(0664)); err != nil {
+		return nil, fmt.Errorf("error writing %s: %w", dstFile, err)
+	}
+
+	return collection, nil
+}
+
+func buildRegionMetadata(metadata *phonenumbers.PhoneMetadataCollection, varName, dstFile string) error {
+	regionMap := phonenumbers.BuildCountryCodeToRegionMap(metadata)
+
+	// generate our map data
+	data, err := renderMap(dstFile, regionMap)
+	if err != nil {
+		return fmt.Errorf("error generating %s: %w", dstFile, err)
+	}
+
+	if err := os.WriteFile(dstFile, generateBinFile(varName, data), os.FileMode(0664)); err != nil {
+		return fmt.Errorf("error writing %s: %w", dstFile, err)
+	}
+
+	return nil
+}
+
+func buildTimezoneMetadata(srcFile, varName, dstFile string) error {
+	body, err := os.ReadFile("_build/" + srcFile)
+	if err != nil {
+		return fmt.Errorf("error reading %s: %w", srcFile, err)
+	}
+
+	// build our map of prefix to timezones
+	prefixMap := make(map[int][]string)
+	for _, line := range strings.Split(string(body), "\n") {
+		if strings.HasPrefix(line, "#") {
+			continue
+		}
+
+		if strings.TrimSpace(line) == "" {
+			continue
+		}
+
+		fields := strings.Split(line, "|")
+		if len(fields) != 2 {
+			return fmt.Errorf("invalid format in timezone file: %s", line)
+		}
+
+		zones := strings.Split(fields[1], "&")
+		if len(zones) < 1 {
+			return fmt.Errorf("invalid format in timezone file: %s", line)
+		}
+
+		// parse our prefix
+		prefix, err := strconv.Atoi(fields[0])
+		if err != nil {
+			return fmt.Errorf("invalid prefix in line: %s", line)
+		}
+		prefixMap[prefix] = zones
+	}
+
+	// generate our map data
+	data, err := renderMap(dstFile, prefixMap)
+	if err != nil {
+		return fmt.Errorf("error generating %s: %w", dstFile, err)
+	}
+
+	if err := os.WriteFile(dstFile, generateBinFile(varName, data), os.FileMode(0664)); err != nil {
+		return fmt.Errorf("error writing %s: %w", dstFile, err)
+	}
+
+	return nil
+}
+
 type prefixBuild struct {
 	url     string
 	dir     string
 	srcPath string
 	varName string
 }
 
-const (
-	metadataURL  = "https://raw.githubusercontent.com/googlei18n/libphonenumber/master/resources/PhoneNumberMetadata.xml"
-	metadataPath = "metadata_bin.go"
-
-	shortNumberMetadataURL  = "https://raw.githubusercontent.com/googlei18n/libphonenumber/master/resources/ShortNumberMetadata.xml"
-	shortNumberMetadataPath = "shortnumber_metadata_bin.go"
-
-	tzURL  = "https://raw.githubusercontent.com/googlei18n/libphonenumber/master/resources/timezones/map_data.txt"
-	tzPath = "prefix_to_timezone_bin.go"
-	tzVar  = "timezoneMapData"
-
-	regionPath = "countrycode_to_region_bin.go"
-	regionVar  = "regionMapData"
-)
-
 var carrier = prefixBuild{
 	url:     "https://github.com/googlei18n/libphonenumber/trunk/resources/carrier",
 	dir:     "carrier",
@@ -60,20 +188,6 @@ var geocoding = prefixBuild{
 	varName: "geocodingMapData",
 }
 
-func fetchURL(url string) []byte {
-	resp, err := http.Get(url)
-	if err != nil || resp.StatusCode != 200 {
-		log.Fatalf("Error fetching URL '%s': %s", url, err)
-	}
-	defer resp.Body.Close()
-	body, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		log.Fatalf("Error reading body: %s", err)
-	}
-
-	return body
-}
-
 func svnExport(dir string, url string) {
 	os.RemoveAll(dir)
 	cmd := exec.Command(
@@ -122,56 +236,13 @@ func writeFile(filePath string, data []byte) {
 	}
 
 	fmt.Printf("Writing new %s\n", filePath)
-	err := ioutil.WriteFile(filePath, data, os.FileMode(0664))
+	err := os.WriteFile(filePath, data, os.FileMode(0664))
 	if err != nil {
 		log.Fatalf("Error writing '%s': %s", filePath, err)
 	}
 }
 
-func buildRegions(metadata *phonenumbers.PhoneMetadataCollection) {
-	log.Println("Building region map")
-	regionMap := phonenumbers.BuildCountryCodeToRegionMap(metadata)
-	writeIntStringArrayMap(regionPath, regionVar, regionMap)
-}
-
-func buildTimezones() {
-	log.Println("Building timezone map")
-	body := fetchURL(tzURL)
-
-	// build our map of prefix to timezones
-	prefixMap := make(map[int][]string)
-	for _, line := range strings.Split(string(body), "\n") {
-		if strings.HasPrefix(line, "#") {
-			continue
-		}
-
-		if strings.TrimSpace(line) == "" {
-			continue
-		}
-
-		fields := strings.Split(line, "|")
-		if len(fields) != 2 {
-			log.Fatalf("Invalid format in timezone file: %s", line)
-		}
-
-		zones := strings.Split(fields[1], "&")
-		if len(zones) < 1 {
-			log.Fatalf("Invalid format in timezone file: %s", line)
-		}
-
-		// parse our prefix
-		prefix, err := strconv.Atoi(fields[0])
-		if err != nil {
-			log.Fatalf("Invalid prefix in line: %s", line)
-		}
-		prefixMap[prefix] = zones
-	}
-
-	// then write our file
-	writeIntStringArrayMap(tzPath, tzVar, prefixMap)
-}
-
-func writeIntStringArrayMap(path string, varName string, prefixMap map[int][]string) {
+func renderMap(path string, prefixMap map[int][]string) ([]byte, error) {
 	// build lists of our keys and values
 	keys := make([]int, 0, len(prefixMap))
 	values := make([]string, 0, 255)
@@ -200,15 +271,15 @@ func writeIntStringArrayMap(path string, varName string, prefixMap map[int][]str
 	// first write our values, as length of string and raw bytes
 	joinedValues := strings.Join(values, "\n")
 	if err := binary.Write(data, binary.LittleEndian, uint32(len(joinedValues))); err != nil {
-		log.Fatal(err)
+		return nil, err
 	}
 	if err := binary.Write(data, binary.LittleEndian, []byte(joinedValues)); err != nil {
-		log.Fatal(err)
+		return nil, err
 	}
 
 	// then the number of keys
 	if err := binary.Write(data, binary.LittleEndian, uint32(len(keys))); err != nil {
-		log.Fatal(err)
+		return nil, err
 	}
 
 	// we write our key / value pairs as a varint of the difference of the previous prefix
@@ -220,72 +291,29 @@ func writeIntStringArrayMap(path string, varName string, prefixMap map[int][]str
 		diff := key - last
 		l := binary.PutUvarint(intBuf, uint64(diff))
 		if err := binary.Write(data, binary.LittleEndian, intBuf[:l]); err != nil {
-			log.Fatal(err)
+			return nil, err
 		}
 
 		// then our values
 		values := prefixMap[key]
 
 		// write our number of values
 		if err := binary.Write(data, binary.LittleEndian, uint8(len(values))); err != nil {
-			log.Fatal(err)
+			return nil, err
 		}
 
 		// then each value as the interned index
 		for _, v := range values {
 			valueIntern := internMap[v]
 			if err := binary.Write(data, binary.LittleEndian, uint16(valueIntern)); err != nil {
-				log.Fatal(err)
+				return nil, err
 			}
 		}
 
 		last = key
 	}
 
-	// then write our file
-	writeFile(path, generateBinFile(varName, data.Bytes()))
-}
-
-func buildMetadata() *phonenumbers.PhoneMetadataCollection {
-	log.Println("Fetching PhoneNumberMetadata.xml from Github")
-	body := fetchURL(metadataURL)
-
-	log.Println("Building new metadata collection")
-	collection, err := phonenumbers.BuildPhoneMetadataCollection(body, false, false, false)
-	if err != nil {
-		log.Fatalf("Error converting XML: %s", err)
-	}
-
-	// write it out as a protobuf
-	data, err := proto.Marshal(collection)
-	if err != nil {
-		log.Fatalf("Error marshalling metadata: %v", err)
-	}
-
-	log.Println("Writing new metadata_bin.go")
-	writeFile(metadataPath, generateBinFile("metadataData", data))
-	return collection
-}
-
-func buildShortNumberMetadata() *phonenumbers.PhoneMetadataCollection {
-	log.Println("Fetching ShortNumberMetadata.xml from Github")
-	body := fetchURL(shortNumberMetadataURL)
-
-	log.Println("Building new short number metadata collection")
-	collection, err := phonenumbers.BuildPhoneMetadataCollection(body, false, false, true)
-	if err != nil {
-		log.Fatalf("Error converting XML: %s", err)
-	}
-
-	// write it out as a protobuf
-	data, err := proto.Marshal(collection)
-	if err != nil {
-		log.Fatalf("Error marshalling metadata: %v", err)
-	}
-
-	log.Println("Writing new metadata_bin.go")
-	writeFile(shortNumberMetadataPath, generateBinFile("shortNumberMetadataData", data))
-	return collection
+	return data.Bytes(), nil
 }
 
 // generates the file contents for a data file
@@ -476,12 +504,3 @@ func readMappingsForDir(dir string) map[int]string {
 	log.Printf("Read %d mappings in %s\n", len(mappings), dir)
 	return mappings
 }
-
-func main() {
-	metadata := buildMetadata()
-	buildShortNumberMetadata()
-	buildRegions(metadata)
-	buildTimezones()
-	buildPrefixData(&carrier)
-	buildPrefixData(&geocoding)
-}