Refactor buildmetadata to not require svn export

.gitignore

@@ -10,3 +10,4 @@ geocoding
 buildmetadata
 functions/*
 dist/
+_build

cmd/buildmetadata/main.go

@@ -24,6 +24,92 @@ import (
 	"github.com/nyaruka/phonenumbers"
 )
 
+func main() {
+	if err := buildMetadata(); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+func buildMetadata() error {
+	fmt.Print("Cloning upstream repo... ")
+
+	if err := cloneUpstreamRepo("[email protected]:google/libphonenumber.git"); err != nil {
+		return err
+	}
+
+	fmt.Print("OK\nBuilding number metadata...")
+
+	metadata, err := buildNumberMetadata("resources/PhoneNumberMetadata.xml", "metadataData", "metadata_bin.go", false)
+	if err != nil {
+		return err
+	}
+
+	fmt.Print("OK\nBuilding short number metadata...")
+
+	_, err = buildNumberMetadata("resources/ShortNumberMetadata.xml", "shortNumberMetadataData", "shortnumber_metadata_bin.go", true)
+	if err != nil {
+		return err
+	}
+
+	fmt.Print("OK\nBuilding region metadata...")
+
+	if err := buildRegionMetadata(metadata, "regionMapData", "countrycode_to_region_bin.go"); err != nil {
+		return err
+	}
+
+	fmt.Print("OK\nBuilding timezone metadata...")
+
+	buildTimezones()
+	buildPrefixData(&carrier)
+	buildPrefixData(&geocoding)
+	return nil
+}
+
+func cloneUpstreamRepo(url string) error {
+	os.RemoveAll("_build")
+
+	cmd := exec.Command("git", "clone", "--depth=1", url, "_build")
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("error cloning upstream repo: %w", err)
+	}
+
+	return nil
+}
+
+func buildNumberMetadata(srcFile, varName, dstFile string, short bool) (*phonenumbers.PhoneMetadataCollection, error) {
+	body, err := os.ReadFile("_build/" + srcFile)
+	if err != nil {
+		return nil, fmt.Errorf("error reading %s: %w", srcFile, err)
+	}
+
+	collection, err := phonenumbers.BuildPhoneMetadataCollection(body, false, false, short)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing %s: %w", srcFile, err)
+	}
+
+	data, err := proto.Marshal(collection)
+	if err != nil {
+		return nil, fmt.Errorf("error marshaling metadata as protobuf: %w", err)
+	}
+
+	if err := os.WriteFile(dstFile, generateBinFile(varName, data), os.FileMode(0664)); err != nil {
+		return nil, fmt.Errorf("error writing %s: %w", dstFile, err)
+	}
+
+	return collection, nil
+}
+
+func buildRegionMetadata(metadata *phonenumbers.PhoneMetadataCollection, varName, dstFile string) error {
+	regionMap := phonenumbers.BuildCountryCodeToRegionMap(metadata)
+
+	if err := writeIntStringArrayMap(dstFile, varName, regionMap); err != nil {
+		return fmt.Errorf("error writing %s: %w", dstFile, err)
+	}
+
+	return nil
+}
+
 type prefixBuild struct {
 	url     string
 	dir     string
@@ -32,18 +118,9 @@ type prefixBuild struct {
 }
 
 const (
-	metadataURL  = "https://raw.githubusercontent.com/googlei18n/libphonenumber/master/resources/PhoneNumberMetadata.xml"
-	metadataPath = "metadata_bin.go"
-
-	shortNumberMetadataURL  = "https://raw.githubusercontent.com/googlei18n/libphonenumber/master/resources/ShortNumberMetadata.xml"
-	shortNumberMetadataPath = "shortnumber_metadata_bin.go"
-
 	tzURL  = "https://raw.githubusercontent.com/googlei18n/libphonenumber/master/resources/timezones/map_data.txt"
 	tzPath = "prefix_to_timezone_bin.go"
 	tzVar  = "timezoneMapData"
-
-	regionPath = "countrycode_to_region_bin.go"
-	regionVar  = "regionMapData"
 )
 
 var carrier = prefixBuild{
@@ -122,18 +199,12 @@ func writeFile(filePath string, data []byte) {
 	}
 
 	fmt.Printf("Writing new %s\n", filePath)
-	err := ioutil.WriteFile(filePath, data, os.FileMode(0664))
+	err := os.WriteFile(filePath, data, os.FileMode(0664))
 	if err != nil {
 		log.Fatalf("Error writing '%s': %s", filePath, err)
 	}
 }
 
-func buildRegions(metadata *phonenumbers.PhoneMetadataCollection) {
-	log.Println("Building region map")
-	regionMap := phonenumbers.BuildCountryCodeToRegionMap(metadata)
-	writeIntStringArrayMap(regionPath, regionVar, regionMap)
-}
-
 func buildTimezones() {
 	log.Println("Building timezone map")
 	body := fetchURL(tzURL)
@@ -171,7 +242,7 @@ func buildTimezones() {
 	writeIntStringArrayMap(tzPath, tzVar, prefixMap)
 }
 
-func writeIntStringArrayMap(path string, varName string, prefixMap map[int][]string) {
+func writeIntStringArrayMap(path string, varName string, prefixMap map[int][]string) error {
 	// build lists of our keys and values
 	keys := make([]int, 0, len(prefixMap))
 	values := make([]string, 0, 255)
@@ -200,15 +271,15 @@ func writeIntStringArrayMap(path string, varName string, prefixMap map[int][]str
 	// first write our values, as length of string and raw bytes
 	joinedValues := strings.Join(values, "\n")
 	if err := binary.Write(data, binary.LittleEndian, uint32(len(joinedValues))); err != nil {
-		log.Fatal(err)
+		return err
 	}
 	if err := binary.Write(data, binary.LittleEndian, []byte(joinedValues)); err != nil {
-		log.Fatal(err)
+		return err
 	}
 
 	// then the number of keys
 	if err := binary.Write(data, binary.LittleEndian, uint32(len(keys))); err != nil {
-		log.Fatal(err)
+		return err
 	}
 
 	// we write our key / value pairs as a varint of the difference of the previous prefix
@@ -220,22 +291,22 @@ func writeIntStringArrayMap(path string, varName string, prefixMap map[int][]str
 		diff := key - last
 		l := binary.PutUvarint(intBuf, uint64(diff))
 		if err := binary.Write(data, binary.LittleEndian, intBuf[:l]); err != nil {
-			log.Fatal(err)
+			return err
 		}
 
 		// then our values
 		values := prefixMap[key]
 
 		// write our number of values
 		if err := binary.Write(data, binary.LittleEndian, uint8(len(values))); err != nil {
-			log.Fatal(err)
+			return err
 		}
 
 		// then each value as the interned index
 		for _, v := range values {
 			valueIntern := internMap[v]
 			if err := binary.Write(data, binary.LittleEndian, uint16(valueIntern)); err != nil {
-				log.Fatal(err)
+				return err
 			}
 		}
 
@@ -244,48 +315,8 @@ func writeIntStringArrayMap(path string, varName string, prefixMap map[int][]str
 
 	// then write our file
 	writeFile(path, generateBinFile(varName, data.Bytes()))
-}
-
-func buildMetadata() *phonenumbers.PhoneMetadataCollection {
-	log.Println("Fetching PhoneNumberMetadata.xml from Github")
-	body := fetchURL(metadataURL)
 
-	log.Println("Building new metadata collection")
-	collection, err := phonenumbers.BuildPhoneMetadataCollection(body, false, false, false)
-	if err != nil {
-		log.Fatalf("Error converting XML: %s", err)
-	}
-
-	// write it out as a protobuf
-	data, err := proto.Marshal(collection)
-	if err != nil {
-		log.Fatalf("Error marshalling metadata: %v", err)
-	}
-
-	log.Println("Writing new metadata_bin.go")
-	writeFile(metadataPath, generateBinFile("metadataData", data))
-	return collection
-}
-
-func buildShortNumberMetadata() *phonenumbers.PhoneMetadataCollection {
-	log.Println("Fetching ShortNumberMetadata.xml from Github")
-	body := fetchURL(shortNumberMetadataURL)
-
-	log.Println("Building new short number metadata collection")
-	collection, err := phonenumbers.BuildPhoneMetadataCollection(body, false, false, true)
-	if err != nil {
-		log.Fatalf("Error converting XML: %s", err)
-	}
-
-	// write it out as a protobuf
-	data, err := proto.Marshal(collection)
-	if err != nil {
-		log.Fatalf("Error marshalling metadata: %v", err)
-	}
-
-	log.Println("Writing new metadata_bin.go")
-	writeFile(shortNumberMetadataPath, generateBinFile("shortNumberMetadataData", data))
-	return collection
+	return nil
 }
 
 // generates the file contents for a data file
@@ -476,12 +507,3 @@ func readMappingsForDir(dir string) map[int]string {
 	log.Printf("Read %d mappings in %s\n", len(mappings), dir)
 	return mappings
 }
-
-func main() {
-	metadata := buildMetadata()
-	buildShortNumberMetadata()
-	buildRegions(metadata)
-	buildTimezones()
-	buildPrefixData(&carrier)
-	buildPrefixData(&geocoding)
-}