From ae0c46d6f1b1cd56a519042516a575fb7d35212f Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 4 Sep 2023 11:44:59 -0500 Subject: [PATCH] Move some locales code from goflow/envs --- go.mod | 1 + go.sum | 2 ++ i18n/country.go | 29 ++++++++++++++++ i18n/country_test.go | 29 ++++++++++++++++ i18n/language._test.go | 36 +++++++++++++++++++ i18n/language.go | 50 ++++++++++++++++++++++++++ i18n/locale.go | 79 ++++++++++++++++++++++++++++++++++++++++++ i18n/locale_test.go | 62 +++++++++++++++++++++++++++++++++ 8 files changed, 288 insertions(+) create mode 100644 i18n/country.go create mode 100644 i18n/country_test.go create mode 100644 i18n/language._test.go create mode 100644 i18n/language.go create mode 100644 i18n/locale.go create mode 100644 i18n/locale_test.go diff --git a/go.mod b/go.mod index 1b44d85..4edaaab 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/jmoiron/sqlx v1.3.5 github.com/lib/pq v1.10.9 github.com/nyaruka/librato v1.0.0 + github.com/nyaruka/null/v2 v2.0.3 github.com/nyaruka/phonenumbers v1.1.8 github.com/pkg/errors v0.9.1 github.com/shopspring/decimal v1.3.1 diff --git a/go.sum b/go.sum index 2659528..1024d8c 100644 --- a/go.sum +++ b/go.sum @@ -39,6 +39,8 @@ github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRU github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0= github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg= +github.com/nyaruka/null/v2 v2.0.3 h1:rdmMRQyVzrOF3Jff/gpU/7BDR9mQX0lcLl4yImsA3kw= +github.com/nyaruka/null/v2 v2.0.3/go.mod h1:OCVeCkCXwrg5/qE6RU0c1oUVZBy+ZDrT+xYg1XSaIWA= github.com/nyaruka/phonenumbers v1.1.8 h1:mjFu85FeoH2Wy18aOMUvxqi1GgAqiQSJsa/cCC5yu2s= github.com/nyaruka/phonenumbers v1.1.8/go.mod h1:DC7jZd321FqUe+qWSNcHi10tyIyGNXGcNbfkPvdp1Vs= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= diff --git a/i18n/country.go b/i18n/country.go new file mode 100644 index 0000000..5fef6ed --- /dev/null +++ b/i18n/country.go @@ -0,0 +1,29 @@ +package i18n + +import ( + "database/sql/driver" + + "github.com/nyaruka/null/v2" + "github.com/nyaruka/phonenumbers" +) + +// Country is a ISO 3166-1 alpha-2 country code +type Country string + +// NilCountry represents our nil, or unknown country +var NilCountry = Country("") + +// DeriveCountryFromTel attempts to derive a country code (e.g. RW) from a phone number +func DeriveCountryFromTel(number string) Country { + parsed, err := phonenumbers.Parse(number, "") + if err != nil { + return "" + } + return Country(phonenumbers.GetRegionCodeForNumber(parsed)) +} + +// Place nicely with NULLs if persisting to a database or JSON +func (c *Country) Scan(value any) error { return null.ScanString(value, c) } +func (c Country) Value() (driver.Value, error) { return null.StringValue(c) } +func (c Country) MarshalJSON() ([]byte, error) { return null.MarshalString(c) } +func (c *Country) UnmarshalJSON(b []byte) error { return null.UnmarshalString(b, c) } diff --git a/i18n/country_test.go b/i18n/country_test.go new file mode 100644 index 0000000..609dc4f --- /dev/null +++ b/i18n/country_test.go @@ -0,0 +1,29 @@ +package i18n_test + +import ( + "testing" + + "github.com/nyaruka/gocommon/i18n" + "github.com/stretchr/testify/assert" +) + +func TestDeriveCountryFromTel(t *testing.T) { + assert.Equal(t, i18n.Country("RW"), i18n.DeriveCountryFromTel("+250788383383")) + assert.Equal(t, i18n.Country("EC"), i18n.DeriveCountryFromTel("+593979000000")) + assert.Equal(t, i18n.NilCountry, i18n.DeriveCountryFromTel("1234")) + + v, err := i18n.Country("RW").Value() + assert.NoError(t, err) + assert.Equal(t, "RW", v) + + v, err = i18n.NilCountry.Value() + assert.NoError(t, err) + assert.Nil(t, v) + + var c i18n.Country + assert.NoError(t, c.Scan("RW")) + assert.Equal(t, i18n.Country("RW"), c) + + assert.NoError(t, c.Scan(nil)) + assert.Equal(t, i18n.NilCountry, c) +} diff --git a/i18n/language._test.go b/i18n/language._test.go new file mode 100644 index 0000000..190a8c4 --- /dev/null +++ b/i18n/language._test.go @@ -0,0 +1,36 @@ +package i18n_test + +import ( + "testing" + + "github.com/nyaruka/gocommon/i18n" + + "github.com/stretchr/testify/assert" +) + +func TestLanguage(t *testing.T) { + lang, err := i18n.ParseLanguage("ENG") + assert.NoError(t, err) + assert.Equal(t, i18n.Language("eng"), lang) + + _, err = i18n.ParseLanguage("base") + assert.EqualError(t, err, "iso-639-3 codes must be 3 characters, got: base") + + _, err = i18n.ParseLanguage("xzx") + assert.EqualError(t, err, "unrecognized language code: xzx") + + v, err := i18n.Language("eng").Value() + assert.NoError(t, err) + assert.Equal(t, "eng", v) + + v, err = i18n.NilLanguage.Value() + assert.NoError(t, err) + assert.Nil(t, v) + + var l i18n.Language + assert.NoError(t, l.Scan("eng")) + assert.Equal(t, i18n.Language("eng"), l) + + assert.NoError(t, l.Scan(nil)) + assert.Equal(t, i18n.NilLanguage, l) +} diff --git a/i18n/language.go b/i18n/language.go new file mode 100644 index 0000000..2572296 --- /dev/null +++ b/i18n/language.go @@ -0,0 +1,50 @@ +package i18n + +import ( + "database/sql/driver" + + "github.com/nyaruka/null/v2" + "github.com/pkg/errors" + "golang.org/x/text/language" +) + +// Language is holds a an ISO-639-3 language code. +type Language string + +// ISO639_1 returns the ISO-639-1 2-letter code for this language if it has one +func (l Language) ISO639_1() string { + base, err := language.ParseBase(string(l)) + if err != nil { + return "" + } + code := base.String() + + // not all languages have a 2-letter code + if len(code) != 2 { + return "" + } + return code +} + +// NilLanguage represents our nil, or unknown language +var NilLanguage = Language("") + +// ParseLanguage returns a new Language for the passed in language string, or an error if not found +func ParseLanguage(lang string) (Language, error) { + if len(lang) != 3 { + return NilLanguage, errors.Errorf("iso-639-3 codes must be 3 characters, got: %s", lang) + } + + base, err := language.ParseBase(lang) + if err != nil { + return NilLanguage, errors.Errorf("unrecognized language code: %s", lang) + } + + return Language(base.ISO3()), nil +} + +// Place nicely with NULLs if persisting to a database or JSON +func (l *Language) Scan(value any) error { return null.ScanString(value, l) } +func (l Language) Value() (driver.Value, error) { return null.StringValue(l) } +func (l Language) MarshalJSON() ([]byte, error) { return null.MarshalString(l) } +func (l *Language) UnmarshalJSON(b []byte) error { return null.UnmarshalString(b, l) } diff --git a/i18n/locale.go b/i18n/locale.go new file mode 100644 index 0000000..b40c85f --- /dev/null +++ b/i18n/locale.go @@ -0,0 +1,79 @@ +package i18n + +import ( + "database/sql/driver" + "fmt" + "strings" + + "github.com/nyaruka/null/v2" + "golang.org/x/text/language" +) + +// Locale is the combination of a language and optional country, e.g. US English, Brazilian Portuguese, encoded as the +// language code followed by the country code, e.g. eng-US, por-BR. Every locale is valid BCP47 language tag, tho not +// every BCP47 language tag is a valid goflow locale because we only use ISO-639-3 3 letter codes to represent language. +type Locale string + +// NewLocale creates a new locale +func NewLocale(l Language, c Country) Locale { + if l == NilLanguage { + return NilLocale + } + if c == NilCountry { + return Locale(l) // e.g. "eng", "por" + } + return Locale(fmt.Sprintf("%s-%s", l, c)) // e.g. "eng-US", "por-BR" +} + +func (l Locale) Split() (Language, Country) { + if l == NilLocale || len(l) < 3 { + return NilLanguage, NilCountry + } + + parts := strings.SplitN(string(l), "-", 2) + lang := Language(parts[0]) + country := NilCountry + if len(parts) > 1 { + country = Country(parts[1]) + } + + return lang, country +} + +func (l Locale) tag() language.Tag { + return language.MustParse(string(l)) +} + +var NilLocale = Locale("") + +// Place nicely with NULLs if persisting to a database or JSON +func (l *Locale) Scan(value any) error { return null.ScanString(value, l) } +func (l Locale) Value() (driver.Value, error) { return null.StringValue(l) } +func (l Locale) MarshalJSON() ([]byte, error) { return null.MarshalString(l) } +func (l *Locale) UnmarshalJSON(b []byte) error { return null.UnmarshalString(b, l) } + +// BCP47Matcher helps find best matching locale from a set of available locales +type BCP47Matcher struct { + codes []string + matcher language.Matcher +} + +// NewBCP47Matcher creates a new BCP47 matcher from the set of available locales which must be valid BCP47 tags. +func NewBCP47Matcher(codes ...string) *BCP47Matcher { + tags := make([]language.Tag, len(codes)) + for i := range codes { + tags[i] = language.MustParse(codes[i]) + } + return &BCP47Matcher{codes: codes, matcher: language.NewMatcher(tags)} +} + +func (m *BCP47Matcher) ForLocales(preferred ...Locale) string { + prefTags := make([]language.Tag, len(preferred)) + for i := range preferred { + prefTags[i] = preferred[i].tag() + } + + // see https://github.com/golang/go/issues/24211 + _, idx, _ := m.matcher.Match(prefTags...) + return m.codes[idx] +} diff --git a/i18n/locale_test.go b/i18n/locale_test.go new file mode 100644 index 0000000..8b02fde --- /dev/null +++ b/i18n/locale_test.go @@ -0,0 +1,62 @@ +package i18n_test + +import ( + "testing" + + "github.com/nyaruka/gocommon/i18n" + "github.com/stretchr/testify/assert" +) + +func TestLocale(t *testing.T) { + assert.Equal(t, i18n.Locale(""), i18n.NewLocale("", "")) + assert.Equal(t, i18n.Locale(""), i18n.NewLocale("", "US")) // invalid without language + assert.Equal(t, i18n.Locale("eng"), i18n.NewLocale("eng", "")) // valid without country + assert.Equal(t, i18n.Locale("eng-US"), i18n.NewLocale("eng", "US")) + + l, c := i18n.Locale("eng-US").Split() + assert.Equal(t, i18n.Language("eng"), l) + assert.Equal(t, i18n.Country("US"), c) + + l, c = i18n.NilLocale.Split() + assert.Equal(t, i18n.NilLanguage, l) + assert.Equal(t, i18n.NilCountry, c) + + v, err := i18n.NewLocale("eng", "US").Value() + assert.NoError(t, err) + assert.Equal(t, "eng-US", v) + + v, err = i18n.NilLanguage.Value() + assert.NoError(t, err) + assert.Nil(t, v) + + var lc i18n.Locale + assert.NoError(t, lc.Scan("eng-US")) + assert.Equal(t, i18n.Locale("eng-US"), lc) + + assert.NoError(t, lc.Scan(nil)) + assert.Equal(t, i18n.NilLocale, lc) +} + +func TesBCP47Matcher(t *testing.T) { + tests := []struct { + preferred []i18n.Locale + available []string + best string + }{ + {preferred: []i18n.Locale{"eng-US"}, available: []string{"es_EC", "en-US"}, best: "en-US"}, + {preferred: []i18n.Locale{"eng-US"}, available: []string{"es", "en"}, best: "en"}, + {preferred: []i18n.Locale{"eng"}, available: []string{"es-US", "en-UK"}, best: "en-UK"}, + {preferred: []i18n.Locale{"eng", "fra"}, available: []string{"fr-CA", "en-RW"}, best: "en-RW"}, + {preferred: []i18n.Locale{"eng", "fra"}, available: []string{"fra-CA", "eng-RW"}, best: "eng-RW"}, + {preferred: []i18n.Locale{"fra", "eng"}, available: []string{"fra-CA", "eng-RW"}, best: "fra-CA"}, + {preferred: []i18n.Locale{"spa"}, available: []string{"es-EC", "es-MX", "es-ES"}, best: "es-ES"}, + {preferred: []i18n.Locale{}, available: []string{"es_EC", "en-US"}, best: "es_EC"}, + } + + for _, tc := range tests { + m := i18n.NewBCP47Matcher(tc.available...) + best := m.ForLocales(tc.preferred...) + + assert.Equal(t, tc.best, best, "locale mismatch for preferred=%v available=%s", tc.preferred, tc.available) + } +}