diff --git a/core/commands/connectors/db_connector.go b/core/commands/connectors/db_connector.go index acc7a76..2d85509 100644 --- a/core/commands/connectors/db_connector.go +++ b/core/commands/connectors/db_connector.go @@ -8,6 +8,7 @@ import ( "github.com/godruoyi/go-snowflake" "github.com/lib/pq" "github.com/meekyphotos/experive-cli/core/utils" + "log" "regexp" "strings" ) @@ -94,6 +95,7 @@ func (p *PgConnector) Write(data []map[string]interface{}) error { } _, err := stmt.Exec(vals...) if err != nil { + log.Println(err) return err } } diff --git a/core/commands/load_osm.go b/core/commands/load_osm.go index f8ff240..0037641 100644 --- a/core/commands/load_osm.go +++ b/core/commands/load_osm.go @@ -3,16 +3,18 @@ package commands import ( "github.com/meekyphotos/experive-cli/core/commands/connectors" "github.com/meekyphotos/experive-cli/core/commands/pipeline" + "github.com/meekyphotos/experive-cli/core/dataproviders" "github.com/meekyphotos/experive-cli/core/utils" "github.com/urfave/cli/v2" + "github.com/valyala/fastjson" + "os" "sync" "time" ) type OsmRunner struct { - NodeConnector connectors.Connector - WaysConnector connectors.Connector - RelationsConnector connectors.Connector + store dataproviders.Store + NodeConnector connectors.Connector } // ID int64 @@ -28,18 +30,6 @@ var osmFields = []connectors.Column{ {Name: "address", Type: connectors.Jsonb}, } -var wayFields = []connectors.Column{ - {Name: "osm_id", Type: connectors.Bigint, Indexed: true}, - {Name: "extratags", Type: connectors.Jsonb}, - {Name: "node_ids", Type: connectors.Jsonb}, -} - -var relationFields = []connectors.Column{ - {Name: "osm_id", Type: connectors.Bigint, Indexed: true}, - {Name: "extratags", Type: connectors.Jsonb}, - {Name: "members", Type: connectors.Jsonb}, -} - func determineNodesCols(c *utils.Config) []connectors.Column { cols := make([]connectors.Column, 0) cols = append(cols, osmFields...) @@ -63,69 +53,88 @@ func (r OsmRunner) Run(c *utils.Config) error { if dbErr != nil { return dbErr } - - r.WaysConnector = &connectors.PgConnector{ - Config: c, TableName: c.TableName + "_ways", Db: pg.Db, - } - r.RelationsConnector = &connectors.PgConnector{ - Config: c, TableName: c.TableName + "_relations", Db: pg.Db, - } + r.store = dataproviders.Store{} + os.RemoveAll("./db.tmp") // try to delete all to cleanup previous run + r.store.Open("./db.tmp") + defer func() { + os.RemoveAll("./db.tmp") + }() + defer r.store.Close() // no need to close ways & relations defer r.NodeConnector.Close() dbErr = r.NodeConnector.Init(determineNodesCols(c)) if dbErr != nil { return dbErr } - dbErr = r.WaysConnector.Init(wayFields) - if dbErr != nil { - return dbErr - } - dbErr = r.RelationsConnector.Init(relationFields) - if dbErr != nil { - return dbErr - } - nodeChannel, wayChannel, relationChannel, err := pipeline.ReadFromPbf(c.File, &pipeline.NoopBeat{}) + nodeChannel, wayChannel, err := pipeline.ReadFromPbf(c.File, &pipeline.NoopBeat{}) if err != nil { return err } - nodeRequests := pipeline.BatchRequest(nodeChannel, 10000, time.Second) - wayRequests := pipeline.BatchRequest(wayChannel, 10000, time.Second) - relationRequests := pipeline.BatchRequest(relationChannel, 10000, time.Second) - var pgWorkers sync.WaitGroup + nodeRequests := pipeline.BatchINodes(nodeChannel, 10000, time.Second) + var postProcessingWorkers sync.WaitGroup nodeBeat := &pipeline.ProgressBarBeat{OperationName: "Nodes"} - relationBeat := &pipeline.ProgressBarBeat{OperationName: "Relations"} waysBeat := &pipeline.ProgressBarBeat{OperationName: "Ways"} + actualBeat := &pipeline.ProgressBarBeat{OperationName: "Node written"} - pgWorkers.Add(1) + postProcessingWorkers.Add(1) go func() { - err := pipeline.ProcessChannel(nodeRequests, r.NodeConnector, nodeBeat) + err := pipeline.ProcessINodes(nodeRequests, r.store, nodeBeat) if err != nil { panic(err) } - pgWorkers.Done() + postProcessingWorkers.Done() }() - pgWorkers.Add(1) + postProcessingWorkers.Add(1) go func() { - err := pipeline.ProcessChannel(wayRequests, r.WaysConnector, waysBeat) - if err != nil { - panic(err) - } - pgWorkers.Done() + pipeline.ProcessNodeEnrichment(wayChannel, r.store, waysBeat) + postProcessingWorkers.Done() }() - pgWorkers.Add(1) + postProcessingWorkers.Wait() + + // I'm completely done with post processing.. now I should start writing stuff + storeChannel := r.store.Stream(func(value *fastjson.Value) map[string]interface{} { + baseObject := map[string]interface{}{ + "osm_id": value.GetInt64("osm_id"), + "osm_type": string(value.GetStringBytes("osm_type")), + "class": string(value.GetStringBytes("class")), + "type": string(value.GetStringBytes("type")), + "latitude": value.GetFloat64("latitude"), + "longitude": value.GetFloat64("longitude"), + "name": "", + "address": "", + "extratags": "", + } + name := value.GetObject("name") + if name != nil { + baseObject["name"] = string(name.MarshalTo([]byte{})) + } + address := value.GetObject("address") + if address != nil { + baseObject["address"] = string(address.MarshalTo([]byte{})) + } + + extratags := value.GetObject("extratags") + if extratags != nil { + baseObject["extratags"] = string(extratags.MarshalTo([]byte{})) + } + return baseObject + }) + rowsChannel := pipeline.BatchRequest(storeChannel, 10000, time.Second) + + var pgWorker sync.WaitGroup + pgWorker.Add(1) go func() { - err := pipeline.ProcessChannel(relationRequests, r.RelationsConnector, relationBeat) + err := pipeline.ProcessChannel(rowsChannel, r.NodeConnector, actualBeat) if err != nil { panic(err) } - pgWorkers.Done() + pgWorker.Done() }() - - pgWorkers.Wait() + pgWorker.Wait() return r.NodeConnector.CreateIndexes() } diff --git a/core/commands/osm/data.go b/core/commands/osm/data.go index 8e98523..e87a77f 100644 --- a/core/commands/osm/data.go +++ b/core/commands/osm/data.go @@ -1,6 +1,8 @@ package osm -import "time" +import ( + "time" +) type BoundingBox struct { Left float64 @@ -30,11 +32,14 @@ type Info struct { } type Node struct { - Content map[string]interface{} + Id int64 + Content []byte } type Way struct { - Content map[string]interface{} + Id int64 + Tags map[string]string + NodeIds []int64 } type MemberType int diff --git a/core/commands/osm/decode_tag.go b/core/commands/osm/decode_tag.go index 87acab0..d06a487 100644 --- a/core/commands/osm/decode_tag.go +++ b/core/commands/osm/decode_tag.go @@ -70,6 +70,7 @@ keyLoop: } key := string(keyBytes) val := string(stringTable[valueIDs[index]]) + if strings.Contains(key, "name") { names[key] = val } else { @@ -87,7 +88,9 @@ type tagUnpacker struct { var openPar = []byte(`{`) var openWithComma = []byte(`,"`) +var comma = []byte(`,`) var keyVal = []byte(`":"`) +var keyValPrimitive = []byte(`":`) var quotes = []byte(`"`) var endPar = []byte(`}`) var nameBytes = []byte(`name`) @@ -114,7 +117,26 @@ func (js *json) close() { func (js *json) toString() string { return js.buffer.String() } + func (js *json) add(key []byte, val []byte) { + js.prepareForKey() + js.buffer.Write(key) + js.buffer.Write(keyVal) + cleaned := carriageReturn.ReplaceAll(val, []byte{}) + js.buffer.Write(cleaned) + js.buffer.Write(quotes) +} + +func (js *json) addPrimitive(key []byte, val []byte) { + if len(val) > 0 { + js.prepareForKey() + js.buffer.Write(key) + js.buffer.Write(keyValPrimitive) + js.buffer.Write(val) + } +} + +func (js *json) prepareForKey() { if js.started { js.buffer.Write(openWithComma) } else { @@ -122,16 +144,11 @@ func (js *json) add(key []byte, val []byte) { js.buffer.Write(openPar) js.buffer.Write(quotes) } - js.buffer.Write(key) - js.buffer.Write(keyVal) - cleaned := carriageReturn.ReplaceAll(val, []byte{}) - js.buffer.Write(cleaned) - js.buffer.Write(quotes) } // Make tags map from stringtable and array of IDs (used in DenseNodes encoding). -func (tu *tagUnpacker) next() (string, string, string, string, string) { - var class, osmType string +func (tu *tagUnpacker) next() (string, string, string, []byte, []byte) { + var class, osmType []byte tagsJson := newJson() nameJson := newJson() addressJson := newJson() @@ -155,8 +172,8 @@ keyLoop: valBytes := tu.stringTable[valID] for _, b := range classDefiningAttributes { if bytes.Equal(b, keyBytes) { - class = string(b) - osmType = string(valBytes) + class = b + osmType = valBytes break // add key anyway } } diff --git a/core/commands/osm/decoder_data.go b/core/commands/osm/decoder_data.go index c9c7268..c178856 100644 --- a/core/commands/osm/decoder_data.go +++ b/core/commands/osm/decoder_data.go @@ -1,6 +1,7 @@ package osm import ( + "fmt" "github.com/meekyphotos/experive-cli/core/commands/pbf" "google.golang.org/protobuf/proto" ) @@ -48,6 +49,16 @@ func (dec *dataDecoder) parsePrimitiveGroup(pb *pbf.PrimitiveBlock, pg *pbf.Prim //} } +var osmIdBytes = []byte("osm_id") +var osmTypeBytes = []byte("osm_type") +var classBytes = []byte("class") +var typeBytes = []byte("type") +var latitudeBytes = []byte("latitude") +var longitudeBytes = []byte("longitude") +var metadataBytes = []byte("extratags") +var namesBytes = []byte("name") +var addressesBytes = []byte("address") + func (dec *dataDecoder) parseNodes(pb *pbf.PrimitiveBlock, nodes []*pbf.Node) { st := pb.GetStringtable().GetS() granularity := int64(pb.GetGranularity()) @@ -64,20 +75,7 @@ func (dec *dataDecoder) parseNodes(pb *pbf.PrimitiveBlock, nodes []*pbf.Node) { longitude := 1e-9 * float64(lonOffset+(granularity*lon)) tags, names, class, osmType := ExtractInfo(st, node.GetKeys(), node.GetVals()) - if len(tags) != 2 || len(names) != 2 { - - dec.q = append(dec.q, &Node{ - Content: map[string]interface{}{ - "osm_id": id, - "class": class, - "type": osmType, - "latitude": latitude, - "longitude": longitude, - "metadata": tags, - "names": names, - }, - }) - } + dec.addNodeQueue(tags, names, id, []byte(class), []byte(osmType), "", latitude, longitude) } } @@ -100,21 +98,27 @@ func (dec *dataDecoder) parseDenseNodes(pb *pbf.PrimitiveBlock, dn *pbf.DenseNod latitude := 1e-9 * float64(latOffset+(granularity*lat)) longitude := 1e-9 * float64(lonOffset+(granularity*lon)) tags, names, address, class, osmType := tu.next() - if len(tags) != 0 || len(names) != 0 { - dec.q = append(dec.q, &Node{ - Content: map[string]interface{}{ - "osm_id": id, - "osm_type": 'N', - "class": class, - "type": osmType, - "name": names, - "address": address, - "latitude": latitude, - "longitude": longitude, - "extratags": tags, - }, - }) - } + dec.addNodeQueue(tags, names, id, class, osmType, address, latitude, longitude) + } +} + +func (dec *dataDecoder) addNodeQueue(tags string, names string, id int64, class []byte, osmType []byte, address string, latitude float64, longitude float64) { + if len(tags) != 0 || len(names) != 0 { + json := newJson() + json.addPrimitive(osmIdBytes, []byte(fmt.Sprintf("%d", id))) + json.add(osmTypeBytes, []byte("N")) + json.add(classBytes, class) + json.add(typeBytes, osmType) + json.addPrimitive(namesBytes, []byte(names)) + json.addPrimitive(addressBytes, []byte(address)) + json.addPrimitive(metadataBytes, []byte(tags)) + json.addPrimitive(latitudeBytes, []byte(fmt.Sprintf("%f", latitude))) + json.addPrimitive(longitudeBytes, []byte(fmt.Sprintf("%f", longitude))) + json.close() + dec.q = append(dec.q, &Node{ + Id: id, + Content: []byte(json.toString()), + }) } } @@ -135,11 +139,7 @@ func (dec *dataDecoder) parseWays(pb *pbf.PrimitiveBlock, ways []*pbf.Way) { } dec.q = append(dec.q, &Way{ - Content: map[string]interface{}{ - "osm_id": id, - "extratags": tags, - "node_ids": nodeIDs, - }, + id, tags, nodeIDs, }) } } diff --git a/core/commands/pipeline/connector_processor.go b/core/commands/pipeline/connector_processor.go index 620c110..b1b43ea 100644 --- a/core/commands/pipeline/connector_processor.go +++ b/core/commands/pipeline/connector_processor.go @@ -2,6 +2,7 @@ package pipeline import ( "github.com/meekyphotos/experive-cli/core/commands/connectors" + "github.com/meekyphotos/experive-cli/core/dataproviders" "time" ) @@ -62,3 +63,40 @@ func BatchRequest(values <-chan map[string]interface{}, maxItems int, maxTimeout return batches } + +func BatchINodes(values <-chan *dataproviders.INode, maxItems int, maxTimeout time.Duration) chan []*dataproviders.INode { + batches := make(chan []*dataproviders.INode) + + go func() { + defer close(batches) + + for keepGoing := true; keepGoing; { + var batch []*dataproviders.INode + expire := time.After(maxTimeout) + for { + select { + case value, ok := <-values: + if !ok { + keepGoing = false + goto done + } + + batch = append(batch, value) + if len(batch) == maxItems { + goto done + } + + case <-expire: + goto done + } + } + + done: + if len(batch) > 0 { + batches <- batch + } + } + }() + + return batches +} diff --git a/core/commands/pipeline/enrich_node_with_ways.go b/core/commands/pipeline/enrich_node_with_ways.go new file mode 100644 index 0000000..2d8e4d9 --- /dev/null +++ b/core/commands/pipeline/enrich_node_with_ways.go @@ -0,0 +1,71 @@ +package pipeline + +import ( + "github.com/meekyphotos/experive-cli/core/commands/osm" + "github.com/meekyphotos/experive-cli/core/dataproviders" + "github.com/valyala/fastjson" + "strings" +) + +func ProcessNodeEnrichment(wayChannel <-chan *osm.Way, store dataproviders.Store, beat Heartbeat) { + beat.Start() + defer beat.Done() + arena := fastjson.Arena{} + for { + select { + case content := <-wayChannel: + if content == nil { + return + } + arena.Reset() + if len(content.NodeIds) > 0 { + nodes := store.FindMany(content.NodeIds...) + if len(nodes) > 0 { + for id, n := range nodes { + var address, extratags *fastjson.Value + if n.Exists("address") { + address = n.Get("address") + } else { + address = arena.NewObject() + + } + + if n.Exists("extratags") { + extratags = n.Get("extratags") + } else { + extratags = arena.NewObject() + } + + delete(content.Tags, "source") + if len(content.Tags) > 0 { + for k, v := range content.Tags { + if strings.HasPrefix(k, "name") { + lang := k[4:] + if lang == "" || lang == ":it" || lang == ":en" || lang == ":nl" { + address.Set("addr:name"+lang, arena.NewString(v)) + } + } else if strings.HasPrefix(k, "addr:") { + address.Set(k, arena.NewString(v)) + } else if "admin_level" == k { + address.Set("addr:admin_level", arena.NewString(v)) + } else { + extratags.Set("way:"+k, arena.NewString(v)) + } + } + + n.Set("address", address) + n.Set("extratags", extratags) + store.Save(&dataproviders.INode{ + Id: id, + Content: n.MarshalTo([]byte{}), + }) + } + } + } + } + beat.Beat(1) + default: + } + } + +} diff --git a/core/commands/pipeline/pbf_reader.go b/core/commands/pipeline/pbf_reader.go index 59704a8..b07022d 100644 --- a/core/commands/pipeline/pbf_reader.go +++ b/core/commands/pipeline/pbf_reader.go @@ -2,19 +2,19 @@ package pipeline import ( "github.com/meekyphotos/experive-cli/core/commands/osm" + "github.com/meekyphotos/experive-cli/core/dataproviders" "io" "os" "runtime" ) -func ReadFromPbf(path string, heartbeat Heartbeat) (chan map[string]interface{}, chan map[string]interface{}, chan map[string]interface{}, error) { +func ReadFromPbf(path string, heartbeat Heartbeat) (chan *dataproviders.INode, chan *osm.Way, error) { f, err := os.Open(path) if err != nil { - return nil, nil, nil, err + return nil, nil, err } - outNodes := make(chan map[string]interface{}, 100000) - outWays := make(chan map[string]interface{}, 100000) - outRelations := make(chan map[string]interface{}, 100000) + outNodes := make(chan *dataproviders.INode, 100000) + outWays := make(chan *osm.Way, 100000) heartbeat.Start() go func() { defer f.Close() @@ -25,6 +25,7 @@ func ReadFromPbf(path string, heartbeat Heartbeat) (chan map[string]interface{}, } d := osm.NewDecoder(open) d.SetBufferSize(osm.MaxBlobSize) + d.Skip(false, false, true) if err := d.Start(runtime.GOMAXPROCS(-1)); err != nil { panic(err) } @@ -40,20 +41,20 @@ func ReadFromPbf(path string, heartbeat Heartbeat) (chan map[string]interface{}, switch v.(type) { case *osm.Node: node := v.(*osm.Node) - outNodes <- node.Content + + outNodes <- &dataproviders.INode{ + Id: node.Id, + Content: node.Content, + } case *osm.Way: node := v.(*osm.Way) - outWays <- node.Content + outWays <- node case *osm.Relation: - node := v.(*osm.Relation) - outRelations <- node.Content - default: } } close(outNodes) close(outWays) - close(outRelations) }() - return outNodes, outWays, outRelations, nil + return outNodes, outWays, nil } diff --git a/core/commands/pipeline/temp_store.go b/core/commands/pipeline/temp_store.go new file mode 100644 index 0000000..d056d20 --- /dev/null +++ b/core/commands/pipeline/temp_store.go @@ -0,0 +1,23 @@ +package pipeline + +import ( + "github.com/meekyphotos/experive-cli/core/dataproviders" +) + +func ProcessINodes(channel chan []*dataproviders.INode, store dataproviders.Store, beat Heartbeat) error { + beat.Start() + defer beat.Done() + for { + select { + case content := <-channel: + i := len(content) + if i == 0 { + return nil + } + store.SaveMany(content...) + beat.Beat(i) + default: + } + } + +} diff --git a/core/dataproviders/redis.go b/core/dataproviders/redis.go index b79390a..72a5edc 100644 --- a/core/dataproviders/redis.go +++ b/core/dataproviders/redis.go @@ -2,6 +2,7 @@ package dataproviders import ( "encoding/binary" + "math" "github.com/dgraph-io/badger/v3" "github.com/valyala/fastjson" @@ -22,16 +23,62 @@ func (s *Store) Open(path string) { s.db = db s.pool = &fastjson.ParserPool{} } +func (s *Store) Close() error { + return s.db.Close() +} + +type Mapper func(*fastjson.Value) map[string]interface{} + +func (s *Store) Stream(mapper Mapper) chan map[string]interface{} { + out := make(chan map[string]interface{}, 10000) + go func() { + err := s.db.View(func(txn *badger.Txn) error { + itr := txn.NewIterator(badger.IteratorOptions{ + PrefetchValues: true, + PrefetchSize: 10000, + AllVersions: false, + }) + defer itr.Close() + for itr.Rewind(); itr.Valid(); itr.Next() { + item := itr.Item() + item.Value(func(val []byte) error { + parser := s.pool.Get() + defer s.pool.Put(parser) + + json, err := parser.ParseBytes(val) + if err != nil { + return err + } + out <- mapper(json) + return nil + }) + } + close(out) + return nil + }) + if err != nil { + panic(err) + } + }() + + return out +} -func uint64ToBytes(i uint64) []byte { +func Uint64ToBytes(i uint64) []byte { var buf [8]byte binary.BigEndian.PutUint64(buf[:], i) return buf[:] } -func (s *Store) Save(node INode) { +func Float64ToBytes(f float64) []byte { + var buf [8]byte + binary.BigEndian.PutUint64(buf[:], math.Float64bits(f)) + return buf[:] +} + +func (s *Store) Save(node *INode) { err := s.db.Update(func(txn *badger.Txn) error { - return saveInTransaction(txn, &node) + return saveInTransaction(txn, node) }) if err != nil { panic(err) @@ -39,22 +86,22 @@ func (s *Store) Save(node INode) { } type INode struct { - Id uint64 + Id int64 Content []byte } func saveInTransaction(txn *badger.Txn, n *INode) error { - err := txn.Set(uint64ToBytes(n.Id), n.Content) + err := txn.Set(Uint64ToBytes(uint64(n.Id)), n.Content) if err != nil { return err } return nil } -func (s *Store) SaveMany(nodes ...INode) { +func (s *Store) SaveMany(nodes ...*INode) { err := s.db.Update(func(txn *badger.Txn) error { for _, n := range nodes { - err := saveInTransaction(txn, &n) + err := saveInTransaction(txn, n) if err != nil { return err } @@ -66,7 +113,7 @@ func (s *Store) SaveMany(nodes ...INode) { } } -func (s *Store) FindOne(id uint64) *fastjson.Value { +func (s *Store) FindOne(id int64) *fastjson.Value { var json *fastjson.Value err := s.db.View(func(txn *badger.Txn) error { @@ -84,10 +131,12 @@ func (s *Store) FindOne(id uint64) *fastjson.Value { return json } -func (s *Store) readKey(txn *badger.Txn, id uint64) (*fastjson.Value, error) { +func (s *Store) readKey(txn *badger.Txn, id int64) (*fastjson.Value, error) { var json *fastjson.Value - item, err := txn.Get(uint64ToBytes(id)) - if err != nil { + item, err := txn.Get(Uint64ToBytes(uint64(id))) + if err == badger.ErrKeyNotFound { + return nil, nil + } else if err != nil { return nil, err } if item != nil { @@ -107,8 +156,8 @@ func (s *Store) readKey(txn *badger.Txn, id uint64) (*fastjson.Value, error) { } } -func (s *Store) FindMany(ids ...uint64) map[uint64]*fastjson.Value { - var out = make(map[uint64]*fastjson.Value, len(ids)) +func (s *Store) FindMany(ids ...int64) map[int64]*fastjson.Value { + var out = make(map[int64]*fastjson.Value, len(ids)) err := s.db.View(func(txn *badger.Txn) error { for _, id := range ids { diff --git a/go.mod b/go.mod index 1531877..c578218 100644 --- a/go.mod +++ b/go.mod @@ -3,17 +3,14 @@ module github.com/meekyphotos/experive-cli go 1.16 require ( - github.com/dgraph-io/badger/v3 v3.2103.1 // indirect + github.com/dgraph-io/badger/v3 v3.2103.1 github.com/godruoyi/go-snowflake v0.0.2-alpha github.com/jedib0t/go-pretty/v6 v6.2.4 - github.com/kr/pretty v0.1.0 // indirect github.com/lib/pq v1.10.2 github.com/schollz/progressbar/v3 v3.8.2 github.com/stretchr/testify v1.7.0 github.com/urfave/cli/v2 v2.3.0 github.com/valyala/fastjson v1.6.3 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/protobuf v1.26.0 - gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect ) diff --git a/go.sum b/go.sum index dd2f35e..0dfe2a4 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,6 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= @@ -21,6 +22,7 @@ github.com/dgraph-io/badger/v3 v3.2103.1 h1:zaX53IRg7ycxVlkd5pYdCeFp1FynD6qBGQoQ github.com/dgraph-io/badger/v3 v3.2103.1/go.mod h1:dULbq6ehJ5K0cGW/1TQ9iSfUk0gbSiToDWmWmTsJ53E= github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -89,6 +91,7 @@ github.com/schollz/progressbar/v3 v3.8.2/go.mod h1:9KHLdyuXczIsyStQwzvW8xiELskmX github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=