From b65138585adb53d1954e2a6430d8a1e0ad103cce Mon Sep 17 00:00:00 2001 From: Michele Masili Date: Sun, 18 Jul 2021 21:49:31 +0200 Subject: [PATCH] add first osm attempt (missing ways and relations) --- .github/workflows/go.yml | 31 + core/commands/connectors/db_connector.go | 14 +- core/commands/json/osm_parser.go | 6 + core/commands/json/osm_parser_test.go | 49 + core/commands/load_osm.go | 72 +- core/commands/osm/data.go | 68 + core/commands/osm/decode_tag.go | 157 ++ core/commands/osm/decoder.go | 338 ++++ core/commands/osm/decoder_data.go | 190 +++ core/commands/pbf/fileformat.pb.go | 347 ++++ core/commands/pbf/fileformat.proto | 39 + core/commands/pbf/osmformat.pb.go | 1465 +++++++++++++++++ core/commands/pbf/osmformat.proto | 228 +++ core/commands/pipeline/api.go | 37 - core/commands/pipeline/connector_processor.go | 42 +- core/commands/pipeline/pbf_reader.go | 59 + go.mod | 1 + go.sum | 88 + 18 files changed, 3183 insertions(+), 48 deletions(-) create mode 100644 .github/workflows/go.yml create mode 100644 core/commands/json/osm_parser.go create mode 100644 core/commands/json/osm_parser_test.go create mode 100644 core/commands/osm/data.go create mode 100644 core/commands/osm/decode_tag.go create mode 100644 core/commands/osm/decoder.go create mode 100644 core/commands/osm/decoder_data.go create mode 100644 core/commands/pbf/fileformat.pb.go create mode 100644 core/commands/pbf/fileformat.proto create mode 100644 core/commands/pbf/osmformat.pb.go create mode 100644 core/commands/pbf/osmformat.proto create mode 100644 core/commands/pipeline/pbf_reader.go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 0000000..d48a720 --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,31 @@ +name: Go + +on: + push: + tags: + - 'v*.*.*' + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.16 + + - name: Build Windows + run: env GOOS=windows GOARCH=amd64 go build -o experive.exe -v cmd/cli/main.go + + - name: Build Linux x64 + run: env GOOS=linux GOARCH=amd64 go build -o experive cmd/cli/main.go + + - name: Release + uses: softprops/action-gh-release@v1 + with: + files: | + experive.exe + experive + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/core/commands/connectors/db_connector.go b/core/commands/connectors/db_connector.go index f47c519..34d8979 100644 --- a/core/commands/connectors/db_connector.go +++ b/core/commands/connectors/db_connector.go @@ -68,11 +68,17 @@ func (p *PgConnector) Write(data []map[string]interface{}) error { case Text: vals[i] = row[c.Name] case Jsonb: - marshal, err := json.Marshal(row[c.Name]) - if err != nil { - return err + value := row[c.Name] + switch value.(type) { + case string: // already marshalled + vals[i] = value + default: + marshal, err := json.Marshal(value) + if err != nil { + return err + } + vals[i] = string(marshal) } - vals[i] = string(marshal) case Point: if lat, ok := row["latitude"]; ok { if lng, ok := row["longitude"]; ok { diff --git a/core/commands/json/osm_parser.go b/core/commands/json/osm_parser.go new file mode 100644 index 0000000..dd67192 --- /dev/null +++ b/core/commands/json/osm_parser.go @@ -0,0 +1,6 @@ +package json + +type OsmParser struct { + node int64 + ways int64 +} diff --git a/core/commands/json/osm_parser_test.go b/core/commands/json/osm_parser_test.go new file mode 100644 index 0000000..c0cc061 --- /dev/null +++ b/core/commands/json/osm_parser_test.go @@ -0,0 +1,49 @@ +package json + +import ( + "fmt" + "github.com/meekyphotos/experive-cli/core/commands/osm" + "io" + "os" + "runtime" + "testing" +) + +func (o *OsmParser) CountNodes() { + open, err := os.Open("netherlands.osm.pbf") + if err != nil { + panic(err) + } + d := osm.NewDecoder(open) + d.SetBufferSize(osm.MaxBlobSize) + d.Skip(false, true, true) + if err := d.Start(runtime.GOMAXPROCS(-1)); err != nil { + panic(err) + } + for { + v, err := d.Decode() + if err == io.EOF { + break + } + if err != nil { + panic(err) + } + + switch v.(type) { + case *osm.Node: + o.node++ + //fmt.Println(v) + case *osm.Way: + o.ways++ + case *osm.Relation: + default: + } + } +} + +// 9.991.103 in 14.69s only counting and skipping ways and relations +func TestParsing(t *testing.T) { + parser := OsmParser{} + parser.CountNodes() + fmt.Println(parser.node, parser.ways) +} diff --git a/core/commands/load_osm.go b/core/commands/load_osm.go index 0d90dab..288f488 100644 --- a/core/commands/load_osm.go +++ b/core/commands/load_osm.go @@ -1,11 +1,74 @@ package commands import ( - "fmt" + "github.com/meekyphotos/experive-cli/core/commands/connectors" + "github.com/meekyphotos/experive-cli/core/commands/pipeline" + "github.com/meekyphotos/experive-cli/core/utils" "github.com/urfave/cli/v2" + "sync" + "time" ) +type OsmRunner struct { + Connector connectors.Connector +} + +var osmFields = []connectors.Column{ + {Name: "id", Type: connectors.Snowflake}, + {Name: "osm_id", Type: connectors.Bigint, Indexed: true}, + {Name: "class", Type: connectors.Text}, + {Name: "type", Type: connectors.Text}, + {Name: "names", Type: connectors.Jsonb}, +} + +func determineOsmCols(c *utils.Config) []connectors.Column { + cols := make([]connectors.Column, 0) + cols = append(cols, osmFields...) + if c.InclKeyValues { + cols = append(cols, connectors.Column{Name: "metadata", Type: connectors.Jsonb}) + } + if c.UseGeom { + cols = append(cols, geomFields...) + } else { + cols = append(cols, latLngFields...) + } + return cols +} + +func (r OsmRunner) Run(c *utils.Config) error { + dbErr := r.Connector.Connect() + if dbErr != nil { + return dbErr + } + defer r.Connector.Close() + dbErr = r.Connector.Init(determineOsmCols(c)) + if dbErr != nil { + return dbErr + } + channel, err := pipeline.ReadFromPbf(c.File, &pipeline.NoopBeat{}) + if err != nil { + return err + } + requests := pipeline.BatchRequest(channel, 10000, time.Second) + var pgWorkers sync.WaitGroup + pgWorkers.Add(1) + go func() { + err := pipeline.ProcessChannel(requests, r.Connector) + if err != nil { + panic(err) + } + pgWorkers.Done() + }() + pgWorkers.Wait() + return r.Connector.CreateIndexes() +} + func LoadOsmMeta() *cli.Command { + stdAction := utils.DatabaseLoader{ + Runner: OsmRunner{}, + PasswordProvider: utils.TerminalPasswordReader{}, + } + return &cli.Command{ Name: "osm", Usage: "Load a osm dataset into target postgres", @@ -25,14 +88,11 @@ func LoadOsmMeta() *cli.Command { &cli.BoolFlag{Name: "latlong", Value: false, Usage: "Store coordinates in degrees of latitude & longitude."}, &cli.StringFlag{Name: "t", Aliases: []string{"table"}, Value: "planet_data", Usage: "Output table name"}, - &cli.BoolFlag{Name: "j", Aliases: []string{"json"}, Value: false, Usage: "Add tags without column to an additional json (key/value) column in the database tables."}, + &cli.BoolFlag{Name: "j", Aliases: []string{"json"}, Value: true, Usage: "Add tags without column to an additional json (key/value) column in the database tables."}, &cli.StringFlag{Name: "schema", Value: "public", Usage: "Use PostgreSQL schema SCHEMA for all tables, indexes, and functions in the pgsql output (default is no schema, i.e. the public schema is used)."}, }, UseShortOptionHandling: true, - Action: func(context *cli.Context) error { - fmt.Println("work in progress") - return nil - }, + Action: stdAction.DoLoad, } } diff --git a/core/commands/osm/data.go b/core/commands/osm/data.go new file mode 100644 index 0000000..f62679a --- /dev/null +++ b/core/commands/osm/data.go @@ -0,0 +1,68 @@ +package osm + +import "time" + +type BoundingBox struct { + Left float64 + Right float64 + Top float64 + Bottom float64 +} + +type Header struct { + BoundingBox *BoundingBox + RequiredFeatures []string + OptionalFeatures []string + WritingProgram string + Source string + OsmosisReplicationTimestamp time.Time + OsmosisReplicationSequenceNumber int64 + OsmosisReplicationBaseUrl string +} + +type Info struct { + Version int32 + Uid int32 + Timestamp time.Time + Changeset int64 + User string + Visible bool +} + +type Node struct { + OsmId int64 + Class string + Type string + Lat float64 + Lon float64 + Tags string + Names string +} + +type Way struct { + ID int64 + Tags map[string]string + NodeIDs []int64 + Info Info +} + +type MemberType int + +const ( + NodeType MemberType = iota + WayType + RelationType +) + +type Member struct { + ID int64 + Type MemberType + Role string +} + +type Relation struct { + ID int64 + Tags map[string]string + Members []Member + Info Info +} diff --git a/core/commands/osm/decode_tag.go b/core/commands/osm/decode_tag.go new file mode 100644 index 0000000..e668a12 --- /dev/null +++ b/core/commands/osm/decode_tag.go @@ -0,0 +1,157 @@ +package osm + +import ( + "bytes" + "strings" +) + +var classDefiningAttributes = [][]byte{ + []byte("tourism"), + []byte("leisure"), + []byte("shop"), + []byte("amenity"), + []byte("place"), + []byte("highway"), + []byte("man_made"), + []byte("railway"), + []byte("office"), + []byte("historic"), + []byte("aeroway"), + []byte("boundary"), + []byte("emergency"), + []byte("landuse"), + []byte("natural"), + []byte("club"), + []byte("craft"), + []byte("tunnel"), + []byte("public_transport"), +} +var ignoredTags = [][]byte{ + []byte("created_by"), + []byte("source"), + []byte("source:website"), + []byte("ref"), + []byte("crossing"), + []byte("crossing_ref"), + []byte("traffic_signals"), +} + +func extractTags(stringTable [][]byte, keyIDs, valueIDs []uint32) map[string]string { + tags := make(map[string]string, len(keyIDs)) + for index, keyID := range keyIDs { + key := string(stringTable[keyID]) + + val := string(stringTable[valueIDs[index]]) + tags[key] = val + } + return tags +} + +func ExtractInfo(stringTable [][]byte, keyIDs, valueIDs []uint32) (string, string, string, string) { + tags := make(map[string]string) + names := make(map[string]string) + var class, osmType string + +keyLoop: + for index, keyID := range keyIDs { + keyBytes := stringTable[keyID] + for _, b := range ignoredTags { + if bytes.Equal(b, keyBytes) { + continue keyLoop + } + } + for _, b := range classDefiningAttributes { + if bytes.Equal(b, keyBytes) { + class = string(b) + osmType = string(stringTable[valueIDs[index]]) + continue keyLoop + } + } + key := string(keyBytes) + val := string(stringTable[valueIDs[index]]) + if strings.Contains(key, "name") { + names[key] = val + } else { + tags[key] = val + } + } + return "{}", "{}", class, osmType +} + +type tagUnpacker struct { + stringTable [][]byte + keysVals []int32 + index int +} + +var openPar = []byte(`{`) +var openWithComma = []byte(`,"`) +var keyVal = []byte(`":"`) +var quotes = []byte(`"`) +var endPar = []byte(`}`) +var nameBytes = []byte(`name`) + +// Make tags map from stringtable and array of IDs (used in DenseNodes encoding). +func (tu *tagUnpacker) next() (string, string, string, string) { + var class, osmType string + tagsJson := strings.Builder{} + nameJson := strings.Builder{} + tagsJson.Write(openPar) + nameJson.Write(openPar) + firstName := true + firstTag := true +keyLoop: + for tu.index < len(tu.keysVals) { + keyID := tu.keysVals[tu.index] + tu.index++ + if keyID == 0 { + break + } + valID := tu.keysVals[tu.index] + tu.index++ + + keyBytes := tu.stringTable[keyID] + for _, b := range ignoredTags { + if bytes.Equal(b, keyBytes) { + continue keyLoop + } + } + + valBytes := tu.stringTable[valID] + for _, b := range classDefiningAttributes { + if bytes.Equal(b, keyBytes) { + class = string(b) + osmType = string(valBytes) + continue keyLoop + } + } + + if bytes.Contains(keyBytes, nameBytes) { + if !firstName { + nameJson.Write(openWithComma) + } else { + firstName = false + nameJson.Write(quotes) + } + nameJson.Write(keyBytes) + nameJson.Write(keyVal) + nameJson.Write(valBytes) + nameJson.Write(quotes) + } else { + if !firstTag { + tagsJson.Write(openWithComma) + } else { + firstTag = false + tagsJson.Write(quotes) + } + tagsJson.Write(keyBytes) + tagsJson.Write(keyVal) + tagsJson.Write(valBytes) + tagsJson.Write(quotes) + + } + } + tagsJson.Write(endPar) + nameJson.Write(endPar) + return tagsJson.String(), nameJson.String(), class, osmType +} diff --git a/core/commands/osm/decoder.go b/core/commands/osm/decoder.go new file mode 100644 index 0000000..46a3edb --- /dev/null +++ b/core/commands/osm/decoder.go @@ -0,0 +1,338 @@ +package osm + +import ( + "bytes" + "compress/zlib" + "encoding/binary" + "errors" + "fmt" + "github.com/meekyphotos/experive-cli/core/commands/pbf" + "google.golang.org/protobuf/proto" + "io" + "sync" + "time" +) + +const ( + maxBlobHeaderSize = 64 * 1024 + + initialBlobBufSize = 1 * 1024 * 1024 + + // MaxBlobSize is maximum supported blob size. + MaxBlobSize = 32 * 1024 * 1024 +) + +var ( + parseCapabilities = map[string]bool{ + "OsmSchema-V0.6": true, + "DenseNodes": true, + } +) + +type pair struct { + i interface{} + e error +} + +type Decoder struct { + r io.Reader + serializer chan pair + skipNodes bool + skipWays bool + skipRelations bool + + buf *bytes.Buffer + + // store header block + header *Header + // synchronize header deserialization + headerOnce sync.Once + + // for data decoders + inputs []chan<- pair + outputs []<-chan pair +} + +// SetBufferSize sets initial size of decoding buffer. Default value is 1MB, you can set higher value +// (for example, MaxBlobSize) for (probably) faster decoding, or lower value for reduced memory consumption. +// Any value will produce valid results; buffer will grow automatically if required. +func (dec *Decoder) SetBufferSize(n int) { + dec.buf = bytes.NewBuffer(make([]byte, 0, n)) +} + +func (dec *Decoder) Skip(nodes bool, ways bool, relation bool) { + dec.skipNodes = nodes + dec.skipWays = ways + dec.skipRelations = relation +} + +func NewDecoder(r io.Reader) *Decoder { + d := &Decoder{ + r: r, + serializer: make(chan pair, 8000), // typical PrimitiveBlock contains 8k OSM entities + } + d.SetBufferSize(initialBlobBufSize) + return d +} + +// Header returns file header. +func (dec *Decoder) Header() (*Header, error) { + // deserialize the file header + return dec.header, dec.readOSMHeader() +} + +// Start decoding process using n goroutines. +func (dec *Decoder) Start(n int) error { + if n < 1 { + n = 1 + } + + if err := dec.readOSMHeader(); err != nil { + return err + } + + // start data decoders + for i := 0; i < n; i++ { + input := make(chan pair) + output := make(chan pair) + go func() { + dd := new(dataDecoder) + dd.skipNodes = dec.skipNodes + dd.skipWays = dec.skipWays + dd.skipRelations = dec.skipRelations + for p := range input { + if p.e == nil { + // send decoded objects or decoding error + objects, err := dd.Decode(p.i.(*pbf.Blob)) + output <- pair{objects, err} + } else { + // send input error as is + output <- pair{nil, p.e} + } + } + close(output) + }() + + dec.inputs = append(dec.inputs, input) + dec.outputs = append(dec.outputs, output) + } + + // start reading OSMData + go func() { + var inputIndex int + for { + input := dec.inputs[inputIndex] + inputIndex = (inputIndex + 1) % n + + blobHeader, blob, err := dec.readFileBlock() + if err == nil && blobHeader.GetType() != "OSMData" { + err = fmt.Errorf("unexpected fileblock of type %s", blobHeader.GetType()) + } + if err == nil { + // send blob for decoding + input <- pair{blob, nil} + } else { + // send input error as is + input <- pair{nil, err} + for _, input := range dec.inputs { + close(input) + } + return + } + } + }() + + go func() { + var outputIndex int + for { + output := dec.outputs[outputIndex] + outputIndex = (outputIndex + 1) % n + + p := <-output + if p.i != nil { + // send decoded objects one by one + for _, o := range p.i.([]interface{}) { + dec.serializer <- pair{o, nil} + } + } + if p.e != nil { + // send input or decoding error + dec.serializer <- pair{nil, p.e} + close(dec.serializer) + return + } + } + }() + + return nil +} + +// Decode reads the next object from the input stream and returns either a +// pointer to Node, Way or Relation struct representing the underlying OpenStreetMap PBF +// data, or error encountered. The end of the input stream is reported by an io.EOF error. +// +// Decode is safe for parallel execution. Only first error encountered will be returned, +// subsequent invocations will return io.EOF. +func (dec *Decoder) Decode() (interface{}, error) { + p, ok := <-dec.serializer + if !ok { + return nil, io.EOF + } + return p.i, p.e +} + +func (dec *Decoder) readFileBlock() (*pbf.BlobHeader, *pbf.Blob, error) { + blobHeaderSize, err := dec.readBlobHeaderSize() + if err != nil { + return nil, nil, err + } + + blobHeader, err := dec.readBlobHeader(blobHeaderSize) + if err != nil { + return nil, nil, err + } + + blob, err := dec.readBlob(blobHeader) + if err != nil { + return nil, nil, err + } + + return blobHeader, blob, err +} + +func (dec *Decoder) readBlobHeaderSize() (uint32, error) { + dec.buf.Reset() + if _, err := io.CopyN(dec.buf, dec.r, 4); err != nil { + return 0, err + } + + size := binary.BigEndian.Uint32(dec.buf.Bytes()) + + if size >= maxBlobHeaderSize { + return 0, errors.New("BlobHeader size >= 64Kb") + } + return size, nil +} + +func (dec *Decoder) readBlobHeader(size uint32) (*pbf.BlobHeader, error) { + dec.buf.Reset() + if _, err := io.CopyN(dec.buf, dec.r, int64(size)); err != nil { + return nil, err + } + + blobHeader := new(pbf.BlobHeader) + if err := proto.Unmarshal(dec.buf.Bytes(), blobHeader); err != nil { + return nil, err + } + + if blobHeader.GetDatasize() >= MaxBlobSize { + return nil, errors.New("Blob size >= 32Mb") + } + return blobHeader, nil +} + +func (dec *Decoder) readBlob(blobHeader *pbf.BlobHeader) (*pbf.Blob, error) { + dec.buf.Reset() + if _, err := io.CopyN(dec.buf, dec.r, int64(blobHeader.GetDatasize())); err != nil { + return nil, err + } + + blob := new(pbf.Blob) + if err := proto.Unmarshal(dec.buf.Bytes(), blob); err != nil { + return nil, err + } + return blob, nil +} + +func getData(blob *pbf.Blob) ([]byte, error) { + switch blob.Data.(type) { + case *pbf.Blob_Raw: + return blob.GetRaw(), nil + + case *pbf.Blob_ZlibData: + r, err := zlib.NewReader(bytes.NewReader(blob.GetZlibData())) + if err != nil { + return nil, err + } + buf := bytes.NewBuffer(make([]byte, 0, blob.GetRawSize()+bytes.MinRead)) + _, err = buf.ReadFrom(r) + if err != nil { + return nil, err + } + if buf.Len() != int(blob.GetRawSize()) { + err = fmt.Errorf("raw blob data size %d but expected %d", buf.Len(), blob.GetRawSize()) + return nil, err + } + return buf.Bytes(), nil + + default: + return nil, fmt.Errorf("unhandled blob data type %T", blob.Data) + } +} + +func (dec *Decoder) readOSMHeader() error { + var err error + dec.headerOnce.Do(func() { + var blobHeader *pbf.BlobHeader + var blob *pbf.Blob + blobHeader, blob, err = dec.readFileBlock() + if err == nil { + if blobHeader.GetType() == "OSMHeader" { + err = dec.decodeOSMHeader(blob) + } else { + err = fmt.Errorf("unexpected first fileblock of type %s", blobHeader.GetType()) + } + } + }) + + return err +} + +func (dec *Decoder) decodeOSMHeader(blob *pbf.Blob) error { + data, err := getData(blob) + if err != nil { + return err + } + + headerBlock := new(pbf.HeaderBlock) + if err := proto.Unmarshal(data, headerBlock); err != nil { + return err + } + + // Check we have the parse capabilities + requiredFeatures := headerBlock.GetRequiredFeatures() + for _, feature := range requiredFeatures { + if !parseCapabilities[feature] { + return fmt.Errorf("parser does not have %s capability", feature) + } + } + + // Read properties to header struct + header := &Header{ + RequiredFeatures: headerBlock.GetRequiredFeatures(), + OptionalFeatures: headerBlock.GetOptionalFeatures(), + WritingProgram: headerBlock.GetWritingprogram(), + Source: headerBlock.GetSource(), + OsmosisReplicationBaseUrl: headerBlock.GetOsmosisReplicationBaseUrl(), + OsmosisReplicationSequenceNumber: headerBlock.GetOsmosisReplicationSequenceNumber(), + } + + // convert timestamp epoch seconds to golang time structure if it exists + if headerBlock.OsmosisReplicationTimestamp != nil { + header.OsmosisReplicationTimestamp = time.Unix(*headerBlock.OsmosisReplicationTimestamp, 0) + } + // read bounding box if it exists + if headerBlock.Bbox != nil { + // Units are always in nanodegree and do not obey granularity rules. See osmformat.proto + header.BoundingBox = &BoundingBox{ + Left: 1e-9 * float64(*headerBlock.Bbox.Left), + Right: 1e-9 * float64(*headerBlock.Bbox.Right), + Bottom: 1e-9 * float64(*headerBlock.Bbox.Bottom), + Top: 1e-9 * float64(*headerBlock.Bbox.Top), + } + } + + dec.header = header + + return nil +} diff --git a/core/commands/osm/decoder_data.go b/core/commands/osm/decoder_data.go new file mode 100644 index 0000000..4dd7fc4 --- /dev/null +++ b/core/commands/osm/decoder_data.go @@ -0,0 +1,190 @@ +package osm + +import ( + "github.com/meekyphotos/experive-cli/core/commands/pbf" + "google.golang.org/protobuf/proto" + "time" +) + +type dataDecoder struct { + q []interface{} + skipNodes bool + skipWays bool + skipRelations bool +} + +func (dec *dataDecoder) Decode(blob *pbf.Blob) ([]interface{}, error) { + dec.q = make([]interface{}, 0, 8000) // typical PrimitiveBlock contains 8k OSM entities + + data, err := getData(blob) + if err != nil { + return nil, err + } + + primitiveBlock := &pbf.PrimitiveBlock{} + if err := proto.Unmarshal(data, primitiveBlock); err != nil { + return nil, err + } + + dec.parsePrimitiveBlock(primitiveBlock) + return dec.q, nil +} + +func (dec *dataDecoder) parsePrimitiveBlock(pb *pbf.PrimitiveBlock) { + for _, pg := range pb.GetPrimitivegroup() { + dec.parsePrimitiveGroup(pb, pg) + } +} + +func (dec *dataDecoder) parsePrimitiveGroup(pb *pbf.PrimitiveBlock, pg *pbf.PrimitiveGroup) { + if !dec.skipNodes { + dec.parseNodes(pb, pg.GetNodes()) + dec.parseDenseNodes(pb, pg.GetDense()) + } + if !dec.skipWays { + dec.parseWays(pb, pg.GetWays()) + } + if !dec.skipRelations { + dec.parseRelations(pb, pg.GetRelations()) + } +} + +func (dec *dataDecoder) parseNodes(pb *pbf.PrimitiveBlock, nodes []*pbf.Node) { + st := pb.GetStringtable().GetS() + granularity := int64(pb.GetGranularity()) + + latOffset := pb.GetLatOffset() + lonOffset := pb.GetLonOffset() + // identify unwanted keys + for _, node := range nodes { + id := node.GetId() + lat := node.GetLat() + lon := node.GetLon() + + latitude := 1e-9 * float64(latOffset+(granularity*lat)) + longitude := 1e-9 * float64(lonOffset+(granularity*lon)) + + tags, names, class, osmType := ExtractInfo(st, node.GetKeys(), node.GetVals()) + if len(tags) != 2 || len(names) != 2 { + dec.q = append(dec.q, &Node{ + OsmId: id, + Lat: latitude, + Lon: longitude, + Tags: tags, + Names: names, + Class: class, + Type: osmType, + }) + } + } + +} + +func (dec *dataDecoder) parseDenseNodes(pb *pbf.PrimitiveBlock, dn *pbf.DenseNodes) { + st := pb.GetStringtable().GetS() + granularity := int64(pb.GetGranularity()) + latOffset := pb.GetLatOffset() + lonOffset := pb.GetLonOffset() + ids := dn.GetId() + lats := dn.GetLat() + lons := dn.GetLon() + + tu := tagUnpacker{st, dn.GetKeysVals(), 0} + var id, lat, lon int64 + for index := range ids { + id = ids[index] + id + lat = lats[index] + lat + lon = lons[index] + lon + latitude := 1e-9 * float64(latOffset+(granularity*lat)) + longitude := 1e-9 * float64(lonOffset+(granularity*lon)) + tags, names, class, osmType := tu.next() + if len(tags) != 2 || len(names) != 2 { + dec.q = append(dec.q, &Node{id, class, osmType, latitude, longitude, tags, names}) + } + } +} + +func (dec *dataDecoder) parseWays(pb *pbf.PrimitiveBlock, ways []*pbf.Way) { + st := pb.GetStringtable().GetS() + + for _, way := range ways { + id := way.GetId() + + tags := extractTags(st, way.GetKeys(), way.GetVals()) + + refs := way.GetRefs() + var nodeID int64 + nodeIDs := make([]int64, len(refs)) + for index := range refs { + nodeID = refs[index] + nodeID // delta encoding + nodeIDs[index] = nodeID + } + dec.q = append(dec.q, &Way{ID: id, Tags: tags, NodeIDs: nodeIDs}) + } +} + +// Make relation members from stringtable and three parallel arrays of IDs. +func extractMembers(stringTable [][]byte, rel *pbf.Relation) []Member { + memIDs := rel.GetMemids() + types := rel.GetTypes() + roleIDs := rel.GetRolesSid() + + var memID int64 + members := make([]Member, len(memIDs)) + for index := range memIDs { + memID = memIDs[index] + memID // delta encoding + + var memType MemberType + switch types[index] { + case pbf.Relation_NODE: + memType = NodeType + case pbf.Relation_WAY: + memType = WayType + case pbf.Relation_RELATION: + memType = RelationType + } + + role := stringTable[roleIDs[index]] + + members[index] = Member{memID, memType, string(role)} + } + + return members +} + +func (dec *dataDecoder) parseRelations(pb *pbf.PrimitiveBlock, relations []*pbf.Relation) { + st := pb.GetStringtable().GetS() + dateGranularity := int64(pb.GetDateGranularity()) + + for _, rel := range relations { + id := rel.GetId() + tags := extractTags(st, rel.GetKeys(), rel.GetVals()) + members := extractMembers(st, rel) + info := extractInfo(st, rel.GetInfo(), dateGranularity) + + dec.q = append(dec.q, &Relation{id, tags, members, info}) + } +} + +func extractInfo(stringTable [][]byte, i *pbf.Info, dateGranularity int64) Info { + info := Info{Visible: true} + + if i != nil { + info.Version = i.GetVersion() + + millisec := time.Duration(i.GetTimestamp()*dateGranularity) * time.Millisecond + info.Timestamp = time.Unix(0, millisec.Nanoseconds()).UTC() + + info.Changeset = i.GetChangeset() + + info.Uid = i.GetUid() + + info.User = string(stringTable[i.GetUserSid()]) + + if i.Visible != nil { + info.Visible = i.GetVisible() + } + } + + return info +} diff --git a/core/commands/pbf/fileformat.pb.go b/core/commands/pbf/fileformat.pb.go new file mode 100644 index 0000000..3ddc644 --- /dev/null +++ b/core/commands/pbf/fileformat.pb.go @@ -0,0 +1,347 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.26.0 +// protoc v3.5.0 +// source: fileformat.proto + +package pbf + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type Blob struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + RawSize *int32 `protobuf:"varint,2,opt,name=raw_size,json=rawSize" json:"raw_size,omitempty"` // When compressed, the uncompressed size + // Types that are assignable to Data: + // *Blob_Raw + // *Blob_ZlibData + // *Blob_LzmaData + // *Blob_OBSOLETEBzip2Data + // *Blob_Lz4Data + // *Blob_ZstdData + Data isBlob_Data `protobuf_oneof:"data"` +} + +func (x *Blob) Reset() { + *x = Blob{} + if protoimpl.UnsafeEnabled { + mi := &file_fileformat_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Blob) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Blob) ProtoMessage() {} + +func (x *Blob) ProtoReflect() protoreflect.Message { + mi := &file_fileformat_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Blob.ProtoReflect.Descriptor instead. +func (*Blob) Descriptor() ([]byte, []int) { + return file_fileformat_proto_rawDescGZIP(), []int{0} +} + +func (x *Blob) GetRawSize() int32 { + if x != nil && x.RawSize != nil { + return *x.RawSize + } + return 0 +} + +func (m *Blob) GetData() isBlob_Data { + if m != nil { + return m.Data + } + return nil +} + +func (x *Blob) GetRaw() []byte { + if x, ok := x.GetData().(*Blob_Raw); ok { + return x.Raw + } + return nil +} + +func (x *Blob) GetZlibData() []byte { + if x, ok := x.GetData().(*Blob_ZlibData); ok { + return x.ZlibData + } + return nil +} + +func (x *Blob) GetLzmaData() []byte { + if x, ok := x.GetData().(*Blob_LzmaData); ok { + return x.LzmaData + } + return nil +} + +// Deprecated: Do not use. +func (x *Blob) GetOBSOLETEBzip2Data() []byte { + if x, ok := x.GetData().(*Blob_OBSOLETEBzip2Data); ok { + return x.OBSOLETEBzip2Data + } + return nil +} + +func (x *Blob) GetLz4Data() []byte { + if x, ok := x.GetData().(*Blob_Lz4Data); ok { + return x.Lz4Data + } + return nil +} + +func (x *Blob) GetZstdData() []byte { + if x, ok := x.GetData().(*Blob_ZstdData); ok { + return x.ZstdData + } + return nil +} + +type isBlob_Data interface { + isBlob_Data() +} + +type Blob_Raw struct { + Raw []byte `protobuf:"bytes,1,opt,name=raw,oneof"` // No compression +} + +type Blob_ZlibData struct { + // Possible compressed versions of the data. + ZlibData []byte `protobuf:"bytes,3,opt,name=zlib_data,json=zlibData,oneof"` +} + +type Blob_LzmaData struct { + // For LZMA compressed data (optional) + LzmaData []byte `protobuf:"bytes,4,opt,name=lzma_data,json=lzmaData,oneof"` +} + +type Blob_OBSOLETEBzip2Data struct { + // Formerly used for bzip2 compressed data. Deprecated in 2010. + // + // Deprecated: Do not use. + OBSOLETEBzip2Data []byte `protobuf:"bytes,5,opt,name=OBSOLETE_bzip2_data,json=OBSOLETEBzip2Data,oneof"` // Don't reuse this tag number. +} + +type Blob_Lz4Data struct { + // For LZ4 compressed data (optional) + Lz4Data []byte `protobuf:"bytes,6,opt,name=lz4_data,json=lz4Data,oneof"` +} + +type Blob_ZstdData struct { + // For ZSTD compressed data (optional) + ZstdData []byte `protobuf:"bytes,7,opt,name=zstd_data,json=zstdData,oneof"` +} + +func (*Blob_Raw) isBlob_Data() {} + +func (*Blob_ZlibData) isBlob_Data() {} + +func (*Blob_LzmaData) isBlob_Data() {} + +func (*Blob_OBSOLETEBzip2Data) isBlob_Data() {} + +func (*Blob_Lz4Data) isBlob_Data() {} + +func (*Blob_ZstdData) isBlob_Data() {} + +type BlobHeader struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Type *string `protobuf:"bytes,1,req,name=type" json:"type,omitempty"` + Indexdata []byte `protobuf:"bytes,2,opt,name=indexdata" json:"indexdata,omitempty"` + Datasize *int32 `protobuf:"varint,3,req,name=datasize" json:"datasize,omitempty"` +} + +func (x *BlobHeader) Reset() { + *x = BlobHeader{} + if protoimpl.UnsafeEnabled { + mi := &file_fileformat_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *BlobHeader) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BlobHeader) ProtoMessage() {} + +func (x *BlobHeader) ProtoReflect() protoreflect.Message { + mi := &file_fileformat_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BlobHeader.ProtoReflect.Descriptor instead. +func (*BlobHeader) Descriptor() ([]byte, []int) { + return file_fileformat_proto_rawDescGZIP(), []int{1} +} + +func (x *BlobHeader) GetType() string { + if x != nil && x.Type != nil { + return *x.Type + } + return "" +} + +func (x *BlobHeader) GetIndexdata() []byte { + if x != nil { + return x.Indexdata + } + return nil +} + +func (x *BlobHeader) GetDatasize() int32 { + if x != nil && x.Datasize != nil { + return *x.Datasize + } + return 0 +} + +var File_fileformat_proto protoreflect.FileDescriptor + +var file_fileformat_proto_rawDesc = []byte{ + 0x0a, 0x10, 0x66, 0x69, 0x6c, 0x65, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x22, 0xed, 0x01, 0x0a, 0x04, 0x42, 0x6c, 0x6f, 0x62, 0x12, 0x19, 0x0a, 0x08, 0x72, + 0x61, 0x77, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x72, + 0x61, 0x77, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x12, 0x0a, 0x03, 0x72, 0x61, 0x77, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x03, 0x72, 0x61, 0x77, 0x12, 0x1d, 0x0a, 0x09, 0x7a, 0x6c, + 0x69, 0x62, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, + 0x08, 0x7a, 0x6c, 0x69, 0x62, 0x44, 0x61, 0x74, 0x61, 0x12, 0x1d, 0x0a, 0x09, 0x6c, 0x7a, 0x6d, + 0x61, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x08, + 0x6c, 0x7a, 0x6d, 0x61, 0x44, 0x61, 0x74, 0x61, 0x12, 0x34, 0x0a, 0x13, 0x4f, 0x42, 0x53, 0x4f, + 0x4c, 0x45, 0x54, 0x45, 0x5f, 0x62, 0x7a, 0x69, 0x70, 0x32, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x0c, 0x42, 0x02, 0x18, 0x01, 0x48, 0x00, 0x52, 0x11, 0x4f, 0x42, 0x53, + 0x4f, 0x4c, 0x45, 0x54, 0x45, 0x42, 0x7a, 0x69, 0x70, 0x32, 0x44, 0x61, 0x74, 0x61, 0x12, 0x1b, + 0x0a, 0x08, 0x6c, 0x7a, 0x34, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, + 0x48, 0x00, 0x52, 0x07, 0x6c, 0x7a, 0x34, 0x44, 0x61, 0x74, 0x61, 0x12, 0x1d, 0x0a, 0x09, 0x7a, + 0x73, 0x74, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, + 0x52, 0x08, 0x7a, 0x73, 0x74, 0x64, 0x44, 0x61, 0x74, 0x61, 0x42, 0x06, 0x0a, 0x04, 0x64, 0x61, + 0x74, 0x61, 0x22, 0x5a, 0x0a, 0x0a, 0x42, 0x6c, 0x6f, 0x62, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, + 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x02, 0x28, 0x09, 0x52, 0x04, + 0x74, 0x79, 0x70, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x64, 0x61, 0x74, + 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x64, 0x61, + 0x74, 0x61, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x61, 0x74, 0x61, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x03, + 0x20, 0x02, 0x28, 0x05, 0x52, 0x08, 0x64, 0x61, 0x74, 0x61, 0x73, 0x69, 0x7a, 0x65, +} + +var ( + file_fileformat_proto_rawDescOnce sync.Once + file_fileformat_proto_rawDescData = file_fileformat_proto_rawDesc +) + +func file_fileformat_proto_rawDescGZIP() []byte { + file_fileformat_proto_rawDescOnce.Do(func() { + file_fileformat_proto_rawDescData = protoimpl.X.CompressGZIP(file_fileformat_proto_rawDescData) + }) + return file_fileformat_proto_rawDescData +} + +var file_fileformat_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_fileformat_proto_goTypes = []interface{}{ + (*Blob)(nil), // 0: Blob + (*BlobHeader)(nil), // 1: BlobHeader +} +var file_fileformat_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_fileformat_proto_init() } +func file_fileformat_proto_init() { + if File_fileformat_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_fileformat_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Blob); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_fileformat_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*BlobHeader); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_fileformat_proto_msgTypes[0].OneofWrappers = []interface{}{ + (*Blob_Raw)(nil), + (*Blob_ZlibData)(nil), + (*Blob_LzmaData)(nil), + (*Blob_OBSOLETEBzip2Data)(nil), + (*Blob_Lz4Data)(nil), + (*Blob_ZstdData)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_fileformat_proto_rawDesc, + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_fileformat_proto_goTypes, + DependencyIndexes: file_fileformat_proto_depIdxs, + MessageInfos: file_fileformat_proto_msgTypes, + }.Build() + File_fileformat_proto = out.File + file_fileformat_proto_rawDesc = nil + file_fileformat_proto_goTypes = nil + file_fileformat_proto_depIdxs = nil +} diff --git a/core/commands/pbf/fileformat.proto b/core/commands/pbf/fileformat.proto new file mode 100644 index 0000000..528bb4f --- /dev/null +++ b/core/commands/pbf/fileformat.proto @@ -0,0 +1,39 @@ +syntax = "proto2"; + +// +// STORAGE LAYER: Storing primitives. +// + +message Blob { + optional int32 raw_size = 2; // When compressed, the uncompressed size + + oneof data { + bytes raw = 1; // No compression + + // Possible compressed versions of the data. + bytes zlib_data = 3; + + // For LZMA compressed data (optional) + bytes lzma_data = 4; + + // Formerly used for bzip2 compressed data. Deprecated in 2010. + bytes OBSOLETE_bzip2_data = 5 [deprecated=true]; // Don't reuse this tag number. + + // For LZ4 compressed data (optional) + bytes lz4_data = 6; + + // For ZSTD compressed data (optional) + bytes zstd_data = 7; + } +} + +/* A file contains an sequence of fileblock headers, each prefixed by +their length in network byte order, followed by a data block +containing the actual data. Types starting with a "_" are reserved. +*/ + +message BlobHeader { + required string type = 1; + optional bytes indexdata = 2; + required int32 datasize = 3; +} \ No newline at end of file diff --git a/core/commands/pbf/osmformat.pb.go b/core/commands/pbf/osmformat.pb.go new file mode 100644 index 0000000..20a1187 --- /dev/null +++ b/core/commands/pbf/osmformat.pb.go @@ -0,0 +1,1465 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.26.0 +// protoc v3.5.0 +// source: osmformat.proto + +package pbf + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type Relation_MemberType int32 + +const ( + Relation_NODE Relation_MemberType = 0 + Relation_WAY Relation_MemberType = 1 + Relation_RELATION Relation_MemberType = 2 +) + +// Enum value maps for Relation_MemberType. +var ( + Relation_MemberType_name = map[int32]string{ + 0: "NODE", + 1: "WAY", + 2: "RELATION", + } + Relation_MemberType_value = map[string]int32{ + "NODE": 0, + "WAY": 1, + "RELATION": 2, + } +) + +func (x Relation_MemberType) Enum() *Relation_MemberType { + p := new(Relation_MemberType) + *p = x + return p +} + +func (x Relation_MemberType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (Relation_MemberType) Descriptor() protoreflect.EnumDescriptor { + return file_osmformat_proto_enumTypes[0].Descriptor() +} + +func (Relation_MemberType) Type() protoreflect.EnumType { + return &file_osmformat_proto_enumTypes[0] +} + +func (x Relation_MemberType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Do not use. +func (x *Relation_MemberType) UnmarshalJSON(b []byte) error { + num, err := protoimpl.X.UnmarshalJSONEnum(x.Descriptor(), b) + if err != nil { + return err + } + *x = Relation_MemberType(num) + return nil +} + +// Deprecated: Use Relation_MemberType.Descriptor instead. +func (Relation_MemberType) EnumDescriptor() ([]byte, []int) { + return file_osmformat_proto_rawDescGZIP(), []int{11, 0} +} + +type HeaderBlock struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Bbox *HeaderBBox `protobuf:"bytes,1,opt,name=bbox" json:"bbox,omitempty"` + // Additional tags to aid in parsing this dataset + RequiredFeatures []string `protobuf:"bytes,4,rep,name=required_features,json=requiredFeatures" json:"required_features,omitempty"` + OptionalFeatures []string `protobuf:"bytes,5,rep,name=optional_features,json=optionalFeatures" json:"optional_features,omitempty"` + Writingprogram *string `protobuf:"bytes,16,opt,name=writingprogram" json:"writingprogram,omitempty"` + Source *string `protobuf:"bytes,17,opt,name=source" json:"source,omitempty"` // From the bbox field. + // Replication timestamp, expressed in seconds since the epoch, + // otherwise the same value as in the "timestamp=..." field + // in the state.txt file used by Osmosis. + OsmosisReplicationTimestamp *int64 `protobuf:"varint,32,opt,name=osmosis_replication_timestamp,json=osmosisReplicationTimestamp" json:"osmosis_replication_timestamp,omitempty"` + // Replication sequence number (sequenceNumber in state.txt). + OsmosisReplicationSequenceNumber *int64 `protobuf:"varint,33,opt,name=osmosis_replication_sequence_number,json=osmosisReplicationSequenceNumber" json:"osmosis_replication_sequence_number,omitempty"` + // Replication base URL (from Osmosis' configuration.txt file). + OsmosisReplicationBaseUrl *string `protobuf:"bytes,34,opt,name=osmosis_replication_base_url,json=osmosisReplicationBaseUrl" json:"osmosis_replication_base_url,omitempty"` +} + +func (x *HeaderBlock) Reset() { + *x = HeaderBlock{} + if protoimpl.UnsafeEnabled { + mi := &file_osmformat_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *HeaderBlock) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HeaderBlock) ProtoMessage() {} + +func (x *HeaderBlock) ProtoReflect() protoreflect.Message { + mi := &file_osmformat_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HeaderBlock.ProtoReflect.Descriptor instead. +func (*HeaderBlock) Descriptor() ([]byte, []int) { + return file_osmformat_proto_rawDescGZIP(), []int{0} +} + +func (x *HeaderBlock) GetBbox() *HeaderBBox { + if x != nil { + return x.Bbox + } + return nil +} + +func (x *HeaderBlock) GetRequiredFeatures() []string { + if x != nil { + return x.RequiredFeatures + } + return nil +} + +func (x *HeaderBlock) GetOptionalFeatures() []string { + if x != nil { + return x.OptionalFeatures + } + return nil +} + +func (x *HeaderBlock) GetWritingprogram() string { + if x != nil && x.Writingprogram != nil { + return *x.Writingprogram + } + return "" +} + +func (x *HeaderBlock) GetSource() string { + if x != nil && x.Source != nil { + return *x.Source + } + return "" +} + +func (x *HeaderBlock) GetOsmosisReplicationTimestamp() int64 { + if x != nil && x.OsmosisReplicationTimestamp != nil { + return *x.OsmosisReplicationTimestamp + } + return 0 +} + +func (x *HeaderBlock) GetOsmosisReplicationSequenceNumber() int64 { + if x != nil && x.OsmosisReplicationSequenceNumber != nil { + return *x.OsmosisReplicationSequenceNumber + } + return 0 +} + +func (x *HeaderBlock) GetOsmosisReplicationBaseUrl() string { + if x != nil && x.OsmosisReplicationBaseUrl != nil { + return *x.OsmosisReplicationBaseUrl + } + return "" +} + +type HeaderBBox struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Left *int64 `protobuf:"zigzag64,1,req,name=left" json:"left,omitempty"` + Right *int64 `protobuf:"zigzag64,2,req,name=right" json:"right,omitempty"` + Top *int64 `protobuf:"zigzag64,3,req,name=top" json:"top,omitempty"` + Bottom *int64 `protobuf:"zigzag64,4,req,name=bottom" json:"bottom,omitempty"` +} + +func (x *HeaderBBox) Reset() { + *x = HeaderBBox{} + if protoimpl.UnsafeEnabled { + mi := &file_osmformat_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *HeaderBBox) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HeaderBBox) ProtoMessage() {} + +func (x *HeaderBBox) ProtoReflect() protoreflect.Message { + mi := &file_osmformat_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HeaderBBox.ProtoReflect.Descriptor instead. +func (*HeaderBBox) Descriptor() ([]byte, []int) { + return file_osmformat_proto_rawDescGZIP(), []int{1} +} + +func (x *HeaderBBox) GetLeft() int64 { + if x != nil && x.Left != nil { + return *x.Left + } + return 0 +} + +func (x *HeaderBBox) GetRight() int64 { + if x != nil && x.Right != nil { + return *x.Right + } + return 0 +} + +func (x *HeaderBBox) GetTop() int64 { + if x != nil && x.Top != nil { + return *x.Top + } + return 0 +} + +func (x *HeaderBBox) GetBottom() int64 { + if x != nil && x.Bottom != nil { + return *x.Bottom + } + return 0 +} + +type PrimitiveBlock struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Stringtable *StringTable `protobuf:"bytes,1,req,name=stringtable" json:"stringtable,omitempty"` + Primitivegroup []*PrimitiveGroup `protobuf:"bytes,2,rep,name=primitivegroup" json:"primitivegroup,omitempty"` + // Granularity, units of nanodegrees, used to store coordinates in this block. + Granularity *int32 `protobuf:"varint,17,opt,name=granularity,def=100" json:"granularity,omitempty"` + // Offset value between the output coordinates and the granularity grid in units of nanodegrees. + LatOffset *int64 `protobuf:"varint,19,opt,name=lat_offset,json=latOffset,def=0" json:"lat_offset,omitempty"` + LonOffset *int64 `protobuf:"varint,20,opt,name=lon_offset,json=lonOffset,def=0" json:"lon_offset,omitempty"` + // Granularity of dates, normally represented in units of milliseconds since the 1970 epoch. + DateGranularity *int32 `protobuf:"varint,18,opt,name=date_granularity,json=dateGranularity,def=1000" json:"date_granularity,omitempty"` +} + +// Default values for PrimitiveBlock fields. +const ( + Default_PrimitiveBlock_Granularity = int32(100) + Default_PrimitiveBlock_LatOffset = int64(0) + Default_PrimitiveBlock_LonOffset = int64(0) + Default_PrimitiveBlock_DateGranularity = int32(1000) +) + +func (x *PrimitiveBlock) Reset() { + *x = PrimitiveBlock{} + if protoimpl.UnsafeEnabled { + mi := &file_osmformat_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PrimitiveBlock) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PrimitiveBlock) ProtoMessage() {} + +func (x *PrimitiveBlock) ProtoReflect() protoreflect.Message { + mi := &file_osmformat_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PrimitiveBlock.ProtoReflect.Descriptor instead. +func (*PrimitiveBlock) Descriptor() ([]byte, []int) { + return file_osmformat_proto_rawDescGZIP(), []int{2} +} + +func (x *PrimitiveBlock) GetStringtable() *StringTable { + if x != nil { + return x.Stringtable + } + return nil +} + +func (x *PrimitiveBlock) GetPrimitivegroup() []*PrimitiveGroup { + if x != nil { + return x.Primitivegroup + } + return nil +} + +func (x *PrimitiveBlock) GetGranularity() int32 { + if x != nil && x.Granularity != nil { + return *x.Granularity + } + return Default_PrimitiveBlock_Granularity +} + +func (x *PrimitiveBlock) GetLatOffset() int64 { + if x != nil && x.LatOffset != nil { + return *x.LatOffset + } + return Default_PrimitiveBlock_LatOffset +} + +func (x *PrimitiveBlock) GetLonOffset() int64 { + if x != nil && x.LonOffset != nil { + return *x.LonOffset + } + return Default_PrimitiveBlock_LonOffset +} + +func (x *PrimitiveBlock) GetDateGranularity() int32 { + if x != nil && x.DateGranularity != nil { + return *x.DateGranularity + } + return Default_PrimitiveBlock_DateGranularity +} + +// Group of OSMPrimitives. All primitives in a group must be the same type. +type PrimitiveGroup struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Nodes []*Node `protobuf:"bytes,1,rep,name=nodes" json:"nodes,omitempty"` + Dense *DenseNodes `protobuf:"bytes,2,opt,name=dense" json:"dense,omitempty"` + Ways []*Way `protobuf:"bytes,3,rep,name=ways" json:"ways,omitempty"` + Relations []*Relation `protobuf:"bytes,4,rep,name=relations" json:"relations,omitempty"` + Changesets []*ChangeSet `protobuf:"bytes,5,rep,name=changesets" json:"changesets,omitempty"` +} + +func (x *PrimitiveGroup) Reset() { + *x = PrimitiveGroup{} + if protoimpl.UnsafeEnabled { + mi := &file_osmformat_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PrimitiveGroup) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PrimitiveGroup) ProtoMessage() {} + +func (x *PrimitiveGroup) ProtoReflect() protoreflect.Message { + mi := &file_osmformat_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PrimitiveGroup.ProtoReflect.Descriptor instead. +func (*PrimitiveGroup) Descriptor() ([]byte, []int) { + return file_osmformat_proto_rawDescGZIP(), []int{3} +} + +func (x *PrimitiveGroup) GetNodes() []*Node { + if x != nil { + return x.Nodes + } + return nil +} + +func (x *PrimitiveGroup) GetDense() *DenseNodes { + if x != nil { + return x.Dense + } + return nil +} + +func (x *PrimitiveGroup) GetWays() []*Way { + if x != nil { + return x.Ways + } + return nil +} + +func (x *PrimitiveGroup) GetRelations() []*Relation { + if x != nil { + return x.Relations + } + return nil +} + +func (x *PrimitiveGroup) GetChangesets() []*ChangeSet { + if x != nil { + return x.Changesets + } + return nil +} + +//* String table, contains the common strings in each block. +//Note that we reserve index '0' as a delimiter, so the entry at that +//index in the table is ALWAYS blank and unused. +type StringTable struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + S [][]byte `protobuf:"bytes,1,rep,name=s" json:"s,omitempty"` +} + +func (x *StringTable) Reset() { + *x = StringTable{} + if protoimpl.UnsafeEnabled { + mi := &file_osmformat_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StringTable) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StringTable) ProtoMessage() {} + +func (x *StringTable) ProtoReflect() protoreflect.Message { + mi := &file_osmformat_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StringTable.ProtoReflect.Descriptor instead. +func (*StringTable) Descriptor() ([]byte, []int) { + return file_osmformat_proto_rawDescGZIP(), []int{4} +} + +func (x *StringTable) GetS() [][]byte { + if x != nil { + return x.S + } + return nil +} + +// Optional metadata that may be included into each primitive. +type Info struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Version *int32 `protobuf:"varint,1,opt,name=version,def=-1" json:"version,omitempty"` + Timestamp *int64 `protobuf:"varint,2,opt,name=timestamp" json:"timestamp,omitempty"` + Changeset *int64 `protobuf:"varint,3,opt,name=changeset" json:"changeset,omitempty"` + Uid *int32 `protobuf:"varint,4,opt,name=uid" json:"uid,omitempty"` + UserSid *uint32 `protobuf:"varint,5,opt,name=user_sid,json=userSid" json:"user_sid,omitempty"` // String IDs + // The visible flag is used to store history information. It indicates that + // the current object version has been created by a delete operation on the + // OSM API. + // When a writer sets this flag, it MUST add a required_features tag with + // value "HistoricalInformation" to the HeaderBlock. + // If this flag is not available for some object it MUST be assumed to be + // true if the file has the required_features tag "HistoricalInformation" + // set. + Visible *bool `protobuf:"varint,6,opt,name=visible" json:"visible,omitempty"` +} + +// Default values for Info fields. +const ( + Default_Info_Version = int32(-1) +) + +func (x *Info) Reset() { + *x = Info{} + if protoimpl.UnsafeEnabled { + mi := &file_osmformat_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Info) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Info) ProtoMessage() {} + +func (x *Info) ProtoReflect() protoreflect.Message { + mi := &file_osmformat_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Info.ProtoReflect.Descriptor instead. +func (*Info) Descriptor() ([]byte, []int) { + return file_osmformat_proto_rawDescGZIP(), []int{5} +} + +func (x *Info) GetVersion() int32 { + if x != nil && x.Version != nil { + return *x.Version + } + return Default_Info_Version +} + +func (x *Info) GetTimestamp() int64 { + if x != nil && x.Timestamp != nil { + return *x.Timestamp + } + return 0 +} + +func (x *Info) GetChangeset() int64 { + if x != nil && x.Changeset != nil { + return *x.Changeset + } + return 0 +} + +func (x *Info) GetUid() int32 { + if x != nil && x.Uid != nil { + return *x.Uid + } + return 0 +} + +func (x *Info) GetUserSid() uint32 { + if x != nil && x.UserSid != nil { + return *x.UserSid + } + return 0 +} + +func (x *Info) GetVisible() bool { + if x != nil && x.Visible != nil { + return *x.Visible + } + return false +} + +//* Optional metadata that may be included into each primitive. Special dense format used in DenseNodes. +type DenseInfo struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Version []int32 `protobuf:"varint,1,rep,packed,name=version" json:"version,omitempty"` + Timestamp []int64 `protobuf:"zigzag64,2,rep,packed,name=timestamp" json:"timestamp,omitempty"` // DELTA coded + Changeset []int64 `protobuf:"zigzag64,3,rep,packed,name=changeset" json:"changeset,omitempty"` // DELTA coded + Uid []int32 `protobuf:"zigzag32,4,rep,packed,name=uid" json:"uid,omitempty"` // DELTA coded + UserSid []int32 `protobuf:"zigzag32,5,rep,packed,name=user_sid,json=userSid" json:"user_sid,omitempty"` // String IDs for usernames. DELTA coded + // The visible flag is used to store history information. It indicates that + // the current object version has been created by a delete operation on the + // OSM API. + // When a writer sets this flag, it MUST add a required_features tag with + // value "HistoricalInformation" to the HeaderBlock. + // If this flag is not available for some object it MUST be assumed to be + // true if the file has the required_features tag "HistoricalInformation" + // set. + Visible []bool `protobuf:"varint,6,rep,packed,name=visible" json:"visible,omitempty"` +} + +func (x *DenseInfo) Reset() { + *x = DenseInfo{} + if protoimpl.UnsafeEnabled { + mi := &file_osmformat_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DenseInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DenseInfo) ProtoMessage() {} + +func (x *DenseInfo) ProtoReflect() protoreflect.Message { + mi := &file_osmformat_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DenseInfo.ProtoReflect.Descriptor instead. +func (*DenseInfo) Descriptor() ([]byte, []int) { + return file_osmformat_proto_rawDescGZIP(), []int{6} +} + +func (x *DenseInfo) GetVersion() []int32 { + if x != nil { + return x.Version + } + return nil +} + +func (x *DenseInfo) GetTimestamp() []int64 { + if x != nil { + return x.Timestamp + } + return nil +} + +func (x *DenseInfo) GetChangeset() []int64 { + if x != nil { + return x.Changeset + } + return nil +} + +func (x *DenseInfo) GetUid() []int32 { + if x != nil { + return x.Uid + } + return nil +} + +func (x *DenseInfo) GetUserSid() []int32 { + if x != nil { + return x.UserSid + } + return nil +} + +func (x *DenseInfo) GetVisible() []bool { + if x != nil { + return x.Visible + } + return nil +} + +// This is kept for backwards compatibility but not used anywhere. +type ChangeSet struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id *int64 `protobuf:"varint,1,req,name=id" json:"id,omitempty"` +} + +func (x *ChangeSet) Reset() { + *x = ChangeSet{} + if protoimpl.UnsafeEnabled { + mi := &file_osmformat_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ChangeSet) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ChangeSet) ProtoMessage() {} + +func (x *ChangeSet) ProtoReflect() protoreflect.Message { + mi := &file_osmformat_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ChangeSet.ProtoReflect.Descriptor instead. +func (*ChangeSet) Descriptor() ([]byte, []int) { + return file_osmformat_proto_rawDescGZIP(), []int{7} +} + +func (x *ChangeSet) GetId() int64 { + if x != nil && x.Id != nil { + return *x.Id + } + return 0 +} + +type Node struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id *int64 `protobuf:"zigzag64,1,req,name=id" json:"id,omitempty"` + // Parallel arrays. + Keys []uint32 `protobuf:"varint,2,rep,packed,name=keys" json:"keys,omitempty"` // String IDs. + Vals []uint32 `protobuf:"varint,3,rep,packed,name=vals" json:"vals,omitempty"` // String IDs. + Info *Info `protobuf:"bytes,4,opt,name=info" json:"info,omitempty"` // May be omitted in omitmeta + Lat *int64 `protobuf:"zigzag64,8,req,name=lat" json:"lat,omitempty"` + Lon *int64 `protobuf:"zigzag64,9,req,name=lon" json:"lon,omitempty"` +} + +func (x *Node) Reset() { + *x = Node{} + if protoimpl.UnsafeEnabled { + mi := &file_osmformat_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Node) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Node) ProtoMessage() {} + +func (x *Node) ProtoReflect() protoreflect.Message { + mi := &file_osmformat_proto_msgTypes[8] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Node.ProtoReflect.Descriptor instead. +func (*Node) Descriptor() ([]byte, []int) { + return file_osmformat_proto_rawDescGZIP(), []int{8} +} + +func (x *Node) GetId() int64 { + if x != nil && x.Id != nil { + return *x.Id + } + return 0 +} + +func (x *Node) GetKeys() []uint32 { + if x != nil { + return x.Keys + } + return nil +} + +func (x *Node) GetVals() []uint32 { + if x != nil { + return x.Vals + } + return nil +} + +func (x *Node) GetInfo() *Info { + if x != nil { + return x.Info + } + return nil +} + +func (x *Node) GetLat() int64 { + if x != nil && x.Lat != nil { + return *x.Lat + } + return 0 +} + +func (x *Node) GetLon() int64 { + if x != nil && x.Lon != nil { + return *x.Lon + } + return 0 +} + +type DenseNodes struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id []int64 `protobuf:"zigzag64,1,rep,packed,name=id" json:"id,omitempty"` // DELTA coded + Denseinfo *DenseInfo `protobuf:"bytes,5,opt,name=denseinfo" json:"denseinfo,omitempty"` + Lat []int64 `protobuf:"zigzag64,8,rep,packed,name=lat" json:"lat,omitempty"` // DELTA coded + Lon []int64 `protobuf:"zigzag64,9,rep,packed,name=lon" json:"lon,omitempty"` // DELTA coded + // Special packing of keys and vals into one array. May be empty if all nodes in this block are tagless. + KeysVals []int32 `protobuf:"varint,10,rep,packed,name=keys_vals,json=keysVals" json:"keys_vals,omitempty"` +} + +func (x *DenseNodes) Reset() { + *x = DenseNodes{} + if protoimpl.UnsafeEnabled { + mi := &file_osmformat_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DenseNodes) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DenseNodes) ProtoMessage() {} + +func (x *DenseNodes) ProtoReflect() protoreflect.Message { + mi := &file_osmformat_proto_msgTypes[9] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DenseNodes.ProtoReflect.Descriptor instead. +func (*DenseNodes) Descriptor() ([]byte, []int) { + return file_osmformat_proto_rawDescGZIP(), []int{9} +} + +func (x *DenseNodes) GetId() []int64 { + if x != nil { + return x.Id + } + return nil +} + +func (x *DenseNodes) GetDenseinfo() *DenseInfo { + if x != nil { + return x.Denseinfo + } + return nil +} + +func (x *DenseNodes) GetLat() []int64 { + if x != nil { + return x.Lat + } + return nil +} + +func (x *DenseNodes) GetLon() []int64 { + if x != nil { + return x.Lon + } + return nil +} + +func (x *DenseNodes) GetKeysVals() []int32 { + if x != nil { + return x.KeysVals + } + return nil +} + +type Way struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id *int64 `protobuf:"varint,1,req,name=id" json:"id,omitempty"` + // Parallel arrays. + Keys []uint32 `protobuf:"varint,2,rep,packed,name=keys" json:"keys,omitempty"` + Vals []uint32 `protobuf:"varint,3,rep,packed,name=vals" json:"vals,omitempty"` + Info *Info `protobuf:"bytes,4,opt,name=info" json:"info,omitempty"` + Refs []int64 `protobuf:"zigzag64,8,rep,packed,name=refs" json:"refs,omitempty"` // DELTA coded + // The following two fields are optional. They are only used in a special + // format where node locations are also added to the ways. This makes the + // files larger, but allows creating way geometries directly. + // + // If this is used, you MUST set the optional_features tag "LocationsOnWays" + // and the number of values in refs, lat, and lon MUST be the same. + Lat []int64 `protobuf:"zigzag64,9,rep,packed,name=lat" json:"lat,omitempty"` // DELTA coded, optional + Lon []int64 `protobuf:"zigzag64,10,rep,packed,name=lon" json:"lon,omitempty"` // DELTA coded, optional +} + +func (x *Way) Reset() { + *x = Way{} + if protoimpl.UnsafeEnabled { + mi := &file_osmformat_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Way) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Way) ProtoMessage() {} + +func (x *Way) ProtoReflect() protoreflect.Message { + mi := &file_osmformat_proto_msgTypes[10] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Way.ProtoReflect.Descriptor instead. +func (*Way) Descriptor() ([]byte, []int) { + return file_osmformat_proto_rawDescGZIP(), []int{10} +} + +func (x *Way) GetId() int64 { + if x != nil && x.Id != nil { + return *x.Id + } + return 0 +} + +func (x *Way) GetKeys() []uint32 { + if x != nil { + return x.Keys + } + return nil +} + +func (x *Way) GetVals() []uint32 { + if x != nil { + return x.Vals + } + return nil +} + +func (x *Way) GetInfo() *Info { + if x != nil { + return x.Info + } + return nil +} + +func (x *Way) GetRefs() []int64 { + if x != nil { + return x.Refs + } + return nil +} + +func (x *Way) GetLat() []int64 { + if x != nil { + return x.Lat + } + return nil +} + +func (x *Way) GetLon() []int64 { + if x != nil { + return x.Lon + } + return nil +} + +type Relation struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id *int64 `protobuf:"varint,1,req,name=id" json:"id,omitempty"` + // Parallel arrays. + Keys []uint32 `protobuf:"varint,2,rep,packed,name=keys" json:"keys,omitempty"` + Vals []uint32 `protobuf:"varint,3,rep,packed,name=vals" json:"vals,omitempty"` + Info *Info `protobuf:"bytes,4,opt,name=info" json:"info,omitempty"` + // Parallel arrays + RolesSid []int32 `protobuf:"varint,8,rep,packed,name=roles_sid,json=rolesSid" json:"roles_sid,omitempty"` // This should have been defined as uint32 for consistency, but it is now too late to change it + Memids []int64 `protobuf:"zigzag64,9,rep,packed,name=memids" json:"memids,omitempty"` // DELTA encoded + Types []Relation_MemberType `protobuf:"varint,10,rep,packed,name=types,enum=pbf.Relation_MemberType" json:"types,omitempty"` +} + +func (x *Relation) Reset() { + *x = Relation{} + if protoimpl.UnsafeEnabled { + mi := &file_osmformat_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Relation) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Relation) ProtoMessage() {} + +func (x *Relation) ProtoReflect() protoreflect.Message { + mi := &file_osmformat_proto_msgTypes[11] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Relation.ProtoReflect.Descriptor instead. +func (*Relation) Descriptor() ([]byte, []int) { + return file_osmformat_proto_rawDescGZIP(), []int{11} +} + +func (x *Relation) GetId() int64 { + if x != nil && x.Id != nil { + return *x.Id + } + return 0 +} + +func (x *Relation) GetKeys() []uint32 { + if x != nil { + return x.Keys + } + return nil +} + +func (x *Relation) GetVals() []uint32 { + if x != nil { + return x.Vals + } + return nil +} + +func (x *Relation) GetInfo() *Info { + if x != nil { + return x.Info + } + return nil +} + +func (x *Relation) GetRolesSid() []int32 { + if x != nil { + return x.RolesSid + } + return nil +} + +func (x *Relation) GetMemids() []int64 { + if x != nil { + return x.Memids + } + return nil +} + +func (x *Relation) GetTypes() []Relation_MemberType { + if x != nil { + return x.Types + } + return nil +} + +var File_osmformat_proto protoreflect.FileDescriptor + +var file_osmformat_proto_rawDesc = []byte{ + 0x0a, 0x0f, 0x6f, 0x73, 0x6d, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x12, 0x03, 0x70, 0x62, 0x66, 0x22, 0xa0, 0x03, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x64, 0x65, + 0x72, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x12, 0x23, 0x0a, 0x04, 0x62, 0x62, 0x6f, 0x78, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x70, 0x62, 0x66, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, + 0x72, 0x42, 0x42, 0x6f, 0x78, 0x52, 0x04, 0x62, 0x62, 0x6f, 0x78, 0x12, 0x2b, 0x0a, 0x11, 0x72, + 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x5f, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, + 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x10, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, + 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x2b, 0x0a, 0x11, 0x6f, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x61, 0x6c, 0x5f, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x05, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x10, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x46, 0x65, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x26, 0x0a, 0x0e, 0x77, 0x72, 0x69, 0x74, 0x69, 0x6e, 0x67, + 0x70, 0x72, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x18, 0x10, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x77, + 0x72, 0x69, 0x74, 0x69, 0x6e, 0x67, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x12, 0x16, 0x0a, + 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x11, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x42, 0x0a, 0x1d, 0x6f, 0x73, 0x6d, 0x6f, 0x73, 0x69, 0x73, + 0x5f, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x20, 0x20, 0x01, 0x28, 0x03, 0x52, 0x1b, 0x6f, 0x73, + 0x6d, 0x6f, 0x73, 0x69, 0x73, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x4d, 0x0a, 0x23, 0x6f, 0x73, 0x6d, + 0x6f, 0x73, 0x69, 0x73, 0x5f, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x5f, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, + 0x18, 0x21, 0x20, 0x01, 0x28, 0x03, 0x52, 0x20, 0x6f, 0x73, 0x6d, 0x6f, 0x73, 0x69, 0x73, 0x52, + 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x65, 0x71, 0x75, 0x65, 0x6e, + 0x63, 0x65, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x3f, 0x0a, 0x1c, 0x6f, 0x73, 0x6d, 0x6f, + 0x73, 0x69, 0x73, 0x5f, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, + 0x62, 0x61, 0x73, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x22, 0x20, 0x01, 0x28, 0x09, 0x52, 0x19, + 0x6f, 0x73, 0x6d, 0x6f, 0x73, 0x69, 0x73, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x42, 0x61, 0x73, 0x65, 0x55, 0x72, 0x6c, 0x22, 0x60, 0x0a, 0x0a, 0x48, 0x65, 0x61, + 0x64, 0x65, 0x72, 0x42, 0x42, 0x6f, 0x78, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x65, 0x66, 0x74, 0x18, + 0x01, 0x20, 0x02, 0x28, 0x12, 0x52, 0x04, 0x6c, 0x65, 0x66, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x72, + 0x69, 0x67, 0x68, 0x74, 0x18, 0x02, 0x20, 0x02, 0x28, 0x12, 0x52, 0x05, 0x72, 0x69, 0x67, 0x68, + 0x74, 0x12, 0x10, 0x0a, 0x03, 0x74, 0x6f, 0x70, 0x18, 0x03, 0x20, 0x02, 0x28, 0x12, 0x52, 0x03, + 0x74, 0x6f, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x6f, 0x74, 0x74, 0x6f, 0x6d, 0x18, 0x04, 0x20, + 0x02, 0x28, 0x12, 0x52, 0x06, 0x62, 0x6f, 0x74, 0x74, 0x6f, 0x6d, 0x22, 0x9d, 0x02, 0x0a, 0x0e, + 0x50, 0x72, 0x69, 0x6d, 0x69, 0x74, 0x69, 0x76, 0x65, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x12, 0x32, + 0x0a, 0x0b, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x01, 0x20, + 0x02, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x70, 0x62, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, + 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x0b, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x74, 0x61, 0x62, + 0x6c, 0x65, 0x12, 0x3b, 0x0a, 0x0e, 0x70, 0x72, 0x69, 0x6d, 0x69, 0x74, 0x69, 0x76, 0x65, 0x67, + 0x72, 0x6f, 0x75, 0x70, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x70, 0x62, 0x66, + 0x2e, 0x50, 0x72, 0x69, 0x6d, 0x69, 0x74, 0x69, 0x76, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x52, + 0x0e, 0x70, 0x72, 0x69, 0x6d, 0x69, 0x74, 0x69, 0x76, 0x65, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x12, + 0x25, 0x0a, 0x0b, 0x67, 0x72, 0x61, 0x6e, 0x75, 0x6c, 0x61, 0x72, 0x69, 0x74, 0x79, 0x18, 0x11, + 0x20, 0x01, 0x28, 0x05, 0x3a, 0x03, 0x31, 0x30, 0x30, 0x52, 0x0b, 0x67, 0x72, 0x61, 0x6e, 0x75, + 0x6c, 0x61, 0x72, 0x69, 0x74, 0x79, 0x12, 0x20, 0x0a, 0x0a, 0x6c, 0x61, 0x74, 0x5f, 0x6f, 0x66, + 0x66, 0x73, 0x65, 0x74, 0x18, 0x13, 0x20, 0x01, 0x28, 0x03, 0x3a, 0x01, 0x30, 0x52, 0x09, 0x6c, + 0x61, 0x74, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x20, 0x0a, 0x0a, 0x6c, 0x6f, 0x6e, 0x5f, + 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x14, 0x20, 0x01, 0x28, 0x03, 0x3a, 0x01, 0x30, 0x52, + 0x09, 0x6c, 0x6f, 0x6e, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x2f, 0x0a, 0x10, 0x64, 0x61, + 0x74, 0x65, 0x5f, 0x67, 0x72, 0x61, 0x6e, 0x75, 0x6c, 0x61, 0x72, 0x69, 0x74, 0x79, 0x18, 0x12, + 0x20, 0x01, 0x28, 0x05, 0x3a, 0x04, 0x31, 0x30, 0x30, 0x30, 0x52, 0x0f, 0x64, 0x61, 0x74, 0x65, + 0x47, 0x72, 0x61, 0x6e, 0x75, 0x6c, 0x61, 0x72, 0x69, 0x74, 0x79, 0x22, 0xd3, 0x01, 0x0a, 0x0e, + 0x50, 0x72, 0x69, 0x6d, 0x69, 0x74, 0x69, 0x76, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x1f, + 0x0a, 0x05, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x09, 0x2e, + 0x70, 0x62, 0x66, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x05, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x12, + 0x25, 0x0a, 0x05, 0x64, 0x65, 0x6e, 0x73, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, + 0x2e, 0x70, 0x62, 0x66, 0x2e, 0x44, 0x65, 0x6e, 0x73, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, + 0x05, 0x64, 0x65, 0x6e, 0x73, 0x65, 0x12, 0x1c, 0x0a, 0x04, 0x77, 0x61, 0x79, 0x73, 0x18, 0x03, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x08, 0x2e, 0x70, 0x62, 0x66, 0x2e, 0x57, 0x61, 0x79, 0x52, 0x04, + 0x77, 0x61, 0x79, 0x73, 0x12, 0x2b, 0x0a, 0x09, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x70, 0x62, 0x66, 0x2e, 0x52, 0x65, + 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x12, 0x2e, 0x0a, 0x0a, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x65, 0x74, 0x73, 0x18, + 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x62, 0x66, 0x2e, 0x43, 0x68, 0x61, 0x6e, + 0x67, 0x65, 0x53, 0x65, 0x74, 0x52, 0x0a, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x65, 0x74, + 0x73, 0x22, 0x1b, 0x0a, 0x0b, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x54, 0x61, 0x62, 0x6c, 0x65, + 0x12, 0x0c, 0x0a, 0x01, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x01, 0x73, 0x22, 0xa7, + 0x01, 0x0a, 0x04, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x1c, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x3a, 0x02, 0x2d, 0x31, 0x52, 0x07, 0x76, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x65, 0x74, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x65, + 0x74, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, + 0x75, 0x69, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x73, 0x69, 0x64, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x75, 0x73, 0x65, 0x72, 0x53, 0x69, 0x64, 0x12, 0x18, + 0x0a, 0x07, 0x76, 0x69, 0x73, 0x69, 0x62, 0x6c, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x07, 0x76, 0x69, 0x73, 0x69, 0x62, 0x6c, 0x65, 0x22, 0xc0, 0x01, 0x0a, 0x09, 0x44, 0x65, 0x6e, + 0x73, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x1c, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x18, 0x01, 0x20, 0x03, 0x28, 0x05, 0x42, 0x02, 0x10, 0x01, 0x52, 0x07, 0x76, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0x18, 0x02, 0x20, 0x03, 0x28, 0x12, 0x42, 0x02, 0x10, 0x01, 0x52, 0x09, 0x74, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x20, 0x0a, 0x09, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, + 0x73, 0x65, 0x74, 0x18, 0x03, 0x20, 0x03, 0x28, 0x12, 0x42, 0x02, 0x10, 0x01, 0x52, 0x09, 0x63, + 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x65, 0x74, 0x12, 0x14, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, + 0x04, 0x20, 0x03, 0x28, 0x11, 0x42, 0x02, 0x10, 0x01, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x1d, + 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x73, 0x69, 0x64, 0x18, 0x05, 0x20, 0x03, 0x28, 0x11, + 0x42, 0x02, 0x10, 0x01, 0x52, 0x07, 0x75, 0x73, 0x65, 0x72, 0x53, 0x69, 0x64, 0x12, 0x1c, 0x0a, + 0x07, 0x76, 0x69, 0x73, 0x69, 0x62, 0x6c, 0x65, 0x18, 0x06, 0x20, 0x03, 0x28, 0x08, 0x42, 0x02, + 0x10, 0x01, 0x52, 0x07, 0x76, 0x69, 0x73, 0x69, 0x62, 0x6c, 0x65, 0x22, 0x1b, 0x0a, 0x09, 0x43, + 0x68, 0x61, 0x6e, 0x67, 0x65, 0x53, 0x65, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x02, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x22, 0x89, 0x01, 0x0a, 0x04, 0x4e, 0x6f, 0x64, + 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x02, 0x28, 0x12, 0x52, 0x02, 0x69, + 0x64, 0x12, 0x16, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0d, 0x42, + 0x02, 0x10, 0x01, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x16, 0x0a, 0x04, 0x76, 0x61, 0x6c, + 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0d, 0x42, 0x02, 0x10, 0x01, 0x52, 0x04, 0x76, 0x61, 0x6c, + 0x73, 0x12, 0x1d, 0x0a, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x09, 0x2e, 0x70, 0x62, 0x66, 0x2e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, + 0x12, 0x10, 0x0a, 0x03, 0x6c, 0x61, 0x74, 0x18, 0x08, 0x20, 0x02, 0x28, 0x12, 0x52, 0x03, 0x6c, + 0x61, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6c, 0x6f, 0x6e, 0x18, 0x09, 0x20, 0x02, 0x28, 0x12, 0x52, + 0x03, 0x6c, 0x6f, 0x6e, 0x22, 0x9b, 0x01, 0x0a, 0x0a, 0x44, 0x65, 0x6e, 0x73, 0x65, 0x4e, 0x6f, + 0x64, 0x65, 0x73, 0x12, 0x12, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x03, 0x28, 0x12, 0x42, + 0x02, 0x10, 0x01, 0x52, 0x02, 0x69, 0x64, 0x12, 0x2c, 0x0a, 0x09, 0x64, 0x65, 0x6e, 0x73, 0x65, + 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x62, 0x66, + 0x2e, 0x44, 0x65, 0x6e, 0x73, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x09, 0x64, 0x65, 0x6e, 0x73, + 0x65, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x14, 0x0a, 0x03, 0x6c, 0x61, 0x74, 0x18, 0x08, 0x20, 0x03, + 0x28, 0x12, 0x42, 0x02, 0x10, 0x01, 0x52, 0x03, 0x6c, 0x61, 0x74, 0x12, 0x14, 0x0a, 0x03, 0x6c, + 0x6f, 0x6e, 0x18, 0x09, 0x20, 0x03, 0x28, 0x12, 0x42, 0x02, 0x10, 0x01, 0x52, 0x03, 0x6c, 0x6f, + 0x6e, 0x12, 0x1f, 0x0a, 0x09, 0x6b, 0x65, 0x79, 0x73, 0x5f, 0x76, 0x61, 0x6c, 0x73, 0x18, 0x0a, + 0x20, 0x03, 0x28, 0x05, 0x42, 0x02, 0x10, 0x01, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x56, 0x61, + 0x6c, 0x73, 0x22, 0xa8, 0x01, 0x0a, 0x03, 0x57, 0x61, 0x79, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x02, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, 0x16, 0x0a, 0x04, 0x6b, 0x65, + 0x79, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0d, 0x42, 0x02, 0x10, 0x01, 0x52, 0x04, 0x6b, 0x65, + 0x79, 0x73, 0x12, 0x16, 0x0a, 0x04, 0x76, 0x61, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0d, + 0x42, 0x02, 0x10, 0x01, 0x52, 0x04, 0x76, 0x61, 0x6c, 0x73, 0x12, 0x1d, 0x0a, 0x04, 0x69, 0x6e, + 0x66, 0x6f, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x09, 0x2e, 0x70, 0x62, 0x66, 0x2e, 0x49, + 0x6e, 0x66, 0x6f, 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x16, 0x0a, 0x04, 0x72, 0x65, 0x66, + 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x12, 0x42, 0x02, 0x10, 0x01, 0x52, 0x04, 0x72, 0x65, 0x66, + 0x73, 0x12, 0x14, 0x0a, 0x03, 0x6c, 0x61, 0x74, 0x18, 0x09, 0x20, 0x03, 0x28, 0x12, 0x42, 0x02, + 0x10, 0x01, 0x52, 0x03, 0x6c, 0x61, 0x74, 0x12, 0x14, 0x0a, 0x03, 0x6c, 0x6f, 0x6e, 0x18, 0x0a, + 0x20, 0x03, 0x28, 0x12, 0x42, 0x02, 0x10, 0x01, 0x52, 0x03, 0x6c, 0x6f, 0x6e, 0x22, 0x89, 0x02, + 0x0a, 0x08, 0x52, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x02, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, 0x16, 0x0a, 0x04, 0x6b, 0x65, + 0x79, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0d, 0x42, 0x02, 0x10, 0x01, 0x52, 0x04, 0x6b, 0x65, + 0x79, 0x73, 0x12, 0x16, 0x0a, 0x04, 0x76, 0x61, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0d, + 0x42, 0x02, 0x10, 0x01, 0x52, 0x04, 0x76, 0x61, 0x6c, 0x73, 0x12, 0x1d, 0x0a, 0x04, 0x69, 0x6e, + 0x66, 0x6f, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x09, 0x2e, 0x70, 0x62, 0x66, 0x2e, 0x49, + 0x6e, 0x66, 0x6f, 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x1f, 0x0a, 0x09, 0x72, 0x6f, 0x6c, + 0x65, 0x73, 0x5f, 0x73, 0x69, 0x64, 0x18, 0x08, 0x20, 0x03, 0x28, 0x05, 0x42, 0x02, 0x10, 0x01, + 0x52, 0x08, 0x72, 0x6f, 0x6c, 0x65, 0x73, 0x53, 0x69, 0x64, 0x12, 0x1a, 0x0a, 0x06, 0x6d, 0x65, + 0x6d, 0x69, 0x64, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x12, 0x42, 0x02, 0x10, 0x01, 0x52, 0x06, + 0x6d, 0x65, 0x6d, 0x69, 0x64, 0x73, 0x12, 0x32, 0x0a, 0x05, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, + 0x0a, 0x20, 0x03, 0x28, 0x0e, 0x32, 0x18, 0x2e, 0x70, 0x62, 0x66, 0x2e, 0x52, 0x65, 0x6c, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x4d, 0x65, 0x6d, 0x62, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x42, + 0x02, 0x10, 0x01, 0x52, 0x05, 0x74, 0x79, 0x70, 0x65, 0x73, 0x22, 0x2d, 0x0a, 0x0a, 0x4d, 0x65, + 0x6d, 0x62, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x08, 0x0a, 0x04, 0x4e, 0x4f, 0x44, 0x45, + 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x57, 0x41, 0x59, 0x10, 0x01, 0x12, 0x0c, 0x0a, 0x08, 0x52, + 0x45, 0x4c, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x02, +} + +var ( + file_osmformat_proto_rawDescOnce sync.Once + file_osmformat_proto_rawDescData = file_osmformat_proto_rawDesc +) + +func file_osmformat_proto_rawDescGZIP() []byte { + file_osmformat_proto_rawDescOnce.Do(func() { + file_osmformat_proto_rawDescData = protoimpl.X.CompressGZIP(file_osmformat_proto_rawDescData) + }) + return file_osmformat_proto_rawDescData +} + +var file_osmformat_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_osmformat_proto_msgTypes = make([]protoimpl.MessageInfo, 12) +var file_osmformat_proto_goTypes = []interface{}{ + (Relation_MemberType)(0), // 0: pbf.Relation.MemberType + (*HeaderBlock)(nil), // 1: pbf.HeaderBlock + (*HeaderBBox)(nil), // 2: pbf.HeaderBBox + (*PrimitiveBlock)(nil), // 3: pbf.PrimitiveBlock + (*PrimitiveGroup)(nil), // 4: pbf.PrimitiveGroup + (*StringTable)(nil), // 5: pbf.StringTable + (*Info)(nil), // 6: pbf.Info + (*DenseInfo)(nil), // 7: pbf.DenseInfo + (*ChangeSet)(nil), // 8: pbf.ChangeSet + (*Node)(nil), // 9: pbf.Node + (*DenseNodes)(nil), // 10: pbf.DenseNodes + (*Way)(nil), // 11: pbf.Way + (*Relation)(nil), // 12: pbf.Relation +} +var file_osmformat_proto_depIdxs = []int32{ + 2, // 0: pbf.HeaderBlock.bbox:type_name -> pbf.HeaderBBox + 5, // 1: pbf.PrimitiveBlock.stringtable:type_name -> pbf.StringTable + 4, // 2: pbf.PrimitiveBlock.primitivegroup:type_name -> pbf.PrimitiveGroup + 9, // 3: pbf.PrimitiveGroup.nodes:type_name -> pbf.Node + 10, // 4: pbf.PrimitiveGroup.dense:type_name -> pbf.DenseNodes + 11, // 5: pbf.PrimitiveGroup.ways:type_name -> pbf.Way + 12, // 6: pbf.PrimitiveGroup.relations:type_name -> pbf.Relation + 8, // 7: pbf.PrimitiveGroup.changesets:type_name -> pbf.ChangeSet + 6, // 8: pbf.Node.info:type_name -> pbf.Info + 7, // 9: pbf.DenseNodes.denseinfo:type_name -> pbf.DenseInfo + 6, // 10: pbf.Way.info:type_name -> pbf.Info + 6, // 11: pbf.Relation.info:type_name -> pbf.Info + 0, // 12: pbf.Relation.types:type_name -> pbf.Relation.MemberType + 13, // [13:13] is the sub-list for method output_type + 13, // [13:13] is the sub-list for method input_type + 13, // [13:13] is the sub-list for extension type_name + 13, // [13:13] is the sub-list for extension extendee + 0, // [0:13] is the sub-list for field type_name +} + +func init() { file_osmformat_proto_init() } +func file_osmformat_proto_init() { + if File_osmformat_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_osmformat_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HeaderBlock); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_osmformat_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HeaderBBox); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_osmformat_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PrimitiveBlock); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_osmformat_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PrimitiveGroup); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_osmformat_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StringTable); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_osmformat_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Info); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_osmformat_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DenseInfo); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_osmformat_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ChangeSet); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_osmformat_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Node); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_osmformat_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DenseNodes); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_osmformat_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Way); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_osmformat_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Relation); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_osmformat_proto_rawDesc, + NumEnums: 1, + NumMessages: 12, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_osmformat_proto_goTypes, + DependencyIndexes: file_osmformat_proto_depIdxs, + EnumInfos: file_osmformat_proto_enumTypes, + MessageInfos: file_osmformat_proto_msgTypes, + }.Build() + File_osmformat_proto = out.File + file_osmformat_proto_rawDesc = nil + file_osmformat_proto_goTypes = nil + file_osmformat_proto_depIdxs = nil +} diff --git a/core/commands/pbf/osmformat.proto b/core/commands/pbf/osmformat.proto new file mode 100644 index 0000000..7f35ba7 --- /dev/null +++ b/core/commands/pbf/osmformat.proto @@ -0,0 +1,228 @@ +syntax = "proto2"; + +package pbf; + +/* OSM Binary file format +This is the master schema file of the OSM binary file format. This +file is designed to support limited random-access and future +extendability. +A binary OSM file consists of a sequence of FileBlocks (please see +fileformat.proto). The first fileblock contains a serialized instance +of HeaderBlock, followed by a sequence of PrimitiveBlock blocks that +contain the primitives. +Each primitiveblock is designed to be independently parsable. It +contains a string table storing all strings in that block (keys and +values in tags, roles in relations, usernames, etc.) as well as +metadata containing the precision of coordinates or timestamps in that +block. +A primitiveblock contains a sequence of primitive groups, each +containing primitives of the same type (nodes, densenodes, ways, +relations). Coordinates are stored in signed 64-bit integers. Lat&lon +are measured in units nanodegrees. The default of +granularity of 100 nanodegrees corresponds to about 1cm on the ground, +and a full lat or lon fits into 32 bits. +Converting an integer to a latitude or longitude uses the formula: +$OUT = IN * granularity / 10**9$. Many encoding schemes use delta +coding when representing nodes and relations. +*/ + +////////////////////////////////////////////////////////////////////////// +////////////////////////////////////////////////////////////////////////// + +/* Contains the file header. */ + +message HeaderBlock { + optional HeaderBBox bbox = 1; + + /* Additional tags to aid in parsing this dataset */ + repeated string required_features = 4; + repeated string optional_features = 5; + + optional string writingprogram = 16; + optional string source = 17; // From the bbox field. + + /* Tags that allow continuing an Osmosis replication */ + + // Replication timestamp, expressed in seconds since the epoch, + // otherwise the same value as in the "timestamp=..." field + // in the state.txt file used by Osmosis. + optional int64 osmosis_replication_timestamp = 32; + + // Replication sequence number (sequenceNumber in state.txt). + optional int64 osmosis_replication_sequence_number = 33; + + // Replication base URL (from Osmosis' configuration.txt file). + optional string osmosis_replication_base_url = 34; +} + + +/** The bounding box field in the OSM header. BBOX, as used in the OSM +header. Units are always in nanodegrees -- they do not obey +granularity rules. */ + +message HeaderBBox { + required sint64 left = 1; + required sint64 right = 2; + required sint64 top = 3; + required sint64 bottom = 4; +} + + +/////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////// + + +message PrimitiveBlock { + required StringTable stringtable = 1; + repeated PrimitiveGroup primitivegroup = 2; + + // Granularity, units of nanodegrees, used to store coordinates in this block. + optional int32 granularity = 17 [default=100]; + + // Offset value between the output coordinates and the granularity grid in units of nanodegrees. + optional int64 lat_offset = 19 [default=0]; + optional int64 lon_offset = 20 [default=0]; + + // Granularity of dates, normally represented in units of milliseconds since the 1970 epoch. + optional int32 date_granularity = 18 [default=1000]; +} + +// Group of OSMPrimitives. All primitives in a group must be the same type. +message PrimitiveGroup { + repeated Node nodes = 1; + optional DenseNodes dense = 2; + repeated Way ways = 3; + repeated Relation relations = 4; + repeated ChangeSet changesets = 5; +} + + +/** String table, contains the common strings in each block. + Note that we reserve index '0' as a delimiter, so the entry at that + index in the table is ALWAYS blank and unused. + */ +message StringTable { + repeated bytes s = 1; +} + +/* Optional metadata that may be included into each primitive. */ +message Info { + optional int32 version = 1 [default = -1]; + optional int64 timestamp = 2; + optional int64 changeset = 3; + optional int32 uid = 4; + optional uint32 user_sid = 5; // String IDs + + // The visible flag is used to store history information. It indicates that + // the current object version has been created by a delete operation on the + // OSM API. + // When a writer sets this flag, it MUST add a required_features tag with + // value "HistoricalInformation" to the HeaderBlock. + // If this flag is not available for some object it MUST be assumed to be + // true if the file has the required_features tag "HistoricalInformation" + // set. + optional bool visible = 6; +} + +/** Optional metadata that may be included into each primitive. Special dense format used in DenseNodes. */ +message DenseInfo { + repeated int32 version = 1 [packed = true]; + repeated sint64 timestamp = 2 [packed = true]; // DELTA coded + repeated sint64 changeset = 3 [packed = true]; // DELTA coded + repeated sint32 uid = 4 [packed = true]; // DELTA coded + repeated sint32 user_sid = 5 [packed = true]; // String IDs for usernames. DELTA coded + + // The visible flag is used to store history information. It indicates that + // the current object version has been created by a delete operation on the + // OSM API. + // When a writer sets this flag, it MUST add a required_features tag with + // value "HistoricalInformation" to the HeaderBlock. + // If this flag is not available for some object it MUST be assumed to be + // true if the file has the required_features tag "HistoricalInformation" + // set. + repeated bool visible = 6 [packed = true]; +} + + +// This is kept for backwards compatibility but not used anywhere. +message ChangeSet { + required int64 id = 1; +} + + +message Node { + required sint64 id = 1; + + // Parallel arrays. + repeated uint32 keys = 2 [packed = true]; // String IDs. + repeated uint32 vals = 3 [packed = true]; // String IDs. + + optional Info info = 4; // May be omitted in omitmeta + + required sint64 lat = 8; + required sint64 lon = 9; +} + +/* Used to densly represent a sequence of nodes that do not have any tags. +We represent these nodes columnwise as five columns: ID's, lats, and +lons, all delta coded. When metadata is not omitted, +We encode keys & vals for all nodes as a single array of integers +containing key-stringid and val-stringid, using a stringid of 0 as a +delimiter between nodes. + ( ( )* '0' )* + */ + +message DenseNodes { + repeated sint64 id = 1 [packed = true]; // DELTA coded + + optional DenseInfo denseinfo = 5; + + repeated sint64 lat = 8 [packed = true]; // DELTA coded + repeated sint64 lon = 9 [packed = true]; // DELTA coded + + // Special packing of keys and vals into one array. May be empty if all nodes in this block are tagless. + repeated int32 keys_vals = 10 [packed = true]; +} + + +message Way { + required int64 id = 1; + + // Parallel arrays. + repeated uint32 keys = 2 [packed = true]; + repeated uint32 vals = 3 [packed = true]; + + optional Info info = 4; + + repeated sint64 refs = 8 [packed = true]; // DELTA coded + + // The following two fields are optional. They are only used in a special + // format where node locations are also added to the ways. This makes the + // files larger, but allows creating way geometries directly. + // + // If this is used, you MUST set the optional_features tag "LocationsOnWays" + // and the number of values in refs, lat, and lon MUST be the same. + repeated sint64 lat = 9 [packed = true]; // DELTA coded, optional + repeated sint64 lon = 10 [packed = true]; // DELTA coded, optional +} + +message Relation { + enum MemberType { + NODE = 0; + WAY = 1; + RELATION = 2; + } + + required int64 id = 1; + + // Parallel arrays. + repeated uint32 keys = 2 [packed = true]; + repeated uint32 vals = 3 [packed = true]; + + optional Info info = 4; + + // Parallel arrays + repeated int32 roles_sid = 8 [packed = true]; // This should have been defined as uint32 for consistency, but it is now too late to change it + repeated sint64 memids = 9 [packed = true]; // DELTA encoded + repeated MemberType types = 10 [packed = true]; +} \ No newline at end of file diff --git a/core/commands/pipeline/api.go b/core/commands/pipeline/api.go index 71ca1de..14c30da 100644 --- a/core/commands/pipeline/api.go +++ b/core/commands/pipeline/api.go @@ -69,40 +69,3 @@ func (b *ProgressBarBeat) Done() { panic(err) } } - -func BatchRequest(values <-chan map[string]interface{}, maxItems int, maxTimeout time.Duration) chan []map[string]interface{} { - batches := make(chan []map[string]interface{}) - - go func() { - defer close(batches) - - for keepGoing := true; keepGoing; { - var batch []map[string]interface{} - expire := time.After(maxTimeout) - for { - select { - case value, ok := <-values: - if !ok { - keepGoing = false - goto done - } - - batch = append(batch, value) - if len(batch) == maxItems { - goto done - } - - case <-expire: - goto done - } - } - - done: - if len(batch) > 0 { - batches <- batch - } - } - }() - - return batches -} diff --git a/core/commands/pipeline/connector_processor.go b/core/commands/pipeline/connector_processor.go index 1263a11..30377e4 100644 --- a/core/commands/pipeline/connector_processor.go +++ b/core/commands/pipeline/connector_processor.go @@ -1,6 +1,9 @@ package pipeline -import "github.com/meekyphotos/experive-cli/core/commands/connectors" +import ( + "github.com/meekyphotos/experive-cli/core/commands/connectors" + "time" +) func ProcessChannel(channel chan []map[string]interface{}, db connectors.Connector) error { for { @@ -17,3 +20,40 @@ func ProcessChannel(channel chan []map[string]interface{}, db connectors.Connect } } } + +func BatchRequest(values <-chan map[string]interface{}, maxItems int, maxTimeout time.Duration) chan []map[string]interface{} { + batches := make(chan []map[string]interface{}) + + go func() { + defer close(batches) + + for keepGoing := true; keepGoing; { + var batch []map[string]interface{} + expire := time.After(maxTimeout) + for { + select { + case value, ok := <-values: + if !ok { + keepGoing = false + goto done + } + + batch = append(batch, value) + if len(batch) == maxItems { + goto done + } + + case <-expire: + goto done + } + } + + done: + if len(batch) > 0 { + batches <- batch + } + } + }() + + return batches +} diff --git a/core/commands/pipeline/pbf_reader.go b/core/commands/pipeline/pbf_reader.go new file mode 100644 index 0000000..2ac5ccf --- /dev/null +++ b/core/commands/pipeline/pbf_reader.go @@ -0,0 +1,59 @@ +package pipeline + +import ( + "github.com/meekyphotos/experive-cli/core/commands/osm" + "io" + "os" + "runtime" +) + +func ReadFromPbf(path string, heartbeat Heartbeat) (chan map[string]interface{}, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + out := make(chan map[string]interface{}, 100000) + heartbeat.Start() + go func() { + defer f.Close() + defer heartbeat.Done() + open, err := os.Open(path) + if err != nil { + panic(err) + } + d := osm.NewDecoder(open) + d.SetBufferSize(osm.MaxBlobSize) + d.Skip(false, true, true) + if err := d.Start(runtime.GOMAXPROCS(-1)); err != nil { + panic(err) + } + for { + v, err := d.Decode() + if err == io.EOF { + break + } + if err != nil { + panic(err) + } + + switch v.(type) { + case *osm.Node: + m := make(map[string]interface{}, 7) + node := v.(osm.Node) + m["osm_id"] = node.OsmId + m["class"] = node.Class + m["type"] = node.Type + m["latitude"] = node.Lat + m["longitude"] = node.Lon + m["metadata"] = node.Tags + m["names"] = node.Names + out <- m + case *osm.Way: + case *osm.Relation: + default: + } + } + close(out) + }() + return out, nil +} diff --git a/go.mod b/go.mod index 81eae69..b7bb9f0 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/meekyphotos/experive-cli go 1.16 require ( + google.golang.org/protobuf v1.26.0 // indirect github.com/godruoyi/go-snowflake v0.0.2-alpha // indirect github.com/jedib0t/go-pretty/v6 v6.2.4 // indirect github.com/lib/pq v1.10.2 // indirect diff --git a/go.sum b/go.sum index 1efac6d..e943d4b 100644 --- a/go.sum +++ b/go.sum @@ -1,15 +1,47 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/datadog/czlib v0.0.0-20160811164712-4bc9a24e37f2 h1:ISaMhBq2dagaoptFGUyywT5SzpysCbHofX3sCNw1djo= +github.com/datadog/czlib v0.0.0-20160811164712-4bc9a24e37f2/go.mod h1:2yDaWzisHKoQoxm+EU4YgKBaD7g1M0pxy7THWG44Lro= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fzipp/gocyclo v0.3.1/go.mod h1:DJHO6AUmbdqj2ET4Z9iArSuwWgYDRryYt2wASxc7x3E= github.com/godruoyi/go-snowflake v0.0.2-alpha h1:ddz2txqjmq30B4b02H+g/MILCLW5V5Buz2+JW65mGjw= github.com/godruoyi/go-snowflake v0.0.2-alpha/go.mod h1:6JXMZzmleLpSK9pYpg4LXTcAz54mdYXTeXUvVks17+4= +github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/jedib0t/go-pretty/v6 v6.2.4 h1:wdaj2KHD2W+mz8JgJ/Q6L/T5dB7kyqEFI16eLq7GEmk= github.com/jedib0t/go-pretty/v6 v6.2.4/go.mod h1:+nE9fyyHGil+PuISTCrp7avEdo6bqoMwqZnuiK2r2a0= github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= @@ -18,15 +50,27 @@ github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4 github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= +github.com/paulmach/orb v0.1.6 h1:C8klK4r0mR0MnfSk+GvEFFKLrQVwjQ+FlhtXgpaupjg= +github.com/paulmach/orb v0.1.6/go.mod h1:pPwxxs3zoAyosNSbNKn1jiXV2+oovRDObDKfTvRegDI= +github.com/paulmach/osm v0.2.2 h1:fcRB9q4JPMtj/BTiAtRGEJXdJYGopIWWdNE/dk8hKDw= +github.com/paulmach/osm v0.2.2/go.mod h1:bHtjwVUgLRe/C6Uy5+wcvuD4TqrBHBvLP67F+GquY4I= +github.com/paulmach/protoscan v0.1.0 h1:4nM2d0bvdr4pfBC302n1/1QL9oXkenxujFXhLA19aAg= +github.com/paulmach/protoscan v0.1.0/go.mod h1:2c55sl1Hu6/tgRfc8Y8zADsxuSCYC2IrPh0JCqP/yrw= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/qedus/osmpbf v1.2.0 h1:yRm5ECkiUsN9sA+UN9yNnm64AVW2OYhOCb+gBa1FYCU= +github.com/qedus/osmpbf v1.2.0/go.mod h1:Cfv6JyqTZ72BjoW9FyFBQOC2DYJbL78yw+DLhBvSH+M= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/schollz/progressbar/v3 v3.8.2 h1:2kZJwZCpb+E/V79kGO7daeq+hUwUJW0A5QD1Wv455dA= github.com/schollz/progressbar/v3 v3.8.2/go.mod h1:9KHLdyuXczIsyStQwzvW8xiELskmX7fQMaZdN23nAv8= +github.com/shanghuiyang/pbfmap v0.0.0-20200910152534-7aafeafcac66 h1:aIcOcyDAZn+1i9UXy2soTG6tp0I8f9Sk4gP9Rnebrno= +github.com/shanghuiyang/pbfmap v0.0.0-20200910152534-7aafeafcac66/go.mod h1:IkOW7t6bJ60ugQ9hONVtFcrVcKnLW3EM8nqBngoIBOE= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -38,11 +82,26 @@ github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/valyala/fastjson v1.6.3 h1:tAKFnnwmeMGPbwJ7IwxcTPCNr3uIzoIj3/Fh90ra4xc= github.com/valyala/fastjson v1.6.3/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180816055513-1c9583448a9c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4= @@ -53,9 +112,38 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=