From d6e6e2e0bf28a2040ebc1a42aef96ab9b74e5af9 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Fri, 1 Sep 2023 19:42:43 +0200 Subject: [PATCH 01/22] Add initial encoder implementation --- osmpbf/encode.go | 422 +++++++++++++++++++++++++++++++++++++++++++++ osmpbf/zlib_cgo.go | 5 + osmpbf/zlib_go.go | 5 + 3 files changed, 432 insertions(+) create mode 100644 osmpbf/encode.go diff --git a/osmpbf/encode.go b/osmpbf/encode.go new file mode 100644 index 00000000..81d28b07 --- /dev/null +++ b/osmpbf/encode.go @@ -0,0 +1,422 @@ +package osmpbf + +import ( + "bytes" + "io" + "time" + + "github.com/paulmach/osm" + "github.com/paulmach/osm/osmpbf/internal/osmpbf" + "google.golang.org/protobuf/proto" +) + +// Writer is an interface for writing osm data. The format written is the osm +// pbf format. +type Writer interface { + io.Closer + WriteObject(obj osm.Object) error +} + +func NewWriter(w io.Writer) (Writer, error) { + encoder := &encoder{ + stream: w, + reverseStringTable: make(map[string]int), + compress: true, + } + + blockHeader := &osmpbf.HeaderBlock{ + RequiredFeatures: []string{"OsmSchema-V0.6", "DenseNodes"}, + } + blockHeaderData, err := proto.Marshal(blockHeader) + if err != nil { + return nil, err + } + + _, err = encoder.write(blockHeaderData) + if err != nil { + return nil, err + } + + return encoder, nil +} + +type encoder struct { + stream io.Writer + reverseStringTable map[string]int + entities []osm.Object + compress bool +} + +func (e *encoder) Close() error { + return e.flush() +} + +func (e *encoder) flush() error { + if len(e.entities) == 0 { + return nil + } + + block := &osmpbf.PrimitiveBlock{} + encode(block, e.reverseStringTable, e.entities, e.compress) + e.entities = e.entities[:0] + for k := range e.reverseStringTable { + delete(e.reverseStringTable, k) + } + + blockData, err := proto.Marshal(block) + if err != nil { + return nil + } + _, err = e.write(blockData) + return err +} + +func (e *encoder) write(data []byte) (n int, err error) { + blob := &osmpbf.Blob{} + blob.RawSize = proto.Int32(int32(len(data))) + if e.compress { + target := &bytes.Buffer{} + // compress the data + writer := zlibWriter(target) + writer.Write(data) + writer.Close() + blob.ZlibData = target.Bytes() + } else { + blob.Raw = data + } + + blobData, err := proto.Marshal(blob) + if err != nil { + return 0, nil + } + + blobHeader := &osmpbf.BlobHeader{ + Datasize: proto.Int32(int32(len(blobData))), + Type: proto.String(osmDataType), + } + + blobHeaderData, err := proto.Marshal(blobHeader) + if err != nil { + return 0, nil + } + + if _, err = e.stream.Write(blobHeaderData); err != nil { + return 0, nil + } + + return e.stream.Write(blobData) +} + +func (e *encoder) WriteObject(obj osm.Object) error { + e.entities = append(e.entities, obj) + if len(e.entities) >= 8000 { + return e.flush() + } + return nil +} + +func encode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, osmGeos []osm.Object, compressed bool) { + groupIdx := 0 + i := 0 + + if block.Stringtable != nil && block.Stringtable.S != nil { + block.Stringtable.S = nil + } + + for i < len(osmGeos) { + var group *osmpbf.PrimitiveGroup = nil + if groupIdx < len(block.Primitivegroup) { + group = block.Primitivegroup[groupIdx] + if group == nil { + group = &osmpbf.PrimitiveGroup{} + } + if group.Dense != nil { + if group.Dense.Denseinfo != nil { + if group.Dense.Denseinfo.Changeset != nil { + group.Dense.Denseinfo.Changeset = nil + } + if group.Dense.Denseinfo.Timestamp != nil { + group.Dense.Denseinfo.Timestamp = nil + } + if group.Dense.Denseinfo.Uid != nil { + group.Dense.Denseinfo.Uid = nil + } + if group.Dense.Denseinfo.UserSid != nil { + group.Dense.Denseinfo.UserSid = nil + } + if group.Dense.Denseinfo.Version != nil { + group.Dense.Denseinfo.Version = nil + } + } + if group.Dense.Id != nil { + group.Dense.Id = nil + } + if group.Dense.KeysVals != nil { + group.Dense.KeysVals = nil + } + if group.Dense.Lat != nil { + group.Dense.Lat = nil + } + if group.Dense.Lon != nil { + group.Dense.Lon = nil + } + } + if group.Changesets != nil { + group.Changesets = nil + } + if group.Ways != nil { + group.Ways = nil + } + if group.Relations != nil { + group.Relations = nil + } + } else { + group = &osmpbf.PrimitiveGroup{} + block.Primitivegroup = append(block.Primitivegroup, group) + } + + currentNodeCount := 0 + groupType := osmGeos[i].ObjectID().Type() + previousNode := &osm.Node{} + if groupType == osm.TypeNode && compressed && group.Dense == nil { + group.Dense = &osmpbf.DenseNodes{Denseinfo: &osmpbf.DenseInfo{}} + } + for i < len(osmGeos) && osmGeos[i].ObjectID().Type() == groupType { + switch groupType { + case osm.TypeNode: + if compressed { + currentNode := osmGeos[i].(*osm.Node) + EncodeDenseNode(block, reverseStringTable, group.Dense, currentNode, previousNode) + previousNode = currentNode + } else { + if currentNodeCount < len(group.Nodes) { + EncodeNode(block, reverseStringTable, group.Nodes[currentNodeCount], osmGeos[i].(*osm.Node)) + } else { + pbfNode := &osmpbf.Node{} + group.Nodes = append(group.Nodes, EncodeNode(block, reverseStringTable, pbfNode, osmGeos[i].(*osm.Node))) + } + currentNodeCount++ + } + case osm.TypeWay: + group.Ways = append(group.Ways, EncodeWay(block, reverseStringTable, osmGeos[i].(*osm.Way))) + case osm.TypeRelation: + group.Relations = append(group.Relations, EncodeRelation(block, reverseStringTable, osmGeos[i].(*osm.Relation))) + } + i++ + } + + if group.Nodes != nil { + for currentNodeCount < len(group.Nodes) { + group.Nodes = group.Nodes[:len(group.Nodes)-1] + } + } + + groupIdx++ + } + + if groupIdx < len(block.Primitivegroup) { + block.Primitivegroup = block.Primitivegroup[:groupIdx] + } +} + +func EncodeDenseNode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, groupDense *osmpbf.DenseNodes, current, previous *osm.Node) { + groupDense.Id = append(groupDense.Id, int64(current.ID-previous.ID)) + // cast block.Granularity to int64 + granularity := int64(*block.Granularity) + currentLat := EncodeLatLon(current.Lat, int64(*block.LatOffset), granularity) + currentLon := EncodeLatLon(current.Lon, int64(*block.LonOffset), granularity) + previousLat := EncodeLatLon(previous.Lat, int64(*block.LatOffset), granularity) + previousLon := EncodeLatLon(previous.Lon, int64(*block.LonOffset), granularity) + latDiff := currentLat - previousLat + lonDiff := currentLon - previousLon + groupDense.Lat = append(groupDense.Lat, latDiff) + groupDense.Lon = append(groupDense.Lon, lonDiff) + + if current.Tags != nil { + for _, nodeTag := range current.Tags { + groupDense.KeysVals = append(groupDense.KeysVals, EncodeString(block, reverseStringTable, nodeTag.Key)) + groupDense.KeysVals = append(groupDense.KeysVals, EncodeString(block, reverseStringTable, nodeTag.Value)) + } + groupDense.KeysVals = append(groupDense.KeysVals, 0) + } + + if groupDense.Denseinfo != nil { + groupDense.Denseinfo.Changeset = append(groupDense.Denseinfo.Changeset, int64(current.ChangesetID-previous.ChangesetID)) + dateGranularity := int64(*block.DateGranularity) + currentTimeStamp := EncodeTimestamp(current.Timestamp, dateGranularity) + previousTimeStamp := EncodeTimestamp(previous.Timestamp, dateGranularity) + groupDense.Denseinfo.Timestamp = append(groupDense.Denseinfo.Timestamp, currentTimeStamp-previousTimeStamp) + groupDense.Denseinfo.Uid = append(groupDense.Denseinfo.Uid, int32(current.UserID-previous.UserID)) + groupDense.Denseinfo.Version = append(groupDense.Denseinfo.Version, int32(current.Version-previous.Version)) + var previousUserNameId int32 = 0 + if previous.User != "" { + previousUserNameId = EncodeString(block, reverseStringTable, previous.User) + } + currentUserNameId := EncodeString(block, reverseStringTable, current.User) + groupDense.Denseinfo.UserSid = append(groupDense.Denseinfo.UserSid, currentUserNameId-previousUserNameId) + } +} + +func EncodeNode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, pbfNode *osmpbf.Node, node *osm.Node) *osmpbf.Node { + castId := int64(node.ID) + pbfNode.Id = &castId + zero := int32(0) + pbfNode.Info = &osmpbf.Info{ + Version: &zero, + } + if node.ChangesetID != 0 { + changesetId := int64(node.ChangesetID) + pbfNode.Info.Changeset = &changesetId + } + if !node.Timestamp.IsZero() { + timeStamp := EncodeTimestamp(node.Timestamp, int64(*block.DateGranularity)) + pbfNode.Info.Timestamp = &timeStamp + } + if node.UserID != 0 { + userId := int32(node.UserID) + pbfNode.Info.Uid = &userId + } + userSid := uint32(EncodeString(block, reverseStringTable, node.User)) + pbfNode.Info.UserSid = &userSid + if node.Version != 0 { + nodeVersion := int32(node.Version) + pbfNode.Info.Version = &nodeVersion + } + lat := EncodeLatLon(node.Lat, *block.LatOffset, int64(*block.Granularity)) + pbfNode.Lat = &lat + lon := EncodeLatLon(node.Lon, *block.LonOffset, int64(*block.Granularity)) + pbfNode.Lon = &lon + + if len(node.Tags) > 0 { + for _, tag := range node.Tags { + pbfNode.Keys = append(pbfNode.Keys, uint32(EncodeString(block, reverseStringTable, tag.Key))) + pbfNode.Vals = append(pbfNode.Vals, uint32(EncodeString(block, reverseStringTable, tag.Value))) + } + } else { + pbfNode.Keys = nil + pbfNode.Vals = nil + } + return pbfNode +} + +func EncodeWay(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, way *osm.Way) *osmpbf.Way { + pbfWay := &osmpbf.Way{ + Id: (*int64)(&way.ID), + Info: &osmpbf.Info{}, + } + if way.ChangesetID != 0 { + pbfWay.Info.Changeset = (*int64)(&way.ChangesetID) + } + if !way.Timestamp.IsZero() { + timestamp := EncodeTimestamp(way.Timestamp, int64(*block.DateGranularity)) + pbfWay.Info.Timestamp = ×tamp + } + if way.UserID != 0 { + userId := int32(way.UserID) + pbfWay.Info.Uid = &userId + } + userId := uint32(EncodeString(block, reverseStringTable, way.User)) + pbfWay.Info.UserSid = &userId + if way.Version != 0 { + wayVersion := int32(way.Version) + pbfWay.Info.Version = &wayVersion + } + + if len(way.Tags) > 0 { + for _, tag := range way.Tags { + pbfWay.Keys = append(pbfWay.Keys, uint32(EncodeString(block, reverseStringTable, tag.Key))) + pbfWay.Vals = append(pbfWay.Vals, uint32(EncodeString(block, reverseStringTable, tag.Value))) + } + } + + if len(way.Nodes) > 0 { + pbfWay.Refs = append(pbfWay.Refs, int64(way.Nodes[0].ID)) + for i := 1; i < len(way.Nodes); i++ { + pbfWay.Refs = append(pbfWay.Refs, int64(way.Nodes[i].ID)-int64(way.Nodes[i-1].ID)) + } + } + return pbfWay +} + +func EncodeRelation(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, relation *osm.Relation) *osmpbf.Relation { + pbfRelation := &osmpbf.Relation{ + Id: (*int64)(&relation.ID), + Info: &osmpbf.Info{ + Changeset: (*int64)(&relation.ChangesetID), + }, + } + if !relation.Timestamp.IsZero() { + timestamp := EncodeTimestamp(relation.Timestamp, int64(*block.DateGranularity)) + pbfRelation.Info.Timestamp = ×tamp + } + if relation.UserID != 0 { + userId := int32(relation.UserID) + pbfRelation.Info.Uid = &userId + } + userSid := uint32(EncodeString(block, reverseStringTable, relation.User)) + pbfRelation.Info.UserSid = &userSid + relationVersion := int32(relation.Version) + pbfRelation.Info.Version = &relationVersion + + if len(relation.Tags) > 0 { + for _, tag := range relation.Tags { + pbfRelation.Keys = append(pbfRelation.Keys, uint32(EncodeString(block, reverseStringTable, tag.Key))) + pbfRelation.Vals = append(pbfRelation.Vals, uint32(EncodeString(block, reverseStringTable, tag.Value))) + } + } + + if len(relation.Members) > 0 { + pbfRelation.Memids = append(pbfRelation.Memids, relation.Members[0].Ref) + pbfRelation.RolesSid = append(pbfRelation.RolesSid, EncodeString(block, reverseStringTable, relation.Members[0].Role)) + switch relation.Members[0].Type { + case osm.TypeNode: + pbfRelation.Types = append(pbfRelation.Types, osmpbf.Relation_NODE) + case osm.TypeWay: + pbfRelation.Types = append(pbfRelation.Types, osmpbf.Relation_WAY) + case osm.TypeRelation: + pbfRelation.Types = append(pbfRelation.Types, osmpbf.Relation_RELATION) + } + for i := 1; i < len(relation.Members); i++ { + pbfRelation.Memids = append(pbfRelation.Memids, relation.Members[i].Ref-relation.Members[i-1].Ref) + pbfRelation.RolesSid = append(pbfRelation.RolesSid, EncodeString(block, reverseStringTable, relation.Members[i].Role)) + switch relation.Members[i].Type { + case osm.TypeNode: + pbfRelation.Types = append(pbfRelation.Types, osmpbf.Relation_NODE) + case osm.TypeWay: + pbfRelation.Types = append(pbfRelation.Types, osmpbf.Relation_WAY) + case osm.TypeRelation: + pbfRelation.Types = append(pbfRelation.Types, osmpbf.Relation_RELATION) + } + } + } + return pbfRelation +} + +func EncodeLatLon(value float64, offset, granularity int64) int64 { + return (int64(value/.000000001) - offset) / granularity +} + +func EncodeTimestamp(timestamp time.Time, dateGranularity int64) int64 { + return timestamp.Unix() / dateGranularity +} + +func EncodeString(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, value string) int32 { + if value == "" { + return 0 + } + + if block.Stringtable == nil { + block.Stringtable = &osmpbf.StringTable{} + block.Stringtable.S = append(block.Stringtable.S, "") + reverseStringTable[""] = 0 + } + + if id, ok := reverseStringTable[value]; ok { + return int32(id) + } + + block.Stringtable.S = append(block.Stringtable.S, value) + id := len(block.Stringtable.S) - 1 + reverseStringTable[value] = id + return int32(id) +} diff --git a/osmpbf/zlib_cgo.go b/osmpbf/zlib_cgo.go index fbcf3f66..b6aaa949 100644 --- a/osmpbf/zlib_cgo.go +++ b/osmpbf/zlib_cgo.go @@ -1,3 +1,4 @@ +//go:build cgo // +build cgo package osmpbf @@ -12,3 +13,7 @@ import ( func zlibReader(data []byte) (io.ReadCloser, error) { return czlib.NewReader(bytes.NewReader(data)) } + +func zlibWriter(w io.Writer) io.WriteCloser { + return czlib.NewWriter(w) +} diff --git a/osmpbf/zlib_go.go b/osmpbf/zlib_go.go index ddae9b86..6b7b657a 100644 --- a/osmpbf/zlib_go.go +++ b/osmpbf/zlib_go.go @@ -1,3 +1,4 @@ +//go:build !cgo // +build !cgo package osmpbf @@ -11,3 +12,7 @@ import ( func zlibReader(data []byte) (io.ReadCloser, error) { return zlib.NewReader(bytes.NewReader(data)) } + +func zlibWriter(w io.Writer) io.WriteCloser { + return zlib.NewWriter(w) +} From c2da18ded1fd55e80eb733340684eda2a6ae32e1 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Fri, 1 Sep 2023 19:45:16 +0200 Subject: [PATCH 02/22] Fix module references --- annotate/change.go | 4 ++-- annotate/change_test.go | 2 +- annotate/datasource.go | 6 +++--- annotate/edgecases_test.go | 2 +- annotate/errors.go | 4 ++-- annotate/errors_test.go | 2 +- annotate/geo.go | 4 ++-- annotate/geo_test.go | 2 +- annotate/internal/core/compute.go | 4 ++-- annotate/internal/core/compute_test.go | 4 ++-- annotate/internal/core/datasourcer_test.go | 2 +- annotate/internal/core/errors.go | 2 +- annotate/internal/core/types.go | 4 ++-- annotate/internal/core/types_test.go | 4 ++-- annotate/options.go | 22 +++++++++++----------- annotate/order.go | 2 +- annotate/order_test.go | 2 +- annotate/relation.go | 6 +++--- annotate/relation_test.go | 2 +- annotate/shared/child.go | 2 +- annotate/way.go | 6 +++--- annotate/way_test.go | 2 +- go.mod | 2 +- internal/mputil/mputil.go | 2 +- internal/mputil/mputil_test.go | 2 +- json.go | 4 ++-- osmapi/changeset.go | 2 +- osmapi/datasource.go | 7 ++++--- osmapi/live_test.go | 2 +- osmapi/map.go | 2 +- osmapi/map_test.go | 2 +- osmapi/node.go | 2 +- osmapi/node_test.go | 2 +- osmapi/note.go | 2 +- osmapi/note_test.go | 2 +- osmapi/relation.go | 2 +- osmapi/relation_test.go | 2 +- osmapi/user.go | 2 +- osmapi/way.go | 2 +- osmapi/way_test.go | 2 +- osmgeojson/benchmarks_test.go | 2 +- osmgeojson/build_polygon.go | 4 ++-- osmgeojson/build_polygon_test.go | 2 +- osmgeojson/convert.go | 4 ++-- osmgeojson/convert_test.go | 2 +- osmgeojson/options_test.go | 2 +- osmpbf/decode.go | 4 ++-- osmpbf/decode_data.go | 6 +++--- osmpbf/decode_test.go | 2 +- osmpbf/encode.go | 4 ++-- osmpbf/example_stats_test.go | 4 ++-- osmpbf/internal/osmpbf/generate.go | 2 +- osmpbf/scanner.go | 2 +- osmpbf/scanner_test.go | 2 +- osmtest/scanner.go | 2 +- osmtest/scanner_test.go | 2 +- osmxml/example_test.go | 4 ++-- osmxml/scanner.go | 3 ++- osmxml/scanner_test.go | 2 +- replication/changesets.go | 4 ++-- replication/interval.go | 2 +- testdata/compare_scanners.go | 6 +++--- 62 files changed, 101 insertions(+), 99 deletions(-) diff --git a/annotate/change.go b/annotate/change.go index a011676e..421988a4 100644 --- a/annotate/change.go +++ b/annotate/change.go @@ -3,8 +3,8 @@ package annotate import ( "context" - "github.com/paulmach/osm" - "github.com/paulmach/osm/annotate/internal/core" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/annotate/internal/core" ) // Change will annotate a change into a diff. It will use the diff --git a/annotate/change_test.go b/annotate/change_test.go index 5706b589..fc7a5e44 100644 --- a/annotate/change_test.go +++ b/annotate/change_test.go @@ -4,7 +4,7 @@ import ( "context" "testing" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) func TestChange_create(t *testing.T) { diff --git a/annotate/datasource.go b/annotate/datasource.go index 41102f9a..0fdff5e6 100644 --- a/annotate/datasource.go +++ b/annotate/datasource.go @@ -3,9 +3,9 @@ package annotate import ( "context" - "github.com/paulmach/osm" - "github.com/paulmach/osm/annotate/internal/core" - "github.com/paulmach/osm/annotate/shared" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/annotate/internal/core" + "github.com/nextmv-io/osm/annotate/shared" "github.com/paulmach/orb" "github.com/paulmach/orb/planar" diff --git a/annotate/edgecases_test.go b/annotate/edgecases_test.go index 09c1310b..b1e7a8a0 100644 --- a/annotate/edgecases_test.go +++ b/annotate/edgecases_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) func TestEdgeCase_childCreatedAfterParent(t *testing.T) { diff --git a/annotate/errors.go b/annotate/errors.go index 13a7d08f..864c8562 100644 --- a/annotate/errors.go +++ b/annotate/errors.go @@ -4,8 +4,8 @@ import ( "fmt" "time" - "github.com/paulmach/osm" - "github.com/paulmach/osm/annotate/internal/core" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/annotate/internal/core" ) // NoHistoryError is returned if there is no entry in the history diff --git a/annotate/errors_test.go b/annotate/errors_test.go index a4ec64c8..e4f69cea 100644 --- a/annotate/errors_test.go +++ b/annotate/errors_test.go @@ -4,7 +4,7 @@ import ( "errors" "testing" - "github.com/paulmach/osm/annotate/internal/core" + "github.com/nextmv-io/osm/annotate/internal/core" ) func TestMapErrors(t *testing.T) { diff --git a/annotate/geo.go b/annotate/geo.go index 91ba7abd..ae1ee69f 100644 --- a/annotate/geo.go +++ b/annotate/geo.go @@ -4,10 +4,10 @@ import ( "math" "time" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/internal/mputil" "github.com/paulmach/orb" "github.com/paulmach/orb/geo" - "github.com/paulmach/osm" - "github.com/paulmach/osm/internal/mputil" ) func wayPointOnSurface(w *osm.Way) orb.Point { diff --git a/annotate/geo_test.go b/annotate/geo_test.go index bc697d20..818db4ed 100644 --- a/annotate/geo_test.go +++ b/annotate/geo_test.go @@ -4,8 +4,8 @@ import ( "encoding/xml" "testing" + "github.com/nextmv-io/osm" "github.com/paulmach/orb" - "github.com/paulmach/osm" ) func TestWayPointOnSurface(t *testing.T) { diff --git a/annotate/internal/core/compute.go b/annotate/internal/core/compute.go index 531b815d..46966623 100644 --- a/annotate/internal/core/compute.go +++ b/annotate/internal/core/compute.go @@ -5,8 +5,8 @@ import ( "fmt" "time" - "github.com/paulmach/osm" - "github.com/paulmach/osm/annotate/shared" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/annotate/shared" ) // A Datasourcer is something that acts like a datasource allowing us to diff --git a/annotate/internal/core/compute_test.go b/annotate/internal/core/compute_test.go index c8fe94e9..8d02f9ef 100644 --- a/annotate/internal/core/compute_test.go +++ b/annotate/internal/core/compute_test.go @@ -7,8 +7,8 @@ import ( "testing" "time" - "github.com/paulmach/osm" - "github.com/paulmach/osm/annotate/shared" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/annotate/shared" ) var ( diff --git a/annotate/internal/core/datasourcer_test.go b/annotate/internal/core/datasourcer_test.go index 4a2fb4aa..c522d183 100644 --- a/annotate/internal/core/datasourcer_test.go +++ b/annotate/internal/core/datasourcer_test.go @@ -4,7 +4,7 @@ import ( "context" "errors" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) var _ Datasourcer = &TestDS{} diff --git a/annotate/internal/core/errors.go b/annotate/internal/core/errors.go index 451e3ee7..2bba17a3 100644 --- a/annotate/internal/core/errors.go +++ b/annotate/internal/core/errors.go @@ -4,7 +4,7 @@ import ( "fmt" "time" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) // NoHistoryError is returned if there is no entry in the history diff --git a/annotate/internal/core/types.go b/annotate/internal/core/types.go index 87fb037d..8fcc627e 100644 --- a/annotate/internal/core/types.go +++ b/annotate/internal/core/types.go @@ -3,8 +3,8 @@ package core import ( "time" - "github.com/paulmach/osm" - "github.com/paulmach/osm/annotate/shared" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/annotate/shared" ) // A Parent is something that holds children. ie. ways have nodes as children diff --git a/annotate/internal/core/types_test.go b/annotate/internal/core/types_test.go index c5ef266a..44754dcc 100644 --- a/annotate/internal/core/types_test.go +++ b/annotate/internal/core/types_test.go @@ -4,8 +4,8 @@ import ( "testing" "time" - "github.com/paulmach/osm" - "github.com/paulmach/osm/annotate/shared" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/annotate/shared" ) type findVisibleTestCase struct { diff --git a/annotate/options.go b/annotate/options.go index 789e19da..142aacca 100644 --- a/annotate/options.go +++ b/annotate/options.go @@ -3,8 +3,8 @@ package annotate import ( "time" - "github.com/paulmach/osm" - "github.com/paulmach/osm/annotate/internal/core" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/annotate/internal/core" ) // Option is a parameter that can be used for annotating. @@ -28,15 +28,15 @@ func Threshold(t time.Duration) Option { // This should be used when you want to gracefully handle the weird data in OSM. // // Nodes with unclear/inconsistent data will not be annotated. Causes include: -// - redacted data: In 2012 due to the license change data had to be removed. -// This could be some nodes of a way. There exist ways for which some nodes have -// just a single delete version, e.g. way 159081205, node 376130526 -// - data pre element versioning: pre-2012(?) data versions were not kept, so -// for old ways there many be no information about some nodes. For example, -// a node may be updated after a way and there is no way to get the original -// version of the node and way. -// - bad editors: sometimes a node is edited 7 times in a single changeset -// and version 5 is a delete. See node 321452894, part of way 28831147. +// - redacted data: In 2012 due to the license change data had to be removed. +// This could be some nodes of a way. There exist ways for which some nodes have +// just a single delete version, e.g. way 159081205, node 376130526 +// - data pre element versioning: pre-2012(?) data versions were not kept, so +// for old ways there many be no information about some nodes. For example, +// a node may be updated after a way and there is no way to get the original +// version of the node and way. +// - bad editors: sometimes a node is edited 7 times in a single changeset +// and version 5 is a delete. See node 321452894, part of way 28831147. func IgnoreInconsistency(yes bool) Option { return func(o *core.Options) error { o.IgnoreInconsistency = yes diff --git a/annotate/order.go b/annotate/order.go index 022fe127..c324b803 100644 --- a/annotate/order.go +++ b/annotate/order.go @@ -4,7 +4,7 @@ import ( "context" "sync" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) // RelationHistoryDatasourcer is an more strict interface for when we only need the relation history. diff --git a/annotate/order_test.go b/annotate/order_test.go index f9f7010f..e29716c3 100644 --- a/annotate/order_test.go +++ b/annotate/order_test.go @@ -4,7 +4,7 @@ import ( "context" "testing" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) func TestChildFirstOrdering(t *testing.T) { diff --git a/annotate/relation.go b/annotate/relation.go index e1fe2bbe..6de0d68d 100644 --- a/annotate/relation.go +++ b/annotate/relation.go @@ -4,9 +4,9 @@ import ( "context" "time" - "github.com/paulmach/osm" - "github.com/paulmach/osm/annotate/internal/core" - "github.com/paulmach/osm/annotate/shared" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/annotate/internal/core" + "github.com/nextmv-io/osm/annotate/shared" ) // HistoryAsChildrenDatasourcer is an advanced data source that diff --git a/annotate/relation_test.go b/annotate/relation_test.go index 1619bbe6..8eb2e41d 100644 --- a/annotate/relation_test.go +++ b/annotate/relation_test.go @@ -9,8 +9,8 @@ import ( "testing" "time" + "github.com/nextmv-io/osm" "github.com/paulmach/orb" - "github.com/paulmach/osm" ) func TestRelation(t *testing.T) { diff --git a/annotate/shared/child.go b/annotate/shared/child.go index fba94709..0881f4e7 100644 --- a/annotate/shared/child.go +++ b/annotate/shared/child.go @@ -5,7 +5,7 @@ package shared import ( "time" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) // A Child represents a node, way or relation that is a dependent for diff --git a/annotate/way.go b/annotate/way.go index 7e0bc2eb..cb08f045 100644 --- a/annotate/way.go +++ b/annotate/way.go @@ -4,9 +4,9 @@ import ( "context" "time" - "github.com/paulmach/osm" - "github.com/paulmach/osm/annotate/internal/core" - "github.com/paulmach/osm/annotate/shared" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/annotate/internal/core" + "github.com/nextmv-io/osm/annotate/shared" ) // NodeHistoryDatasourcer is an more strict interface for when we only need node history. diff --git a/annotate/way_test.go b/annotate/way_test.go index 47cfc3ec..6dae0f63 100644 --- a/annotate/way_test.go +++ b/annotate/way_test.go @@ -8,7 +8,7 @@ import ( "reflect" "testing" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) func TestWays(t *testing.T) { diff --git a/go.mod b/go.mod index 5ea8c278..53234598 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/paulmach/osm +module github.com/nextmv-io/osm go 1.13 diff --git a/internal/mputil/mputil.go b/internal/mputil/mputil.go index 3e8693de..a4719da3 100644 --- a/internal/mputil/mputil.go +++ b/internal/mputil/mputil.go @@ -3,8 +3,8 @@ package mputil import ( "time" + "github.com/nextmv-io/osm" "github.com/paulmach/orb" - "github.com/paulmach/osm" ) // Segment is a section of a multipolygon with some extra information diff --git a/internal/mputil/mputil_test.go b/internal/mputil/mputil_test.go index 219573fa..b6a7c3de 100644 --- a/internal/mputil/mputil_test.go +++ b/internal/mputil/mputil_test.go @@ -5,8 +5,8 @@ import ( "testing" "time" + "github.com/nextmv-io/osm" "github.com/paulmach/orb" - "github.com/paulmach/osm" ) func TestMultiSegment_LineString(t *testing.T) { diff --git a/json.go b/json.go index 593f8f9e..423931ce 100644 --- a/json.go +++ b/json.go @@ -12,7 +12,7 @@ import ( // // import ( // jsoniter "github.com/json-iterator/go" -// "github.com/paulmach/osm" +// "github.com/nextmv-io/osm" // ) // // var c = jsoniter.Config{ @@ -36,7 +36,7 @@ var CustomJSONMarshaler interface { // // import ( // jsoniter "github.com/json-iterator/go" -// "github.com/paulmach/osm" +// "github.com/nextmv-io/osm" // ) // // var c = jsoniter.Config{ diff --git a/osmapi/changeset.go b/osmapi/changeset.go index 959b4f4c..0610a3a0 100644 --- a/osmapi/changeset.go +++ b/osmapi/changeset.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) // Changeset returns a given changeset from the osm rest api. diff --git a/osmapi/datasource.go b/osmapi/datasource.go index 223e3b34..3ad115fc 100644 --- a/osmapi/datasource.go +++ b/osmapi/datasource.go @@ -8,7 +8,7 @@ import ( "net/http" "time" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) // BaseURL defines the api host. This can be change to hit @@ -18,8 +18,9 @@ const BaseURL = "http://api.openstreetmap.org/api/0.6" // A RateLimiter is something that can wait until its next allowed request. // This interface is met by `golang.org/x/time/rate.Limiter` and is meant // to be used with it. For example: -// // 10 qps -// osmapi.DefaultDatasource.Limiter = rate.NewLimiter(10, 1) +// +// // 10 qps +// osmapi.DefaultDatasource.Limiter = rate.NewLimiter(10, 1) type RateLimiter interface { Wait(context.Context) error } diff --git a/osmapi/live_test.go b/osmapi/live_test.go index f09070bb..6d100dc3 100644 --- a/osmapi/live_test.go +++ b/osmapi/live_test.go @@ -5,7 +5,7 @@ import ( "os" "testing" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" "golang.org/x/time/rate" ) diff --git a/osmapi/map.go b/osmapi/map.go index d0b3a93b..5ee833b8 100644 --- a/osmapi/map.go +++ b/osmapi/map.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) // Map returns the latest elements in the given bounding box. diff --git a/osmapi/map_test.go b/osmapi/map_test.go index c8292257..b4f7bc44 100644 --- a/osmapi/map_test.go +++ b/osmapi/map_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) func TestMap_urls(t *testing.T) { diff --git a/osmapi/node.go b/osmapi/node.go index 13140c3b..3a42d6db 100644 --- a/osmapi/node.go +++ b/osmapi/node.go @@ -5,7 +5,7 @@ import ( "fmt" "strconv" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) // Node returns the latest version of the node from the osm rest api. diff --git a/osmapi/node_test.go b/osmapi/node_test.go index 12b541f4..93254627 100644 --- a/osmapi/node_test.go +++ b/osmapi/node_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) func TestNode_urls(t *testing.T) { diff --git a/osmapi/note.go b/osmapi/note.go index c903e563..72b718d0 100644 --- a/osmapi/note.go +++ b/osmapi/note.go @@ -6,7 +6,7 @@ import ( "net/url" "strings" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) // Note returns the note from the osm rest api. diff --git a/osmapi/note_test.go b/osmapi/note_test.go index ec93f432..3d193f5f 100644 --- a/osmapi/note_test.go +++ b/osmapi/note_test.go @@ -7,7 +7,7 @@ import ( "strings" "testing" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) func TestNote_urls(t *testing.T) { diff --git a/osmapi/relation.go b/osmapi/relation.go index e629e0bc..5e767c70 100644 --- a/osmapi/relation.go +++ b/osmapi/relation.go @@ -5,7 +5,7 @@ import ( "fmt" "strconv" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) // Relation returns the latest version of the relation from the osm rest api. diff --git a/osmapi/relation_test.go b/osmapi/relation_test.go index bea89341..b64cbc37 100644 --- a/osmapi/relation_test.go +++ b/osmapi/relation_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) func TestRelation_urls(t *testing.T) { diff --git a/osmapi/user.go b/osmapi/user.go index 17a7bff5..3b01ce81 100644 --- a/osmapi/user.go +++ b/osmapi/user.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) // User returns the user from the osm rest api. diff --git a/osmapi/way.go b/osmapi/way.go index 5f834fd0..9d75d2d1 100644 --- a/osmapi/way.go +++ b/osmapi/way.go @@ -5,7 +5,7 @@ import ( "fmt" "strconv" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) // Way returns the latest version of the way from the osm rest api. diff --git a/osmapi/way_test.go b/osmapi/way_test.go index 050357ee..6377415f 100644 --- a/osmapi/way_test.go +++ b/osmapi/way_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) func TestWay_urls(t *testing.T) { diff --git a/osmgeojson/benchmarks_test.go b/osmgeojson/benchmarks_test.go index 1dc71dd7..42cf0a3f 100644 --- a/osmgeojson/benchmarks_test.go +++ b/osmgeojson/benchmarks_test.go @@ -5,7 +5,7 @@ import ( "io/ioutil" "testing" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) func BenchmarkConvert(b *testing.B) { diff --git a/osmgeojson/build_polygon.go b/osmgeojson/build_polygon.go index bc00284d..45c5721b 100644 --- a/osmgeojson/build_polygon.go +++ b/osmgeojson/build_polygon.go @@ -3,10 +3,10 @@ package osmgeojson import ( "fmt" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/internal/mputil" "github.com/paulmach/orb" "github.com/paulmach/orb/geojson" - "github.com/paulmach/osm" - "github.com/paulmach/osm/internal/mputil" ) func (ctx *context) buildPolygon(relation *osm.Relation) *geojson.Feature { diff --git a/osmgeojson/build_polygon_test.go b/osmgeojson/build_polygon_test.go index d80750c9..fc684b88 100644 --- a/osmgeojson/build_polygon_test.go +++ b/osmgeojson/build_polygon_test.go @@ -4,9 +4,9 @@ import ( "encoding/xml" "testing" + "github.com/nextmv-io/osm" "github.com/paulmach/orb" "github.com/paulmach/orb/geojson" - "github.com/paulmach/osm" ) func TestConvert_polygon(t *testing.T) { diff --git a/osmgeojson/convert.go b/osmgeojson/convert.go index ddc2013b..e0c8cf19 100644 --- a/osmgeojson/convert.go +++ b/osmgeojson/convert.go @@ -3,10 +3,10 @@ package osmgeojson import ( "fmt" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/internal/mputil" "github.com/paulmach/orb" "github.com/paulmach/orb/geojson" - "github.com/paulmach/osm" - "github.com/paulmach/osm/internal/mputil" ) type context struct { diff --git a/osmgeojson/convert_test.go b/osmgeojson/convert_test.go index 3209d0ec..fbee4bf5 100644 --- a/osmgeojson/convert_test.go +++ b/osmgeojson/convert_test.go @@ -6,9 +6,9 @@ import ( "reflect" "testing" + "github.com/nextmv-io/osm" "github.com/paulmach/orb" "github.com/paulmach/orb/geojson" - "github.com/paulmach/osm" ) func TestConvert(t *testing.T) { diff --git a/osmgeojson/options_test.go b/osmgeojson/options_test.go index 2d74b964..80148247 100644 --- a/osmgeojson/options_test.go +++ b/osmgeojson/options_test.go @@ -4,8 +4,8 @@ import ( "encoding/xml" "testing" + "github.com/nextmv-io/osm" "github.com/paulmach/orb/geojson" - "github.com/paulmach/osm" ) var nodeXML = ` diff --git a/osmpbf/decode.go b/osmpbf/decode.go index eec3892a..54473990 100644 --- a/osmpbf/decode.go +++ b/osmpbf/decode.go @@ -10,8 +10,8 @@ import ( "sync" "time" - "github.com/paulmach/osm" - "github.com/paulmach/osm/osmpbf/internal/osmpbf" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/osmpbf/internal/osmpbf" "google.golang.org/protobuf/proto" ) diff --git a/osmpbf/decode_data.go b/osmpbf/decode_data.go index 1847a4ba..ba782721 100644 --- a/osmpbf/decode_data.go +++ b/osmpbf/decode_data.go @@ -4,10 +4,10 @@ import ( "errors" "time" - "google.golang.org/protobuf/proto" - "github.com/paulmach/osm" - "github.com/paulmach/osm/osmpbf/internal/osmpbf" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/osmpbf/internal/osmpbf" "github.com/paulmach/protoscan" + "google.golang.org/protobuf/proto" ) // dataDecoder is a decoder for Blob with OSMData (PrimitiveBlock). diff --git a/osmpbf/decode_test.go b/osmpbf/decode_test.go index 6b6785c6..8a4a8698 100644 --- a/osmpbf/decode_test.go +++ b/osmpbf/decode_test.go @@ -12,7 +12,7 @@ import ( "testing" "time" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) const ( diff --git a/osmpbf/encode.go b/osmpbf/encode.go index 81d28b07..a65cba54 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -5,8 +5,8 @@ import ( "io" "time" - "github.com/paulmach/osm" - "github.com/paulmach/osm/osmpbf/internal/osmpbf" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/osmpbf/internal/osmpbf" "google.golang.org/protobuf/proto" ) diff --git a/osmpbf/example_stats_test.go b/osmpbf/example_stats_test.go index 9a8182f9..b9d7a049 100644 --- a/osmpbf/example_stats_test.go +++ b/osmpbf/example_stats_test.go @@ -7,8 +7,8 @@ import ( "os" "time" - "github.com/paulmach/osm" - "github.com/paulmach/osm/osmpbf" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/osmpbf" ) // ExampleStats demonstrates how to read a full file and gather some stats. diff --git a/osmpbf/internal/osmpbf/generate.go b/osmpbf/internal/osmpbf/generate.go index 8eee6f9b..a330ef3f 100644 --- a/osmpbf/internal/osmpbf/generate.go +++ b/osmpbf/internal/osmpbf/generate.go @@ -1,3 +1,3 @@ package osmpbf -//go:generate protoc --proto_path=. --go_opt=module=github.com/paulmach/osmpbf/internal/osmpbf --go_out=. fileformat.proto osmformat.proto +//go:generate protoc --proto_path=. --go_opt=module=github.com/nextmv-io/osmpbf/internal/osmpbf --go_out=. fileformat.proto osmformat.proto diff --git a/osmpbf/scanner.go b/osmpbf/scanner.go index b0479029..201b7a43 100644 --- a/osmpbf/scanner.go +++ b/osmpbf/scanner.go @@ -5,7 +5,7 @@ import ( "io" "sync/atomic" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) var _ osm.Scanner = &Scanner{} diff --git a/osmpbf/scanner_test.go b/osmpbf/scanner_test.go index fd47c082..70d9e6db 100644 --- a/osmpbf/scanner_test.go +++ b/osmpbf/scanner_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) var ( diff --git a/osmtest/scanner.go b/osmtest/scanner.go index 1119c865..3a35188e 100644 --- a/osmtest/scanner.go +++ b/osmtest/scanner.go @@ -1,6 +1,6 @@ package osmtest -import "github.com/paulmach/osm" +import "github.com/nextmv-io/osm" // Scanner implements the osm.Scanner interface with // just a list of objects. diff --git a/osmtest/scanner_test.go b/osmtest/scanner_test.go index f3c68207..fff9f7e6 100644 --- a/osmtest/scanner_test.go +++ b/osmtest/scanner_test.go @@ -5,7 +5,7 @@ import ( "reflect" "testing" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) func TestScanner(t *testing.T) { diff --git a/osmxml/example_test.go b/osmxml/example_test.go index 61700dca..37979075 100644 --- a/osmxml/example_test.go +++ b/osmxml/example_test.go @@ -5,8 +5,8 @@ import ( "fmt" "os" - "github.com/paulmach/osm" - "github.com/paulmach/osm/osmxml" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/osmxml" ) func ExampleScanner() { diff --git a/osmxml/scanner.go b/osmxml/scanner.go index b232d357..c0c91926 100644 --- a/osmxml/scanner.go +++ b/osmxml/scanner.go @@ -6,7 +6,7 @@ import ( "io" "strings" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) var _ osm.Scanner = &Scanner{} @@ -126,6 +126,7 @@ Loop: // Object returns the most recent token generated by a call to Scan // as a new osm.Object. This interface is implemented by: +// // *osm.Bounds // *osm.Node // *osm.Way diff --git a/osmxml/scanner_test.go b/osmxml/scanner_test.go index fd85aa7a..daf7dc65 100644 --- a/osmxml/scanner_test.go +++ b/osmxml/scanner_test.go @@ -8,7 +8,7 @@ import ( "os" "testing" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) func TestScanner(t *testing.T) { diff --git a/replication/changesets.go b/replication/changesets.go index f149c8cf..f9132a59 100644 --- a/replication/changesets.go +++ b/replication/changesets.go @@ -10,8 +10,8 @@ import ( "net/http" "strconv" - "github.com/paulmach/osm" - "github.com/paulmach/osm/osmxml" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/osmxml" ) // ChangesetSeqNum indicates the sequence of the changeset replication found here: diff --git a/replication/interval.go b/replication/interval.go index 8c258bd8..0193688a 100644 --- a/replication/interval.go +++ b/replication/interval.go @@ -11,7 +11,7 @@ import ( "strconv" "time" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) var _ SeqNum = MinuteSeqNum(0) diff --git a/testdata/compare_scanners.go b/testdata/compare_scanners.go index f019dd7c..7ab9aeff 100644 --- a/testdata/compare_scanners.go +++ b/testdata/compare_scanners.go @@ -8,9 +8,9 @@ import ( "os" "reflect" - "github.com/paulmach/osm" - "github.com/paulmach/osm/osmpbf" - "github.com/paulmach/osm/osmxml" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/osmpbf" + "github.com/nextmv-io/osm/osmxml" ) func main() { From 3b71c2a7bc01fd75510f8b07b93e0f1745ce1953 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Fri, 1 Sep 2023 20:15:18 +0200 Subject: [PATCH 03/22] Use getters for default values --- osmpbf/encode.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/osmpbf/encode.go b/osmpbf/encode.go index a65cba54..8268b4f1 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -222,11 +222,11 @@ func encode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, osm func EncodeDenseNode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, groupDense *osmpbf.DenseNodes, current, previous *osm.Node) { groupDense.Id = append(groupDense.Id, int64(current.ID-previous.ID)) // cast block.Granularity to int64 - granularity := int64(*block.Granularity) - currentLat := EncodeLatLon(current.Lat, int64(*block.LatOffset), granularity) - currentLon := EncodeLatLon(current.Lon, int64(*block.LonOffset), granularity) - previousLat := EncodeLatLon(previous.Lat, int64(*block.LatOffset), granularity) - previousLon := EncodeLatLon(previous.Lon, int64(*block.LonOffset), granularity) + granularity := block.GetGranularity() + currentLat := EncodeLatLon(current.Lat, block.GetLatOffset(), granularity) + currentLon := EncodeLatLon(current.Lon, block.GetLonOffset(), granularity) + previousLat := EncodeLatLon(previous.Lat, block.GetLatOffset(), granularity) + previousLon := EncodeLatLon(previous.Lon, block.GetLonOffset(), granularity) latDiff := currentLat - previousLat lonDiff := currentLon - previousLon groupDense.Lat = append(groupDense.Lat, latDiff) @@ -242,7 +242,7 @@ func EncodeDenseNode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string if groupDense.Denseinfo != nil { groupDense.Denseinfo.Changeset = append(groupDense.Denseinfo.Changeset, int64(current.ChangesetID-previous.ChangesetID)) - dateGranularity := int64(*block.DateGranularity) + dateGranularity := block.GetDateGranularity() currentTimeStamp := EncodeTimestamp(current.Timestamp, dateGranularity) previousTimeStamp := EncodeTimestamp(previous.Timestamp, dateGranularity) groupDense.Denseinfo.Timestamp = append(groupDense.Denseinfo.Timestamp, currentTimeStamp-previousTimeStamp) @@ -269,7 +269,7 @@ func EncodeNode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, pbfNode.Info.Changeset = &changesetId } if !node.Timestamp.IsZero() { - timeStamp := EncodeTimestamp(node.Timestamp, int64(*block.DateGranularity)) + timeStamp := EncodeTimestamp(node.Timestamp, block.GetDateGranularity()) pbfNode.Info.Timestamp = &timeStamp } if node.UserID != 0 { @@ -282,9 +282,9 @@ func EncodeNode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, nodeVersion := int32(node.Version) pbfNode.Info.Version = &nodeVersion } - lat := EncodeLatLon(node.Lat, *block.LatOffset, int64(*block.Granularity)) + lat := EncodeLatLon(node.Lat, *block.LatOffset, block.GetGranularity()) pbfNode.Lat = &lat - lon := EncodeLatLon(node.Lon, *block.LonOffset, int64(*block.Granularity)) + lon := EncodeLatLon(node.Lon, *block.LonOffset, block.GetGranularity()) pbfNode.Lon = &lon if len(node.Tags) > 0 { @@ -308,7 +308,7 @@ func EncodeWay(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, pbfWay.Info.Changeset = (*int64)(&way.ChangesetID) } if !way.Timestamp.IsZero() { - timestamp := EncodeTimestamp(way.Timestamp, int64(*block.DateGranularity)) + timestamp := EncodeTimestamp(way.Timestamp, block.GetDateGranularity()) pbfWay.Info.Timestamp = ×tamp } if way.UserID != 0 { @@ -346,7 +346,7 @@ func EncodeRelation(block *osmpbf.PrimitiveBlock, reverseStringTable map[string] }, } if !relation.Timestamp.IsZero() { - timestamp := EncodeTimestamp(relation.Timestamp, int64(*block.DateGranularity)) + timestamp := EncodeTimestamp(relation.Timestamp, block.GetDateGranularity()) pbfRelation.Info.Timestamp = ×tamp } if relation.UserID != 0 { @@ -392,12 +392,12 @@ func EncodeRelation(block *osmpbf.PrimitiveBlock, reverseStringTable map[string] return pbfRelation } -func EncodeLatLon(value float64, offset, granularity int64) int64 { - return (int64(value/.000000001) - offset) / granularity +func EncodeLatLon(value float64, offset int64, granularity int32) int64 { + return (int64(value/.000000001) - offset) / int64(granularity) } -func EncodeTimestamp(timestamp time.Time, dateGranularity int64) int64 { - return timestamp.Unix() / dateGranularity +func EncodeTimestamp(timestamp time.Time, dateGranularity int32) int64 { + return timestamp.Unix() / int64(dateGranularity) } func EncodeString(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, value string) int32 { From a6a7508424e0d7bcc273832132ffe54e319c0ff6 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Fri, 1 Sep 2023 21:02:58 +0200 Subject: [PATCH 04/22] Write correct osmType to blob --- osmpbf/encode.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/osmpbf/encode.go b/osmpbf/encode.go index 8268b4f1..39b360fb 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -32,7 +32,7 @@ func NewWriter(w io.Writer) (Writer, error) { return nil, err } - _, err = encoder.write(blockHeaderData) + _, err = encoder.write(blockHeaderData, osmHeaderType) if err != nil { return nil, err } @@ -67,11 +67,11 @@ func (e *encoder) flush() error { if err != nil { return nil } - _, err = e.write(blockData) + _, err = e.write(blockData, osmDataType) return err } -func (e *encoder) write(data []byte) (n int, err error) { +func (e *encoder) write(data []byte, osmType string) (n int, err error) { blob := &osmpbf.Blob{} blob.RawSize = proto.Int32(int32(len(data))) if e.compress { @@ -92,7 +92,7 @@ func (e *encoder) write(data []byte) (n int, err error) { blobHeader := &osmpbf.BlobHeader{ Datasize: proto.Int32(int32(len(blobData))), - Type: proto.String(osmDataType), + Type: proto.String(osmType), } blobHeaderData, err := proto.Marshal(blobHeader) From 86cc279a91cc0caaa0dca2478e6c961f71e3ef37 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Fri, 1 Sep 2023 21:23:39 +0200 Subject: [PATCH 05/22] Writing some info about header sizes --- osmpbf/encode.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/osmpbf/encode.go b/osmpbf/encode.go index 39b360fb..4eec7cd2 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -2,6 +2,7 @@ package osmpbf import ( "bytes" + "fmt" "io" "time" @@ -74,6 +75,7 @@ func (e *encoder) flush() error { func (e *encoder) write(data []byte, osmType string) (n int, err error) { blob := &osmpbf.Blob{} blob.RawSize = proto.Int32(int32(len(data))) + fmt.Println("blob.RawSize", blob.RawSize) if e.compress { target := &bytes.Buffer{} // compress the data @@ -95,6 +97,8 @@ func (e *encoder) write(data []byte, osmType string) (n int, err error) { Type: proto.String(osmType), } + fmt.Println("blobHeader.Datasize", blobHeader.Datasize) + blobHeaderData, err := proto.Marshal(blobHeader) if err != nil { return 0, nil From 53ca2374122e0fc6bf63963decb640f77e1b1c1c Mon Sep 17 00:00:00 2001 From: larsbeck Date: Fri, 1 Sep 2023 21:27:22 +0200 Subject: [PATCH 06/22] More output --- osmpbf/encode.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/osmpbf/encode.go b/osmpbf/encode.go index 4eec7cd2..115c0816 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -33,11 +33,15 @@ func NewWriter(w io.Writer) (Writer, error) { return nil, err } - _, err = encoder.write(blockHeaderData, osmHeaderType) + fmt.Println("blockHeaderData:", len(blockHeaderData)) + + n, err := encoder.write(blockHeaderData, osmHeaderType) if err != nil { return nil, err } + fmt.Println("n:", n) + return encoder, nil } @@ -75,7 +79,6 @@ func (e *encoder) flush() error { func (e *encoder) write(data []byte, osmType string) (n int, err error) { blob := &osmpbf.Blob{} blob.RawSize = proto.Int32(int32(len(data))) - fmt.Println("blob.RawSize", blob.RawSize) if e.compress { target := &bytes.Buffer{} // compress the data @@ -97,8 +100,6 @@ func (e *encoder) write(data []byte, osmType string) (n int, err error) { Type: proto.String(osmType), } - fmt.Println("blobHeader.Datasize", blobHeader.Datasize) - blobHeaderData, err := proto.Marshal(blobHeader) if err != nil { return 0, nil From 99a5385d4cfb7db35dc39896d036e0022762e2ec Mon Sep 17 00:00:00 2001 From: larsbeck Date: Fri, 1 Sep 2023 22:16:52 +0200 Subject: [PATCH 07/22] Write size as binary.BigEndian --- osmpbf/encode.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/osmpbf/encode.go b/osmpbf/encode.go index 115c0816..d748d8b7 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -2,7 +2,7 @@ package osmpbf import ( "bytes" - "fmt" + "encoding/binary" "io" "time" @@ -33,15 +33,11 @@ func NewWriter(w io.Writer) (Writer, error) { return nil, err } - fmt.Println("blockHeaderData:", len(blockHeaderData)) - - n, err := encoder.write(blockHeaderData, osmHeaderType) + _, err = encoder.write(blockHeaderData, osmHeaderType) if err != nil { return nil, err } - fmt.Println("n:", n) - return encoder, nil } @@ -105,6 +101,13 @@ func (e *encoder) write(data []byte, osmType string) (n int, err error) { return 0, nil } + size := make([]byte, 4) + binary.BigEndian.PutUint32(size, uint32(len(blobHeaderData))) + + if _, err = e.stream.Write(size); err != nil { + return 0, nil + } + if _, err = e.stream.Write(blobHeaderData); err != nil { return 0, nil } From 833d1d481f52c9de202dde783e88fbb5f1983c3c Mon Sep 17 00:00:00 2001 From: larsbeck Date: Sat, 2 Sep 2023 12:37:16 +0200 Subject: [PATCH 08/22] Add encoder tests --- osmpbf/encode.go | 4 +- osmpbf/encode_test.go | 227 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 229 insertions(+), 2 deletions(-) create mode 100644 osmpbf/encode_test.go diff --git a/osmpbf/encode.go b/osmpbf/encode.go index d748d8b7..21dbe410 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -92,8 +92,8 @@ func (e *encoder) write(data []byte, osmType string) (n int, err error) { } blobHeader := &osmpbf.BlobHeader{ - Datasize: proto.Int32(int32(len(blobData))), Type: proto.String(osmType), + Datasize: proto.Int32(int32(len(blobData))), } blobHeaderData, err := proto.Marshal(blobHeader) @@ -405,7 +405,7 @@ func EncodeLatLon(value float64, offset int64, granularity int32) int64 { } func EncodeTimestamp(timestamp time.Time, dateGranularity int32) int64 { - return timestamp.Unix() / int64(dateGranularity) + return timestamp.UnixMilli() / int64(dateGranularity) } func EncodeString(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, value string) int32 { diff --git a/osmpbf/encode_test.go b/osmpbf/encode_test.go new file mode 100644 index 00000000..7f0a4846 --- /dev/null +++ b/osmpbf/encode_test.go @@ -0,0 +1,227 @@ +package osmpbf + +import ( + "bytes" + "context" + "fmt" + "io" + "runtime" + "testing" + + "github.com/nextmv-io/osm" +) + +func TestEncodeDecode(t *testing.T) { + buffer := bytes.Buffer{} + writer, err := NewWriter(&buffer) + if err != nil { + t.Fatal(err) + } + + // write node + err = writer.WriteObject(en) + if err != nil { + t.Fatal(err) + } + // write way + err = writer.WriteObject(ew) + if err != nil { + t.Fatal(err) + } + // write relation + err = writer.WriteObject(er) + if err != nil { + t.Fatal(err) + } + writer.Close() + + d := newDecoder(context.Background(), &Scanner{}, &buffer) + err = d.Start(runtime.GOMAXPROCS(-1)) + if err != nil { + t.Fatal(err) + } + + for { + e, err := d.Next() + + if err == io.EOF { + break + } else if err != nil { + t.Fatal(err) + } + + switch v := e.(type) { + case *osm.Node: + err = nodeEquals(en, v) + if err != nil { + t.Fatal(err) + } + case *osm.Way: + err = wayEquals(ew, v) + if err != nil { + t.Fatal(err) + } + case *osm.Relation: + err = relationEquals(er, v) + if err != nil { + t.Fatal(err) + } + } + } + d.Close() +} + +func nodeEquals(en, node *osm.Node) error { + if node.ID != en.ID { + return fmt.Errorf("node id mismatch: %d != %d", node.ID, en.ID) + } + if node.Lat != en.Lat { + return fmt.Errorf("node lat mismatch: %f != %f", node.Lat, en.Lat) + } + if node.Lon != en.Lon { + return fmt.Errorf("node lon mismatch: %f != %f", node.Lon, en.Lon) + } + if node.User != en.User { + return fmt.Errorf("node user mismatch: %s != %s", node.User, en.User) + } + if node.UserID != en.UserID { + return fmt.Errorf("node user id mismatch: %d != %d", node.UserID, en.UserID) + } + if node.Visible != en.Visible { + return fmt.Errorf("node visible mismatch: %v != %v", node.Visible, en.Visible) + } + if node.Version != en.Version { + return fmt.Errorf("node version mismatch: %d != %d", node.Version, en.Version) + } + if node.ChangesetID != en.ChangesetID { + return fmt.Errorf("node changeset id mismatch: %d != %d", node.ChangesetID, en.ChangesetID) + } + // TODO: this currently fails with + // "node timestamp mismatch: 2224-09-20 11:45:12.871345152 +0000 UTC != + // 2009-05-20 10:28:54 +0000 UTC" + // The other timestamps (for ways and relations) work fine. + // if node.Timestamp != en.Timestamp { + // return fmt.Errorf("node timestamp mismatch: %s != %s", node.Timestamp, en.Timestamp) + // } + if len(node.Tags) != len(en.Tags) { + return fmt.Errorf("node tags length mismatch: %d != %d", len(node.Tags), len(en.Tags)) + } + for k, v := range node.Tags { + if en.Tags[k] != v { + return fmt.Errorf("node tag mismatch: %s != %s", en.Tags[k], v) + } + } + return nil +} + +func wayEquals(ew, way *osm.Way) error { + if way.ID != ew.ID { + return fmt.Errorf("way id mismatch: %d != %d", way.ID, ew.ID) + } + if way.User != ew.User { + return fmt.Errorf("way user mismatch: %s != %s", way.User, ew.User) + } + if way.UserID != ew.UserID { + return fmt.Errorf("way user id mismatch: %d != %d", way.UserID, ew.UserID) + } + if way.Visible != ew.Visible { + return fmt.Errorf("way visible mismatch: %v != %v", way.Visible, ew.Visible) + } + if way.Version != ew.Version { + return fmt.Errorf("way version mismatch: %d != %d", way.Version, ew.Version) + } + if way.ChangesetID != ew.ChangesetID { + return fmt.Errorf("way changeset id mismatch: %d != %d", way.ChangesetID, ew.ChangesetID) + } + if way.Timestamp != ew.Timestamp { + return fmt.Errorf("way timestamp mismatch: %s != %s", way.Timestamp, ew.Timestamp) + } + if len(way.Tags) != len(ew.Tags) { + return fmt.Errorf("way tags length mismatch: %d != %d", len(way.Tags), len(ew.Tags)) + } + for k, v := range way.Tags { + if ew.Tags[k] != v { + return fmt.Errorf("way tag mismatch: %s != %s", ew.Tags[k], v) + } + } + if len(way.Nodes) != len(ew.Nodes) { + return fmt.Errorf("way nodes length mismatch: %d != %d", len(way.Nodes), len(ew.Nodes)) + } + for i, v := range way.Nodes { + if ew.Nodes[i] != v { + return fmt.Errorf("way node mismatch: %v != %v", ew.Nodes[i], v) + } + } + return nil +} + +func relationEquals(er, relation *osm.Relation) error { + if relation.ID != er.ID { + return fmt.Errorf("relation id mismatch: %d != %d", relation.ID, er.ID) + } + if relation.User != er.User { + return fmt.Errorf("relation user mismatch: %s != %s", relation.User, er.User) + } + if relation.UserID != er.UserID { + return fmt.Errorf("relation user id mismatch: %d != %d", relation.UserID, er.UserID) + } + if relation.Visible != er.Visible { + return fmt.Errorf("relation visible mismatch: %v != %v", relation.Visible, er.Visible) + } + if relation.Version != er.Version { + return fmt.Errorf("relation version mismatch: %d != %d", relation.Version, er.Version) + } + if relation.ChangesetID != er.ChangesetID { + return fmt.Errorf("relation changeset id mismatch: %d != %d", relation.ChangesetID, er.ChangesetID) + } + if relation.Timestamp != er.Timestamp { + return fmt.Errorf("relation timestamp mismatch: %s != %s", relation.Timestamp, er.Timestamp) + } + if len(relation.Tags) != len(er.Tags) { + return fmt.Errorf("relation tags length mismatch: %d != %d", len(relation.Tags), len(er.Tags)) + } + for k, v := range relation.Tags { + if er.Tags[k] != v { + return fmt.Errorf("relation tag mismatch: %s != %s", er.Tags[k], v) + } + } + if len(relation.Members) != len(er.Members) { + return fmt.Errorf("relation members length mismatch: %d != %d", len(relation.Members), len(er.Members)) + } + for i, member := range relation.Members { + expectedMember := er.Members[i] + if member.ChangesetID != expectedMember.ChangesetID { + return fmt.Errorf("relation member changeset id mismatch: %d != %d", member.ChangesetID, expectedMember.ChangesetID) + } + if member.Role != expectedMember.Role { + return fmt.Errorf("relation member role mismatch: %s != %s", member.Role, expectedMember.Role) + } + if member.Type != expectedMember.Type { + return fmt.Errorf("relation member type mismatch: %s != %s", member.Type, expectedMember.Type) + } + if member.Ref != expectedMember.Ref { + return fmt.Errorf("relation member ref mismatch: %d != %d", member.Ref, expectedMember.Ref) + } + if member.Lat != expectedMember.Lat { + return fmt.Errorf("relation member lat mismatch: %f != %f", member.Lat, expectedMember.Lat) + } + if member.Lon != expectedMember.Lon { + return fmt.Errorf("relation member lon mismatch: %f != %f", member.Lon, expectedMember.Lon) + } + if member.Orientation != expectedMember.Orientation { + return fmt.Errorf("relation member orientation mismatch: %d != %d", member.Orientation, expectedMember.Orientation) + } + if member.Version != expectedMember.Version { + return fmt.Errorf("relation member version mismatch: %d != %d", member.Version, expectedMember.Version) + } + if len(member.Nodes) != len(expectedMember.Nodes) { + return fmt.Errorf("relation member nodes length mismatch: %d != %d", len(member.Nodes), len(expectedMember.Nodes)) + } + for j, node := range member.Nodes { + if expectedMember.Nodes[j] != node { + return fmt.Errorf("relation member node mismatch: %v != %v", expectedMember.Nodes[j], node) + } + } + } + return nil +} From 7589006ff8d9cb4a21850d6462020f12ba7e5b18 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Sat, 2 Sep 2023 16:14:08 +0200 Subject: [PATCH 09/22] Protect write by lock --- osmpbf/encode.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/osmpbf/encode.go b/osmpbf/encode.go index 21dbe410..a02b4c9a 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/binary" "io" + "sync" "time" "github.com/nextmv-io/osm" @@ -45,6 +46,7 @@ type encoder struct { stream io.Writer reverseStringTable map[string]int entities []osm.Object + mu sync.Mutex compress bool } @@ -73,6 +75,8 @@ func (e *encoder) flush() error { } func (e *encoder) write(data []byte, osmType string) (n int, err error) { + e.mu.Lock() + defer e.mu.Unlock() blob := &osmpbf.Blob{} blob.RawSize = proto.Int32(int32(len(data))) if e.compress { From b51b739e117463fa7f5b0c500bb780a7d5807ce4 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Sat, 2 Sep 2023 16:16:16 +0200 Subject: [PATCH 10/22] Protect WriteObject by lock --- osmpbf/encode.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/osmpbf/encode.go b/osmpbf/encode.go index a02b4c9a..9904e02e 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -75,8 +75,6 @@ func (e *encoder) flush() error { } func (e *encoder) write(data []byte, osmType string) (n int, err error) { - e.mu.Lock() - defer e.mu.Unlock() blob := &osmpbf.Blob{} blob.RawSize = proto.Int32(int32(len(data))) if e.compress { @@ -120,6 +118,8 @@ func (e *encoder) write(data []byte, osmType string) (n int, err error) { } func (e *encoder) WriteObject(obj osm.Object) error { + e.mu.Lock() + defer e.mu.Unlock() e.entities = append(e.entities, obj) if len(e.entities) >= 8000 { return e.flush() From e559d412273f491e580cb8baaf0d8bd71ab45c67 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Sat, 2 Sep 2023 17:42:15 +0200 Subject: [PATCH 11/22] Flush if entity type changed --- osmpbf/encode.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/osmpbf/encode.go b/osmpbf/encode.go index 9904e02e..efaa1abb 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -47,6 +47,7 @@ type encoder struct { reverseStringTable map[string]int entities []osm.Object mu sync.Mutex + lastWrittenType osm.Type compress bool } @@ -120,6 +121,14 @@ func (e *encoder) write(data []byte, osmType string) (n int, err error) { func (e *encoder) WriteObject(obj osm.Object) error { e.mu.Lock() defer e.mu.Unlock() + + if e.lastWrittenType != obj.ObjectID().Type() { + if err := e.flush(); err != nil { + return err + } + e.lastWrittenType = obj.ObjectID().Type() + } + e.entities = append(e.entities, obj) if len(e.entities) >= 8000 { return e.flush() From fc1139b5f8f9cada0397c44fadb406a03bad5748 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Sat, 2 Sep 2023 18:25:38 +0200 Subject: [PATCH 12/22] Deactivate tags and denseinfo --- osmpbf/encode.go | 50 ++++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/osmpbf/encode.go b/osmpbf/encode.go index efaa1abb..bcbeb84b 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -253,29 +253,29 @@ func EncodeDenseNode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string groupDense.Lat = append(groupDense.Lat, latDiff) groupDense.Lon = append(groupDense.Lon, lonDiff) - if current.Tags != nil { - for _, nodeTag := range current.Tags { - groupDense.KeysVals = append(groupDense.KeysVals, EncodeString(block, reverseStringTable, nodeTag.Key)) - groupDense.KeysVals = append(groupDense.KeysVals, EncodeString(block, reverseStringTable, nodeTag.Value)) - } - groupDense.KeysVals = append(groupDense.KeysVals, 0) - } - - if groupDense.Denseinfo != nil { - groupDense.Denseinfo.Changeset = append(groupDense.Denseinfo.Changeset, int64(current.ChangesetID-previous.ChangesetID)) - dateGranularity := block.GetDateGranularity() - currentTimeStamp := EncodeTimestamp(current.Timestamp, dateGranularity) - previousTimeStamp := EncodeTimestamp(previous.Timestamp, dateGranularity) - groupDense.Denseinfo.Timestamp = append(groupDense.Denseinfo.Timestamp, currentTimeStamp-previousTimeStamp) - groupDense.Denseinfo.Uid = append(groupDense.Denseinfo.Uid, int32(current.UserID-previous.UserID)) - groupDense.Denseinfo.Version = append(groupDense.Denseinfo.Version, int32(current.Version-previous.Version)) - var previousUserNameId int32 = 0 - if previous.User != "" { - previousUserNameId = EncodeString(block, reverseStringTable, previous.User) - } - currentUserNameId := EncodeString(block, reverseStringTable, current.User) - groupDense.Denseinfo.UserSid = append(groupDense.Denseinfo.UserSid, currentUserNameId-previousUserNameId) - } + // if current.Tags != nil { + // for _, nodeTag := range current.Tags { + // groupDense.KeysVals = append(groupDense.KeysVals, EncodeString(block, reverseStringTable, nodeTag.Key)) + // groupDense.KeysVals = append(groupDense.KeysVals, EncodeString(block, reverseStringTable, nodeTag.Value)) + // } + // groupDense.KeysVals = append(groupDense.KeysVals, 0) + // } + + // if groupDense.Denseinfo != nil { + // groupDense.Denseinfo.Changeset = append(groupDense.Denseinfo.Changeset, int64(current.ChangesetID-previous.ChangesetID)) + // dateGranularity := block.GetDateGranularity() + // currentTimeStamp := EncodeTimestamp(current.Timestamp, dateGranularity) + // previousTimeStamp := EncodeTimestamp(previous.Timestamp, dateGranularity) + // groupDense.Denseinfo.Timestamp = append(groupDense.Denseinfo.Timestamp, currentTimeStamp-previousTimeStamp) + // groupDense.Denseinfo.Uid = append(groupDense.Denseinfo.Uid, int32(current.UserID-previous.UserID)) + // groupDense.Denseinfo.Version = append(groupDense.Denseinfo.Version, int32(current.Version-previous.Version)) + // var previousUserNameId int32 = 0 + // if previous.User != "" { + // previousUserNameId = EncodeString(block, reverseStringTable, previous.User) + // } + // currentUserNameId := EncodeString(block, reverseStringTable, current.User) + // groupDense.Denseinfo.UserSid = append(groupDense.Denseinfo.UserSid, currentUserNameId-previousUserNameId) + // } } func EncodeNode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, pbfNode *osmpbf.Node, node *osm.Node) *osmpbf.Node { @@ -303,9 +303,9 @@ func EncodeNode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, nodeVersion := int32(node.Version) pbfNode.Info.Version = &nodeVersion } - lat := EncodeLatLon(node.Lat, *block.LatOffset, block.GetGranularity()) + lat := EncodeLatLon(node.Lat, block.GetLatOffset(), block.GetGranularity()) pbfNode.Lat = &lat - lon := EncodeLatLon(node.Lon, *block.LonOffset, block.GetGranularity()) + lon := EncodeLatLon(node.Lon, block.GetLonOffset(), block.GetGranularity()) pbfNode.Lon = &lon if len(node.Tags) > 0 { From 80feaae93192e39cbe7aa72286c6131681b48039 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Sat, 2 Sep 2023 18:30:15 +0200 Subject: [PATCH 13/22] Readd tags --- osmpbf/encode.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/osmpbf/encode.go b/osmpbf/encode.go index bcbeb84b..5d4f892e 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -253,13 +253,13 @@ func EncodeDenseNode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string groupDense.Lat = append(groupDense.Lat, latDiff) groupDense.Lon = append(groupDense.Lon, lonDiff) - // if current.Tags != nil { - // for _, nodeTag := range current.Tags { - // groupDense.KeysVals = append(groupDense.KeysVals, EncodeString(block, reverseStringTable, nodeTag.Key)) - // groupDense.KeysVals = append(groupDense.KeysVals, EncodeString(block, reverseStringTable, nodeTag.Value)) - // } - // groupDense.KeysVals = append(groupDense.KeysVals, 0) - // } + if current.Tags != nil { + for _, nodeTag := range current.Tags { + groupDense.KeysVals = append(groupDense.KeysVals, EncodeString(block, reverseStringTable, nodeTag.Key)) + groupDense.KeysVals = append(groupDense.KeysVals, EncodeString(block, reverseStringTable, nodeTag.Value)) + } + groupDense.KeysVals = append(groupDense.KeysVals, 0) + } // if groupDense.Denseinfo != nil { // groupDense.Denseinfo.Changeset = append(groupDense.Denseinfo.Changeset, int64(current.ChangesetID-previous.ChangesetID)) From 768d6e38b249a022c23ea120980a4913d2084f1c Mon Sep 17 00:00:00 2001 From: larsbeck Date: Sat, 2 Sep 2023 18:33:03 +0200 Subject: [PATCH 14/22] Fix tags --- osmpbf/encode.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osmpbf/encode.go b/osmpbf/encode.go index 5d4f892e..ab3c247b 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -253,7 +253,7 @@ func EncodeDenseNode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string groupDense.Lat = append(groupDense.Lat, latDiff) groupDense.Lon = append(groupDense.Lon, lonDiff) - if current.Tags != nil { + if len(current.Tags) > 0 { for _, nodeTag := range current.Tags { groupDense.KeysVals = append(groupDense.KeysVals, EncodeString(block, reverseStringTable, nodeTag.Key)) groupDense.KeysVals = append(groupDense.KeysVals, EncodeString(block, reverseStringTable, nodeTag.Value)) From f47a97543f1a864761a2b800a61d411d354c8241 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Sat, 2 Sep 2023 18:33:52 +0200 Subject: [PATCH 15/22] Write 0 in any case --- osmpbf/encode.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osmpbf/encode.go b/osmpbf/encode.go index ab3c247b..217847f1 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -258,8 +258,8 @@ func EncodeDenseNode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string groupDense.KeysVals = append(groupDense.KeysVals, EncodeString(block, reverseStringTable, nodeTag.Key)) groupDense.KeysVals = append(groupDense.KeysVals, EncodeString(block, reverseStringTable, nodeTag.Value)) } - groupDense.KeysVals = append(groupDense.KeysVals, 0) } + groupDense.KeysVals = append(groupDense.KeysVals, 0) // if groupDense.Denseinfo != nil { // groupDense.Denseinfo.Changeset = append(groupDense.Denseinfo.Changeset, int64(current.ChangesetID-previous.ChangesetID)) From 42429ab8c9717fb67a9101ad7d6859b2e7f76e56 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Sat, 2 Sep 2023 18:35:38 +0200 Subject: [PATCH 16/22] Add denseinfo --- osmpbf/encode.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/osmpbf/encode.go b/osmpbf/encode.go index 217847f1..915b35c6 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -261,21 +261,21 @@ func EncodeDenseNode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string } groupDense.KeysVals = append(groupDense.KeysVals, 0) - // if groupDense.Denseinfo != nil { - // groupDense.Denseinfo.Changeset = append(groupDense.Denseinfo.Changeset, int64(current.ChangesetID-previous.ChangesetID)) - // dateGranularity := block.GetDateGranularity() - // currentTimeStamp := EncodeTimestamp(current.Timestamp, dateGranularity) - // previousTimeStamp := EncodeTimestamp(previous.Timestamp, dateGranularity) - // groupDense.Denseinfo.Timestamp = append(groupDense.Denseinfo.Timestamp, currentTimeStamp-previousTimeStamp) - // groupDense.Denseinfo.Uid = append(groupDense.Denseinfo.Uid, int32(current.UserID-previous.UserID)) - // groupDense.Denseinfo.Version = append(groupDense.Denseinfo.Version, int32(current.Version-previous.Version)) - // var previousUserNameId int32 = 0 - // if previous.User != "" { - // previousUserNameId = EncodeString(block, reverseStringTable, previous.User) - // } - // currentUserNameId := EncodeString(block, reverseStringTable, current.User) - // groupDense.Denseinfo.UserSid = append(groupDense.Denseinfo.UserSid, currentUserNameId-previousUserNameId) - // } + if groupDense.Denseinfo != nil { + groupDense.Denseinfo.Changeset = append(groupDense.Denseinfo.Changeset, int64(current.ChangesetID-previous.ChangesetID)) + dateGranularity := block.GetDateGranularity() + currentTimeStamp := EncodeTimestamp(current.Timestamp, dateGranularity) + previousTimeStamp := EncodeTimestamp(previous.Timestamp, dateGranularity) + groupDense.Denseinfo.Timestamp = append(groupDense.Denseinfo.Timestamp, currentTimeStamp-previousTimeStamp) + groupDense.Denseinfo.Uid = append(groupDense.Denseinfo.Uid, int32(current.UserID-previous.UserID)) + groupDense.Denseinfo.Version = append(groupDense.Denseinfo.Version, int32(current.Version-previous.Version)) + var previousUserNameId int32 = 0 + if previous.User != "" { + previousUserNameId = EncodeString(block, reverseStringTable, previous.User) + } + currentUserNameId := EncodeString(block, reverseStringTable, current.User) + groupDense.Denseinfo.UserSid = append(groupDense.Denseinfo.UserSid, currentUserNameId-previousUserNameId) + } } func EncodeNode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, pbfNode *osmpbf.Node, node *osm.Node) *osmpbf.Node { From fad147b7397ce3478ab3caf8a1e3441d86b189c6 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Sat, 2 Sep 2023 18:38:33 +0200 Subject: [PATCH 17/22] Rearrange fields --- osmpbf/encode.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osmpbf/encode.go b/osmpbf/encode.go index 915b35c6..db3de506 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -45,9 +45,9 @@ func NewWriter(w io.Writer) (Writer, error) { type encoder struct { stream io.Writer reverseStringTable map[string]int + lastWrittenType osm.Type entities []osm.Object mu sync.Mutex - lastWrittenType osm.Type compress bool } From 44f1f4ec7cce5f26ebdf8ae8eee8df83583ffd89 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Wed, 6 Sep 2023 16:07:41 +0200 Subject: [PATCH 18/22] Rename writer to encoder --- osmpbf/encode.go | 20 ++++++++++---------- osmpbf/encode_test.go | 8 ++++---- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/osmpbf/encode.go b/osmpbf/encode.go index db3de506..e869dc6e 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -12,16 +12,16 @@ import ( "google.golang.org/protobuf/proto" ) -// Writer is an interface for writing osm data. The format written is the osm +// Encoder is an interface for encoding osm data. The format written is the osm // pbf format. -type Writer interface { +type Encoder interface { io.Closer - WriteObject(obj osm.Object) error + Encode(obj osm.Object) error } -func NewWriter(w io.Writer) (Writer, error) { +func NewEncoder(w io.Writer) (Encoder, error) { encoder := &encoder{ - stream: w, + writer: w, reverseStringTable: make(map[string]int), compress: true, } @@ -43,7 +43,7 @@ func NewWriter(w io.Writer) (Writer, error) { } type encoder struct { - stream io.Writer + writer io.Writer reverseStringTable map[string]int lastWrittenType osm.Type entities []osm.Object @@ -107,18 +107,18 @@ func (e *encoder) write(data []byte, osmType string) (n int, err error) { size := make([]byte, 4) binary.BigEndian.PutUint32(size, uint32(len(blobHeaderData))) - if _, err = e.stream.Write(size); err != nil { + if _, err = e.writer.Write(size); err != nil { return 0, nil } - if _, err = e.stream.Write(blobHeaderData); err != nil { + if _, err = e.writer.Write(blobHeaderData); err != nil { return 0, nil } - return e.stream.Write(blobData) + return e.writer.Write(blobData) } -func (e *encoder) WriteObject(obj osm.Object) error { +func (e *encoder) Encode(obj osm.Object) error { e.mu.Lock() defer e.mu.Unlock() diff --git a/osmpbf/encode_test.go b/osmpbf/encode_test.go index 7f0a4846..c1089d51 100644 --- a/osmpbf/encode_test.go +++ b/osmpbf/encode_test.go @@ -13,23 +13,23 @@ import ( func TestEncodeDecode(t *testing.T) { buffer := bytes.Buffer{} - writer, err := NewWriter(&buffer) + writer, err := NewEncoder(&buffer) if err != nil { t.Fatal(err) } // write node - err = writer.WriteObject(en) + err = writer.Encode(en) if err != nil { t.Fatal(err) } // write way - err = writer.WriteObject(ew) + err = writer.Encode(ew) if err != nil { t.Fatal(err) } // write relation - err = writer.WriteObject(er) + err = writer.Encode(er) if err != nil { t.Fatal(err) } From 9eeab800374c8551126ba4e33fa7f9ff2910a881 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Mon, 13 Nov 2023 10:47:02 +0100 Subject: [PATCH 19/22] Add OSM encoder --- annotate/change.go | 4 ++-- annotate/change_test.go | 2 +- annotate/datasource.go | 6 +++--- annotate/edgecases_test.go | 2 +- annotate/errors.go | 4 ++-- annotate/errors_test.go | 2 +- annotate/geo.go | 4 ++-- annotate/geo_test.go | 2 +- annotate/internal/core/compute.go | 4 ++-- annotate/internal/core/compute_test.go | 4 ++-- annotate/internal/core/datasourcer_test.go | 2 +- annotate/internal/core/errors.go | 2 +- annotate/internal/core/types.go | 4 ++-- annotate/internal/core/types_test.go | 4 ++-- annotate/options.go | 4 ++-- annotate/order.go | 2 +- annotate/order_test.go | 2 +- annotate/relation.go | 6 +++--- annotate/relation_test.go | 2 +- annotate/shared/child.go | 2 +- annotate/way.go | 6 +++--- annotate/way_test.go | 2 +- go.mod | 2 +- internal/mputil/mputil.go | 2 +- internal/mputil/mputil_test.go | 2 +- json.go | 4 ++-- osmapi/changeset.go | 2 +- osmapi/datasource.go | 2 +- osmapi/live_test.go | 2 +- osmapi/map.go | 2 +- osmapi/map_test.go | 2 +- osmapi/node.go | 2 +- osmapi/node_test.go | 2 +- osmapi/note.go | 2 +- osmapi/note_test.go | 2 +- osmapi/relation.go | 2 +- osmapi/relation_test.go | 2 +- osmapi/user.go | 2 +- osmapi/way.go | 2 +- osmapi/way_test.go | 2 +- osmgeojson/benchmarks_test.go | 2 +- osmgeojson/build_polygon.go | 4 ++-- osmgeojson/build_polygon_test.go | 2 +- osmgeojson/convert.go | 4 ++-- osmgeojson/convert_test.go | 2 +- osmgeojson/options_test.go | 2 +- osmpbf/decode.go | 4 ++-- osmpbf/decode_data.go | 4 ++-- osmpbf/decode_test.go | 2 +- osmpbf/encode.go | 4 ++-- osmpbf/encode_test.go | 2 +- osmpbf/example_stats_test.go | 4 ++-- osmpbf/internal/osmpbf/generate.go | 2 +- osmpbf/scanner.go | 2 +- osmpbf/scanner_test.go | 2 +- osmtest/scanner.go | 2 +- osmtest/scanner_test.go | 2 +- osmxml/example_test.go | 4 ++-- osmxml/scanner.go | 2 +- osmxml/scanner_test.go | 2 +- replication/changesets.go | 4 ++-- replication/interval.go | 2 +- testdata/compare_scanners.go | 6 +++--- 63 files changed, 88 insertions(+), 88 deletions(-) diff --git a/annotate/change.go b/annotate/change.go index 421988a4..a011676e 100644 --- a/annotate/change.go +++ b/annotate/change.go @@ -3,8 +3,8 @@ package annotate import ( "context" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/annotate/internal/core" + "github.com/paulmach/osm" + "github.com/paulmach/osm/annotate/internal/core" ) // Change will annotate a change into a diff. It will use the diff --git a/annotate/change_test.go b/annotate/change_test.go index fc7a5e44..5706b589 100644 --- a/annotate/change_test.go +++ b/annotate/change_test.go @@ -4,7 +4,7 @@ import ( "context" "testing" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) func TestChange_create(t *testing.T) { diff --git a/annotate/datasource.go b/annotate/datasource.go index 0fdff5e6..41102f9a 100644 --- a/annotate/datasource.go +++ b/annotate/datasource.go @@ -3,9 +3,9 @@ package annotate import ( "context" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/annotate/internal/core" - "github.com/nextmv-io/osm/annotate/shared" + "github.com/paulmach/osm" + "github.com/paulmach/osm/annotate/internal/core" + "github.com/paulmach/osm/annotate/shared" "github.com/paulmach/orb" "github.com/paulmach/orb/planar" diff --git a/annotate/edgecases_test.go b/annotate/edgecases_test.go index b1e7a8a0..09c1310b 100644 --- a/annotate/edgecases_test.go +++ b/annotate/edgecases_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) func TestEdgeCase_childCreatedAfterParent(t *testing.T) { diff --git a/annotate/errors.go b/annotate/errors.go index 864c8562..13a7d08f 100644 --- a/annotate/errors.go +++ b/annotate/errors.go @@ -4,8 +4,8 @@ import ( "fmt" "time" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/annotate/internal/core" + "github.com/paulmach/osm" + "github.com/paulmach/osm/annotate/internal/core" ) // NoHistoryError is returned if there is no entry in the history diff --git a/annotate/errors_test.go b/annotate/errors_test.go index e4f69cea..a4ec64c8 100644 --- a/annotate/errors_test.go +++ b/annotate/errors_test.go @@ -4,7 +4,7 @@ import ( "errors" "testing" - "github.com/nextmv-io/osm/annotate/internal/core" + "github.com/paulmach/osm/annotate/internal/core" ) func TestMapErrors(t *testing.T) { diff --git a/annotate/geo.go b/annotate/geo.go index ae1ee69f..91ba7abd 100644 --- a/annotate/geo.go +++ b/annotate/geo.go @@ -4,10 +4,10 @@ import ( "math" "time" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/internal/mputil" "github.com/paulmach/orb" "github.com/paulmach/orb/geo" + "github.com/paulmach/osm" + "github.com/paulmach/osm/internal/mputil" ) func wayPointOnSurface(w *osm.Way) orb.Point { diff --git a/annotate/geo_test.go b/annotate/geo_test.go index 818db4ed..bc697d20 100644 --- a/annotate/geo_test.go +++ b/annotate/geo_test.go @@ -4,8 +4,8 @@ import ( "encoding/xml" "testing" - "github.com/nextmv-io/osm" "github.com/paulmach/orb" + "github.com/paulmach/osm" ) func TestWayPointOnSurface(t *testing.T) { diff --git a/annotate/internal/core/compute.go b/annotate/internal/core/compute.go index 46966623..531b815d 100644 --- a/annotate/internal/core/compute.go +++ b/annotate/internal/core/compute.go @@ -5,8 +5,8 @@ import ( "fmt" "time" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/annotate/shared" + "github.com/paulmach/osm" + "github.com/paulmach/osm/annotate/shared" ) // A Datasourcer is something that acts like a datasource allowing us to diff --git a/annotate/internal/core/compute_test.go b/annotate/internal/core/compute_test.go index 8d02f9ef..c8fe94e9 100644 --- a/annotate/internal/core/compute_test.go +++ b/annotate/internal/core/compute_test.go @@ -7,8 +7,8 @@ import ( "testing" "time" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/annotate/shared" + "github.com/paulmach/osm" + "github.com/paulmach/osm/annotate/shared" ) var ( diff --git a/annotate/internal/core/datasourcer_test.go b/annotate/internal/core/datasourcer_test.go index c522d183..4a2fb4aa 100644 --- a/annotate/internal/core/datasourcer_test.go +++ b/annotate/internal/core/datasourcer_test.go @@ -4,7 +4,7 @@ import ( "context" "errors" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) var _ Datasourcer = &TestDS{} diff --git a/annotate/internal/core/errors.go b/annotate/internal/core/errors.go index 2bba17a3..451e3ee7 100644 --- a/annotate/internal/core/errors.go +++ b/annotate/internal/core/errors.go @@ -4,7 +4,7 @@ import ( "fmt" "time" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) // NoHistoryError is returned if there is no entry in the history diff --git a/annotate/internal/core/types.go b/annotate/internal/core/types.go index 8fcc627e..87fb037d 100644 --- a/annotate/internal/core/types.go +++ b/annotate/internal/core/types.go @@ -3,8 +3,8 @@ package core import ( "time" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/annotate/shared" + "github.com/paulmach/osm" + "github.com/paulmach/osm/annotate/shared" ) // A Parent is something that holds children. ie. ways have nodes as children diff --git a/annotate/internal/core/types_test.go b/annotate/internal/core/types_test.go index 44754dcc..c5ef266a 100644 --- a/annotate/internal/core/types_test.go +++ b/annotate/internal/core/types_test.go @@ -4,8 +4,8 @@ import ( "testing" "time" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/annotate/shared" + "github.com/paulmach/osm" + "github.com/paulmach/osm/annotate/shared" ) type findVisibleTestCase struct { diff --git a/annotate/options.go b/annotate/options.go index 142aacca..4251de96 100644 --- a/annotate/options.go +++ b/annotate/options.go @@ -3,8 +3,8 @@ package annotate import ( "time" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/annotate/internal/core" + "github.com/paulmach/osm" + "github.com/paulmach/osm/annotate/internal/core" ) // Option is a parameter that can be used for annotating. diff --git a/annotate/order.go b/annotate/order.go index c324b803..022fe127 100644 --- a/annotate/order.go +++ b/annotate/order.go @@ -4,7 +4,7 @@ import ( "context" "sync" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) // RelationHistoryDatasourcer is an more strict interface for when we only need the relation history. diff --git a/annotate/order_test.go b/annotate/order_test.go index e29716c3..f9f7010f 100644 --- a/annotate/order_test.go +++ b/annotate/order_test.go @@ -4,7 +4,7 @@ import ( "context" "testing" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) func TestChildFirstOrdering(t *testing.T) { diff --git a/annotate/relation.go b/annotate/relation.go index 6de0d68d..e1fe2bbe 100644 --- a/annotate/relation.go +++ b/annotate/relation.go @@ -4,9 +4,9 @@ import ( "context" "time" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/annotate/internal/core" - "github.com/nextmv-io/osm/annotate/shared" + "github.com/paulmach/osm" + "github.com/paulmach/osm/annotate/internal/core" + "github.com/paulmach/osm/annotate/shared" ) // HistoryAsChildrenDatasourcer is an advanced data source that diff --git a/annotate/relation_test.go b/annotate/relation_test.go index 8eb2e41d..1619bbe6 100644 --- a/annotate/relation_test.go +++ b/annotate/relation_test.go @@ -9,8 +9,8 @@ import ( "testing" "time" - "github.com/nextmv-io/osm" "github.com/paulmach/orb" + "github.com/paulmach/osm" ) func TestRelation(t *testing.T) { diff --git a/annotate/shared/child.go b/annotate/shared/child.go index 0881f4e7..fba94709 100644 --- a/annotate/shared/child.go +++ b/annotate/shared/child.go @@ -5,7 +5,7 @@ package shared import ( "time" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) // A Child represents a node, way or relation that is a dependent for diff --git a/annotate/way.go b/annotate/way.go index cb08f045..7e0bc2eb 100644 --- a/annotate/way.go +++ b/annotate/way.go @@ -4,9 +4,9 @@ import ( "context" "time" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/annotate/internal/core" - "github.com/nextmv-io/osm/annotate/shared" + "github.com/paulmach/osm" + "github.com/paulmach/osm/annotate/internal/core" + "github.com/paulmach/osm/annotate/shared" ) // NodeHistoryDatasourcer is an more strict interface for when we only need node history. diff --git a/annotate/way_test.go b/annotate/way_test.go index 6dae0f63..47cfc3ec 100644 --- a/annotate/way_test.go +++ b/annotate/way_test.go @@ -8,7 +8,7 @@ import ( "reflect" "testing" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) func TestWays(t *testing.T) { diff --git a/go.mod b/go.mod index 53234598..5ea8c278 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/nextmv-io/osm +module github.com/paulmach/osm go 1.13 diff --git a/internal/mputil/mputil.go b/internal/mputil/mputil.go index a4719da3..3e8693de 100644 --- a/internal/mputil/mputil.go +++ b/internal/mputil/mputil.go @@ -3,8 +3,8 @@ package mputil import ( "time" - "github.com/nextmv-io/osm" "github.com/paulmach/orb" + "github.com/paulmach/osm" ) // Segment is a section of a multipolygon with some extra information diff --git a/internal/mputil/mputil_test.go b/internal/mputil/mputil_test.go index b6a7c3de..219573fa 100644 --- a/internal/mputil/mputil_test.go +++ b/internal/mputil/mputil_test.go @@ -5,8 +5,8 @@ import ( "testing" "time" - "github.com/nextmv-io/osm" "github.com/paulmach/orb" + "github.com/paulmach/osm" ) func TestMultiSegment_LineString(t *testing.T) { diff --git a/json.go b/json.go index 423931ce..593f8f9e 100644 --- a/json.go +++ b/json.go @@ -12,7 +12,7 @@ import ( // // import ( // jsoniter "github.com/json-iterator/go" -// "github.com/nextmv-io/osm" +// "github.com/paulmach/osm" // ) // // var c = jsoniter.Config{ @@ -36,7 +36,7 @@ var CustomJSONMarshaler interface { // // import ( // jsoniter "github.com/json-iterator/go" -// "github.com/nextmv-io/osm" +// "github.com/paulmach/osm" // ) // // var c = jsoniter.Config{ diff --git a/osmapi/changeset.go b/osmapi/changeset.go index 0610a3a0..959b4f4c 100644 --- a/osmapi/changeset.go +++ b/osmapi/changeset.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) // Changeset returns a given changeset from the osm rest api. diff --git a/osmapi/datasource.go b/osmapi/datasource.go index 3ad115fc..cc06a252 100644 --- a/osmapi/datasource.go +++ b/osmapi/datasource.go @@ -8,7 +8,7 @@ import ( "net/http" "time" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) // BaseURL defines the api host. This can be change to hit diff --git a/osmapi/live_test.go b/osmapi/live_test.go index 6d100dc3..f09070bb 100644 --- a/osmapi/live_test.go +++ b/osmapi/live_test.go @@ -5,7 +5,7 @@ import ( "os" "testing" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" "golang.org/x/time/rate" ) diff --git a/osmapi/map.go b/osmapi/map.go index 5ee833b8..d0b3a93b 100644 --- a/osmapi/map.go +++ b/osmapi/map.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) // Map returns the latest elements in the given bounding box. diff --git a/osmapi/map_test.go b/osmapi/map_test.go index b4f7bc44..c8292257 100644 --- a/osmapi/map_test.go +++ b/osmapi/map_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) func TestMap_urls(t *testing.T) { diff --git a/osmapi/node.go b/osmapi/node.go index 3a42d6db..13140c3b 100644 --- a/osmapi/node.go +++ b/osmapi/node.go @@ -5,7 +5,7 @@ import ( "fmt" "strconv" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) // Node returns the latest version of the node from the osm rest api. diff --git a/osmapi/node_test.go b/osmapi/node_test.go index 93254627..12b541f4 100644 --- a/osmapi/node_test.go +++ b/osmapi/node_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) func TestNode_urls(t *testing.T) { diff --git a/osmapi/note.go b/osmapi/note.go index 72b718d0..c903e563 100644 --- a/osmapi/note.go +++ b/osmapi/note.go @@ -6,7 +6,7 @@ import ( "net/url" "strings" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) // Note returns the note from the osm rest api. diff --git a/osmapi/note_test.go b/osmapi/note_test.go index 3d193f5f..ec93f432 100644 --- a/osmapi/note_test.go +++ b/osmapi/note_test.go @@ -7,7 +7,7 @@ import ( "strings" "testing" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) func TestNote_urls(t *testing.T) { diff --git a/osmapi/relation.go b/osmapi/relation.go index 5e767c70..e629e0bc 100644 --- a/osmapi/relation.go +++ b/osmapi/relation.go @@ -5,7 +5,7 @@ import ( "fmt" "strconv" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) // Relation returns the latest version of the relation from the osm rest api. diff --git a/osmapi/relation_test.go b/osmapi/relation_test.go index b64cbc37..bea89341 100644 --- a/osmapi/relation_test.go +++ b/osmapi/relation_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) func TestRelation_urls(t *testing.T) { diff --git a/osmapi/user.go b/osmapi/user.go index 3b01ce81..17a7bff5 100644 --- a/osmapi/user.go +++ b/osmapi/user.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) // User returns the user from the osm rest api. diff --git a/osmapi/way.go b/osmapi/way.go index 9d75d2d1..5f834fd0 100644 --- a/osmapi/way.go +++ b/osmapi/way.go @@ -5,7 +5,7 @@ import ( "fmt" "strconv" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) // Way returns the latest version of the way from the osm rest api. diff --git a/osmapi/way_test.go b/osmapi/way_test.go index 6377415f..050357ee 100644 --- a/osmapi/way_test.go +++ b/osmapi/way_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) func TestWay_urls(t *testing.T) { diff --git a/osmgeojson/benchmarks_test.go b/osmgeojson/benchmarks_test.go index 42cf0a3f..1dc71dd7 100644 --- a/osmgeojson/benchmarks_test.go +++ b/osmgeojson/benchmarks_test.go @@ -5,7 +5,7 @@ import ( "io/ioutil" "testing" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) func BenchmarkConvert(b *testing.B) { diff --git a/osmgeojson/build_polygon.go b/osmgeojson/build_polygon.go index 45c5721b..bc00284d 100644 --- a/osmgeojson/build_polygon.go +++ b/osmgeojson/build_polygon.go @@ -3,10 +3,10 @@ package osmgeojson import ( "fmt" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/internal/mputil" "github.com/paulmach/orb" "github.com/paulmach/orb/geojson" + "github.com/paulmach/osm" + "github.com/paulmach/osm/internal/mputil" ) func (ctx *context) buildPolygon(relation *osm.Relation) *geojson.Feature { diff --git a/osmgeojson/build_polygon_test.go b/osmgeojson/build_polygon_test.go index fc684b88..d80750c9 100644 --- a/osmgeojson/build_polygon_test.go +++ b/osmgeojson/build_polygon_test.go @@ -4,9 +4,9 @@ import ( "encoding/xml" "testing" - "github.com/nextmv-io/osm" "github.com/paulmach/orb" "github.com/paulmach/orb/geojson" + "github.com/paulmach/osm" ) func TestConvert_polygon(t *testing.T) { diff --git a/osmgeojson/convert.go b/osmgeojson/convert.go index e0c8cf19..ddc2013b 100644 --- a/osmgeojson/convert.go +++ b/osmgeojson/convert.go @@ -3,10 +3,10 @@ package osmgeojson import ( "fmt" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/internal/mputil" "github.com/paulmach/orb" "github.com/paulmach/orb/geojson" + "github.com/paulmach/osm" + "github.com/paulmach/osm/internal/mputil" ) type context struct { diff --git a/osmgeojson/convert_test.go b/osmgeojson/convert_test.go index fbee4bf5..3209d0ec 100644 --- a/osmgeojson/convert_test.go +++ b/osmgeojson/convert_test.go @@ -6,9 +6,9 @@ import ( "reflect" "testing" - "github.com/nextmv-io/osm" "github.com/paulmach/orb" "github.com/paulmach/orb/geojson" + "github.com/paulmach/osm" ) func TestConvert(t *testing.T) { diff --git a/osmgeojson/options_test.go b/osmgeojson/options_test.go index 80148247..2d74b964 100644 --- a/osmgeojson/options_test.go +++ b/osmgeojson/options_test.go @@ -4,8 +4,8 @@ import ( "encoding/xml" "testing" - "github.com/nextmv-io/osm" "github.com/paulmach/orb/geojson" + "github.com/paulmach/osm" ) var nodeXML = ` diff --git a/osmpbf/decode.go b/osmpbf/decode.go index 54473990..eec3892a 100644 --- a/osmpbf/decode.go +++ b/osmpbf/decode.go @@ -10,8 +10,8 @@ import ( "sync" "time" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/osmpbf/internal/osmpbf" + "github.com/paulmach/osm" + "github.com/paulmach/osm/osmpbf/internal/osmpbf" "google.golang.org/protobuf/proto" ) diff --git a/osmpbf/decode_data.go b/osmpbf/decode_data.go index ba782721..1d652b38 100644 --- a/osmpbf/decode_data.go +++ b/osmpbf/decode_data.go @@ -4,8 +4,8 @@ import ( "errors" "time" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/osmpbf/internal/osmpbf" + "github.com/paulmach/osm" + "github.com/paulmach/osm/osmpbf/internal/osmpbf" "github.com/paulmach/protoscan" "google.golang.org/protobuf/proto" ) diff --git a/osmpbf/decode_test.go b/osmpbf/decode_test.go index 8a4a8698..6b6785c6 100644 --- a/osmpbf/decode_test.go +++ b/osmpbf/decode_test.go @@ -12,7 +12,7 @@ import ( "testing" "time" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) const ( diff --git a/osmpbf/encode.go b/osmpbf/encode.go index e869dc6e..9a32ea37 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -7,8 +7,8 @@ import ( "sync" "time" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/osmpbf/internal/osmpbf" + "github.com/paulmach/osm" + "github.com/paulmach/osm/osmpbf/internal/osmpbf" "google.golang.org/protobuf/proto" ) diff --git a/osmpbf/encode_test.go b/osmpbf/encode_test.go index c1089d51..8f1e86a4 100644 --- a/osmpbf/encode_test.go +++ b/osmpbf/encode_test.go @@ -8,7 +8,7 @@ import ( "runtime" "testing" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) func TestEncodeDecode(t *testing.T) { diff --git a/osmpbf/example_stats_test.go b/osmpbf/example_stats_test.go index b9d7a049..9a8182f9 100644 --- a/osmpbf/example_stats_test.go +++ b/osmpbf/example_stats_test.go @@ -7,8 +7,8 @@ import ( "os" "time" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/osmpbf" + "github.com/paulmach/osm" + "github.com/paulmach/osm/osmpbf" ) // ExampleStats demonstrates how to read a full file and gather some stats. diff --git a/osmpbf/internal/osmpbf/generate.go b/osmpbf/internal/osmpbf/generate.go index a330ef3f..8eee6f9b 100644 --- a/osmpbf/internal/osmpbf/generate.go +++ b/osmpbf/internal/osmpbf/generate.go @@ -1,3 +1,3 @@ package osmpbf -//go:generate protoc --proto_path=. --go_opt=module=github.com/nextmv-io/osmpbf/internal/osmpbf --go_out=. fileformat.proto osmformat.proto +//go:generate protoc --proto_path=. --go_opt=module=github.com/paulmach/osmpbf/internal/osmpbf --go_out=. fileformat.proto osmformat.proto diff --git a/osmpbf/scanner.go b/osmpbf/scanner.go index 201b7a43..b0479029 100644 --- a/osmpbf/scanner.go +++ b/osmpbf/scanner.go @@ -5,7 +5,7 @@ import ( "io" "sync/atomic" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) var _ osm.Scanner = &Scanner{} diff --git a/osmpbf/scanner_test.go b/osmpbf/scanner_test.go index 70d9e6db..fd47c082 100644 --- a/osmpbf/scanner_test.go +++ b/osmpbf/scanner_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) var ( diff --git a/osmtest/scanner.go b/osmtest/scanner.go index 3a35188e..1119c865 100644 --- a/osmtest/scanner.go +++ b/osmtest/scanner.go @@ -1,6 +1,6 @@ package osmtest -import "github.com/nextmv-io/osm" +import "github.com/paulmach/osm" // Scanner implements the osm.Scanner interface with // just a list of objects. diff --git a/osmtest/scanner_test.go b/osmtest/scanner_test.go index fff9f7e6..f3c68207 100644 --- a/osmtest/scanner_test.go +++ b/osmtest/scanner_test.go @@ -5,7 +5,7 @@ import ( "reflect" "testing" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) func TestScanner(t *testing.T) { diff --git a/osmxml/example_test.go b/osmxml/example_test.go index 37979075..61700dca 100644 --- a/osmxml/example_test.go +++ b/osmxml/example_test.go @@ -5,8 +5,8 @@ import ( "fmt" "os" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/osmxml" + "github.com/paulmach/osm" + "github.com/paulmach/osm/osmxml" ) func ExampleScanner() { diff --git a/osmxml/scanner.go b/osmxml/scanner.go index c0c91926..bdd0efe3 100644 --- a/osmxml/scanner.go +++ b/osmxml/scanner.go @@ -6,7 +6,7 @@ import ( "io" "strings" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) var _ osm.Scanner = &Scanner{} diff --git a/osmxml/scanner_test.go b/osmxml/scanner_test.go index daf7dc65..fd85aa7a 100644 --- a/osmxml/scanner_test.go +++ b/osmxml/scanner_test.go @@ -8,7 +8,7 @@ import ( "os" "testing" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) func TestScanner(t *testing.T) { diff --git a/replication/changesets.go b/replication/changesets.go index f9132a59..f149c8cf 100644 --- a/replication/changesets.go +++ b/replication/changesets.go @@ -10,8 +10,8 @@ import ( "net/http" "strconv" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/osmxml" + "github.com/paulmach/osm" + "github.com/paulmach/osm/osmxml" ) // ChangesetSeqNum indicates the sequence of the changeset replication found here: diff --git a/replication/interval.go b/replication/interval.go index 0193688a..8c258bd8 100644 --- a/replication/interval.go +++ b/replication/interval.go @@ -11,7 +11,7 @@ import ( "strconv" "time" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) var _ SeqNum = MinuteSeqNum(0) diff --git a/testdata/compare_scanners.go b/testdata/compare_scanners.go index 7ab9aeff..f019dd7c 100644 --- a/testdata/compare_scanners.go +++ b/testdata/compare_scanners.go @@ -8,9 +8,9 @@ import ( "os" "reflect" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/osmpbf" - "github.com/nextmv-io/osm/osmxml" + "github.com/paulmach/osm" + "github.com/paulmach/osm/osmpbf" + "github.com/paulmach/osm/osmxml" ) func main() { From 48c0be54b812cd4365231b500dab94449ed5fc42 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Mon, 13 Nov 2023 10:52:23 +0100 Subject: [PATCH 20/22] Revert auto-formatting related changes --- annotate/options.go | 18 +++++++++--------- osmapi/datasource.go | 4 ++-- osmpbf/decode_data.go | 2 +- osmxml/scanner.go | 1 - 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/annotate/options.go b/annotate/options.go index 4251de96..789e19da 100644 --- a/annotate/options.go +++ b/annotate/options.go @@ -28,15 +28,15 @@ func Threshold(t time.Duration) Option { // This should be used when you want to gracefully handle the weird data in OSM. // // Nodes with unclear/inconsistent data will not be annotated. Causes include: -// - redacted data: In 2012 due to the license change data had to be removed. -// This could be some nodes of a way. There exist ways for which some nodes have -// just a single delete version, e.g. way 159081205, node 376130526 -// - data pre element versioning: pre-2012(?) data versions were not kept, so -// for old ways there many be no information about some nodes. For example, -// a node may be updated after a way and there is no way to get the original -// version of the node and way. -// - bad editors: sometimes a node is edited 7 times in a single changeset -// and version 5 is a delete. See node 321452894, part of way 28831147. +// - redacted data: In 2012 due to the license change data had to be removed. +// This could be some nodes of a way. There exist ways for which some nodes have +// just a single delete version, e.g. way 159081205, node 376130526 +// - data pre element versioning: pre-2012(?) data versions were not kept, so +// for old ways there many be no information about some nodes. For example, +// a node may be updated after a way and there is no way to get the original +// version of the node and way. +// - bad editors: sometimes a node is edited 7 times in a single changeset +// and version 5 is a delete. See node 321452894, part of way 28831147. func IgnoreInconsistency(yes bool) Option { return func(o *core.Options) error { o.IgnoreInconsistency = yes diff --git a/osmapi/datasource.go b/osmapi/datasource.go index cc06a252..28685832 100644 --- a/osmapi/datasource.go +++ b/osmapi/datasource.go @@ -19,8 +19,8 @@ const BaseURL = "http://api.openstreetmap.org/api/0.6" // This interface is met by `golang.org/x/time/rate.Limiter` and is meant // to be used with it. For example: // -// // 10 qps -// osmapi.DefaultDatasource.Limiter = rate.NewLimiter(10, 1) +// // 10 qps +// osmapi.DefaultDatasource.Limiter = rate.NewLimiter(10, 1) type RateLimiter interface { Wait(context.Context) error } diff --git a/osmpbf/decode_data.go b/osmpbf/decode_data.go index 1d652b38..1847a4ba 100644 --- a/osmpbf/decode_data.go +++ b/osmpbf/decode_data.go @@ -4,10 +4,10 @@ import ( "errors" "time" + "google.golang.org/protobuf/proto" "github.com/paulmach/osm" "github.com/paulmach/osm/osmpbf/internal/osmpbf" "github.com/paulmach/protoscan" - "google.golang.org/protobuf/proto" ) // dataDecoder is a decoder for Blob with OSMData (PrimitiveBlock). diff --git a/osmxml/scanner.go b/osmxml/scanner.go index bdd0efe3..b232d357 100644 --- a/osmxml/scanner.go +++ b/osmxml/scanner.go @@ -126,7 +126,6 @@ Loop: // Object returns the most recent token generated by a call to Scan // as a new osm.Object. This interface is implemented by: -// // *osm.Bounds // *osm.Node // *osm.Way From 73659be68233ded0ca0b7b65735acb45692f3d5b Mon Sep 17 00:00:00 2001 From: larsbeck Date: Mon, 13 Nov 2023 10:53:20 +0100 Subject: [PATCH 21/22] Remove auto-formatting newline comment --- osmapi/datasource.go | 1 - 1 file changed, 1 deletion(-) diff --git a/osmapi/datasource.go b/osmapi/datasource.go index 28685832..223e3b34 100644 --- a/osmapi/datasource.go +++ b/osmapi/datasource.go @@ -18,7 +18,6 @@ const BaseURL = "http://api.openstreetmap.org/api/0.6" // A RateLimiter is something that can wait until its next allowed request. // This interface is met by `golang.org/x/time/rate.Limiter` and is meant // to be used with it. For example: -// // // 10 qps // osmapi.DefaultDatasource.Limiter = rate.NewLimiter(10, 1) type RateLimiter interface { From af1fec1a55b1e303f94aec27164b15100dde4d16 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Mon, 6 May 2024 10:58:25 +0200 Subject: [PATCH 22/22] Fix delta encoding of a node's timestamp on DenseInfo field --- osmpbf/encode.go | 18 ++++++++++++++---- osmpbf/encode_test.go | 10 +++------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/osmpbf/encode.go b/osmpbf/encode.go index 9a32ea37..796213b7 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -264,9 +264,15 @@ func EncodeDenseNode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string if groupDense.Denseinfo != nil { groupDense.Denseinfo.Changeset = append(groupDense.Denseinfo.Changeset, int64(current.ChangesetID-previous.ChangesetID)) dateGranularity := block.GetDateGranularity() - currentTimeStamp := EncodeTimestamp(current.Timestamp, dateGranularity) - previousTimeStamp := EncodeTimestamp(previous.Timestamp, dateGranularity) - groupDense.Denseinfo.Timestamp = append(groupDense.Denseinfo.Timestamp, currentTimeStamp-previousTimeStamp) + if !current.Timestamp.IsZero() { + currentTimeStamp := EncodeTimestamp(current.Timestamp, dateGranularity) + if !previous.Timestamp.IsZero() { + previousTimeStamp := EncodeTimestamp(previous.Timestamp, dateGranularity) + groupDense.Denseinfo.Timestamp = append(groupDense.Denseinfo.Timestamp, currentTimeStamp-previousTimeStamp) + } else { + groupDense.Denseinfo.Timestamp = append(groupDense.Denseinfo.Timestamp, currentTimeStamp) + } + } groupDense.Denseinfo.Uid = append(groupDense.Denseinfo.Uid, int32(current.UserID-previous.UserID)) groupDense.Denseinfo.Version = append(groupDense.Denseinfo.Version, int32(current.Version-previous.Version)) var previousUserNameId int32 = 0 @@ -418,7 +424,11 @@ func EncodeLatLon(value float64, offset int64, granularity int32) int64 { } func EncodeTimestamp(timestamp time.Time, dateGranularity int32) int64 { - return timestamp.UnixMilli() / int64(dateGranularity) + ts := timestamp.UnixMilli() / int64(dateGranularity) + if ts < 0 { + return 0 + } + return ts } func EncodeString(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, value string) int32 { diff --git a/osmpbf/encode_test.go b/osmpbf/encode_test.go index 8f1e86a4..578944a7 100644 --- a/osmpbf/encode_test.go +++ b/osmpbf/encode_test.go @@ -96,13 +96,9 @@ func nodeEquals(en, node *osm.Node) error { if node.ChangesetID != en.ChangesetID { return fmt.Errorf("node changeset id mismatch: %d != %d", node.ChangesetID, en.ChangesetID) } - // TODO: this currently fails with - // "node timestamp mismatch: 2224-09-20 11:45:12.871345152 +0000 UTC != - // 2009-05-20 10:28:54 +0000 UTC" - // The other timestamps (for ways and relations) work fine. - // if node.Timestamp != en.Timestamp { - // return fmt.Errorf("node timestamp mismatch: %s != %s", node.Timestamp, en.Timestamp) - // } + if node.Timestamp != en.Timestamp { + return fmt.Errorf("node timestamp mismatch: %s != %s", node.Timestamp, en.Timestamp) + } if len(node.Tags) != len(en.Tags) { return fmt.Errorf("node tags length mismatch: %d != %d", len(node.Tags), len(en.Tags)) }