From d6e6e2e0bf28a2040ebc1a42aef96ab9b74e5af9 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Fri, 1 Sep 2023 19:42:43 +0200 Subject: [PATCH 01/29] Add initial encoder implementation --- osmpbf/encode.go | 422 +++++++++++++++++++++++++++++++++++++++++++++ osmpbf/zlib_cgo.go | 5 + osmpbf/zlib_go.go | 5 + 3 files changed, 432 insertions(+) create mode 100644 osmpbf/encode.go diff --git a/osmpbf/encode.go b/osmpbf/encode.go new file mode 100644 index 00000000..81d28b07 --- /dev/null +++ b/osmpbf/encode.go @@ -0,0 +1,422 @@ +package osmpbf + +import ( + "bytes" + "io" + "time" + + "github.com/paulmach/osm" + "github.com/paulmach/osm/osmpbf/internal/osmpbf" + "google.golang.org/protobuf/proto" +) + +// Writer is an interface for writing osm data. The format written is the osm +// pbf format. +type Writer interface { + io.Closer + WriteObject(obj osm.Object) error +} + +func NewWriter(w io.Writer) (Writer, error) { + encoder := &encoder{ + stream: w, + reverseStringTable: make(map[string]int), + compress: true, + } + + blockHeader := &osmpbf.HeaderBlock{ + RequiredFeatures: []string{"OsmSchema-V0.6", "DenseNodes"}, + } + blockHeaderData, err := proto.Marshal(blockHeader) + if err != nil { + return nil, err + } + + _, err = encoder.write(blockHeaderData) + if err != nil { + return nil, err + } + + return encoder, nil +} + +type encoder struct { + stream io.Writer + reverseStringTable map[string]int + entities []osm.Object + compress bool +} + +func (e *encoder) Close() error { + return e.flush() +} + +func (e *encoder) flush() error { + if len(e.entities) == 0 { + return nil + } + + block := &osmpbf.PrimitiveBlock{} + encode(block, e.reverseStringTable, e.entities, e.compress) + e.entities = e.entities[:0] + for k := range e.reverseStringTable { + delete(e.reverseStringTable, k) + } + + blockData, err := proto.Marshal(block) + if err != nil { + return nil + } + _, err = e.write(blockData) + return err +} + +func (e *encoder) write(data []byte) (n int, err error) { + blob := &osmpbf.Blob{} + blob.RawSize = proto.Int32(int32(len(data))) + if e.compress { + target := &bytes.Buffer{} + // compress the data + writer := zlibWriter(target) + writer.Write(data) + writer.Close() + blob.ZlibData = target.Bytes() + } else { + blob.Raw = data + } + + blobData, err := proto.Marshal(blob) + if err != nil { + return 0, nil + } + + blobHeader := &osmpbf.BlobHeader{ + Datasize: proto.Int32(int32(len(blobData))), + Type: proto.String(osmDataType), + } + + blobHeaderData, err := proto.Marshal(blobHeader) + if err != nil { + return 0, nil + } + + if _, err = e.stream.Write(blobHeaderData); err != nil { + return 0, nil + } + + return e.stream.Write(blobData) +} + +func (e *encoder) WriteObject(obj osm.Object) error { + e.entities = append(e.entities, obj) + if len(e.entities) >= 8000 { + return e.flush() + } + return nil +} + +func encode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, osmGeos []osm.Object, compressed bool) { + groupIdx := 0 + i := 0 + + if block.Stringtable != nil && block.Stringtable.S != nil { + block.Stringtable.S = nil + } + + for i < len(osmGeos) { + var group *osmpbf.PrimitiveGroup = nil + if groupIdx < len(block.Primitivegroup) { + group = block.Primitivegroup[groupIdx] + if group == nil { + group = &osmpbf.PrimitiveGroup{} + } + if group.Dense != nil { + if group.Dense.Denseinfo != nil { + if group.Dense.Denseinfo.Changeset != nil { + group.Dense.Denseinfo.Changeset = nil + } + if group.Dense.Denseinfo.Timestamp != nil { + group.Dense.Denseinfo.Timestamp = nil + } + if group.Dense.Denseinfo.Uid != nil { + group.Dense.Denseinfo.Uid = nil + } + if group.Dense.Denseinfo.UserSid != nil { + group.Dense.Denseinfo.UserSid = nil + } + if group.Dense.Denseinfo.Version != nil { + group.Dense.Denseinfo.Version = nil + } + } + if group.Dense.Id != nil { + group.Dense.Id = nil + } + if group.Dense.KeysVals != nil { + group.Dense.KeysVals = nil + } + if group.Dense.Lat != nil { + group.Dense.Lat = nil + } + if group.Dense.Lon != nil { + group.Dense.Lon = nil + } + } + if group.Changesets != nil { + group.Changesets = nil + } + if group.Ways != nil { + group.Ways = nil + } + if group.Relations != nil { + group.Relations = nil + } + } else { + group = &osmpbf.PrimitiveGroup{} + block.Primitivegroup = append(block.Primitivegroup, group) + } + + currentNodeCount := 0 + groupType := osmGeos[i].ObjectID().Type() + previousNode := &osm.Node{} + if groupType == osm.TypeNode && compressed && group.Dense == nil { + group.Dense = &osmpbf.DenseNodes{Denseinfo: &osmpbf.DenseInfo{}} + } + for i < len(osmGeos) && osmGeos[i].ObjectID().Type() == groupType { + switch groupType { + case osm.TypeNode: + if compressed { + currentNode := osmGeos[i].(*osm.Node) + EncodeDenseNode(block, reverseStringTable, group.Dense, currentNode, previousNode) + previousNode = currentNode + } else { + if currentNodeCount < len(group.Nodes) { + EncodeNode(block, reverseStringTable, group.Nodes[currentNodeCount], osmGeos[i].(*osm.Node)) + } else { + pbfNode := &osmpbf.Node{} + group.Nodes = append(group.Nodes, EncodeNode(block, reverseStringTable, pbfNode, osmGeos[i].(*osm.Node))) + } + currentNodeCount++ + } + case osm.TypeWay: + group.Ways = append(group.Ways, EncodeWay(block, reverseStringTable, osmGeos[i].(*osm.Way))) + case osm.TypeRelation: + group.Relations = append(group.Relations, EncodeRelation(block, reverseStringTable, osmGeos[i].(*osm.Relation))) + } + i++ + } + + if group.Nodes != nil { + for currentNodeCount < len(group.Nodes) { + group.Nodes = group.Nodes[:len(group.Nodes)-1] + } + } + + groupIdx++ + } + + if groupIdx < len(block.Primitivegroup) { + block.Primitivegroup = block.Primitivegroup[:groupIdx] + } +} + +func EncodeDenseNode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, groupDense *osmpbf.DenseNodes, current, previous *osm.Node) { + groupDense.Id = append(groupDense.Id, int64(current.ID-previous.ID)) + // cast block.Granularity to int64 + granularity := int64(*block.Granularity) + currentLat := EncodeLatLon(current.Lat, int64(*block.LatOffset), granularity) + currentLon := EncodeLatLon(current.Lon, int64(*block.LonOffset), granularity) + previousLat := EncodeLatLon(previous.Lat, int64(*block.LatOffset), granularity) + previousLon := EncodeLatLon(previous.Lon, int64(*block.LonOffset), granularity) + latDiff := currentLat - previousLat + lonDiff := currentLon - previousLon + groupDense.Lat = append(groupDense.Lat, latDiff) + groupDense.Lon = append(groupDense.Lon, lonDiff) + + if current.Tags != nil { + for _, nodeTag := range current.Tags { + groupDense.KeysVals = append(groupDense.KeysVals, EncodeString(block, reverseStringTable, nodeTag.Key)) + groupDense.KeysVals = append(groupDense.KeysVals, EncodeString(block, reverseStringTable, nodeTag.Value)) + } + groupDense.KeysVals = append(groupDense.KeysVals, 0) + } + + if groupDense.Denseinfo != nil { + groupDense.Denseinfo.Changeset = append(groupDense.Denseinfo.Changeset, int64(current.ChangesetID-previous.ChangesetID)) + dateGranularity := int64(*block.DateGranularity) + currentTimeStamp := EncodeTimestamp(current.Timestamp, dateGranularity) + previousTimeStamp := EncodeTimestamp(previous.Timestamp, dateGranularity) + groupDense.Denseinfo.Timestamp = append(groupDense.Denseinfo.Timestamp, currentTimeStamp-previousTimeStamp) + groupDense.Denseinfo.Uid = append(groupDense.Denseinfo.Uid, int32(current.UserID-previous.UserID)) + groupDense.Denseinfo.Version = append(groupDense.Denseinfo.Version, int32(current.Version-previous.Version)) + var previousUserNameId int32 = 0 + if previous.User != "" { + previousUserNameId = EncodeString(block, reverseStringTable, previous.User) + } + currentUserNameId := EncodeString(block, reverseStringTable, current.User) + groupDense.Denseinfo.UserSid = append(groupDense.Denseinfo.UserSid, currentUserNameId-previousUserNameId) + } +} + +func EncodeNode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, pbfNode *osmpbf.Node, node *osm.Node) *osmpbf.Node { + castId := int64(node.ID) + pbfNode.Id = &castId + zero := int32(0) + pbfNode.Info = &osmpbf.Info{ + Version: &zero, + } + if node.ChangesetID != 0 { + changesetId := int64(node.ChangesetID) + pbfNode.Info.Changeset = &changesetId + } + if !node.Timestamp.IsZero() { + timeStamp := EncodeTimestamp(node.Timestamp, int64(*block.DateGranularity)) + pbfNode.Info.Timestamp = &timeStamp + } + if node.UserID != 0 { + userId := int32(node.UserID) + pbfNode.Info.Uid = &userId + } + userSid := uint32(EncodeString(block, reverseStringTable, node.User)) + pbfNode.Info.UserSid = &userSid + if node.Version != 0 { + nodeVersion := int32(node.Version) + pbfNode.Info.Version = &nodeVersion + } + lat := EncodeLatLon(node.Lat, *block.LatOffset, int64(*block.Granularity)) + pbfNode.Lat = &lat + lon := EncodeLatLon(node.Lon, *block.LonOffset, int64(*block.Granularity)) + pbfNode.Lon = &lon + + if len(node.Tags) > 0 { + for _, tag := range node.Tags { + pbfNode.Keys = append(pbfNode.Keys, uint32(EncodeString(block, reverseStringTable, tag.Key))) + pbfNode.Vals = append(pbfNode.Vals, uint32(EncodeString(block, reverseStringTable, tag.Value))) + } + } else { + pbfNode.Keys = nil + pbfNode.Vals = nil + } + return pbfNode +} + +func EncodeWay(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, way *osm.Way) *osmpbf.Way { + pbfWay := &osmpbf.Way{ + Id: (*int64)(&way.ID), + Info: &osmpbf.Info{}, + } + if way.ChangesetID != 0 { + pbfWay.Info.Changeset = (*int64)(&way.ChangesetID) + } + if !way.Timestamp.IsZero() { + timestamp := EncodeTimestamp(way.Timestamp, int64(*block.DateGranularity)) + pbfWay.Info.Timestamp = ×tamp + } + if way.UserID != 0 { + userId := int32(way.UserID) + pbfWay.Info.Uid = &userId + } + userId := uint32(EncodeString(block, reverseStringTable, way.User)) + pbfWay.Info.UserSid = &userId + if way.Version != 0 { + wayVersion := int32(way.Version) + pbfWay.Info.Version = &wayVersion + } + + if len(way.Tags) > 0 { + for _, tag := range way.Tags { + pbfWay.Keys = append(pbfWay.Keys, uint32(EncodeString(block, reverseStringTable, tag.Key))) + pbfWay.Vals = append(pbfWay.Vals, uint32(EncodeString(block, reverseStringTable, tag.Value))) + } + } + + if len(way.Nodes) > 0 { + pbfWay.Refs = append(pbfWay.Refs, int64(way.Nodes[0].ID)) + for i := 1; i < len(way.Nodes); i++ { + pbfWay.Refs = append(pbfWay.Refs, int64(way.Nodes[i].ID)-int64(way.Nodes[i-1].ID)) + } + } + return pbfWay +} + +func EncodeRelation(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, relation *osm.Relation) *osmpbf.Relation { + pbfRelation := &osmpbf.Relation{ + Id: (*int64)(&relation.ID), + Info: &osmpbf.Info{ + Changeset: (*int64)(&relation.ChangesetID), + }, + } + if !relation.Timestamp.IsZero() { + timestamp := EncodeTimestamp(relation.Timestamp, int64(*block.DateGranularity)) + pbfRelation.Info.Timestamp = ×tamp + } + if relation.UserID != 0 { + userId := int32(relation.UserID) + pbfRelation.Info.Uid = &userId + } + userSid := uint32(EncodeString(block, reverseStringTable, relation.User)) + pbfRelation.Info.UserSid = &userSid + relationVersion := int32(relation.Version) + pbfRelation.Info.Version = &relationVersion + + if len(relation.Tags) > 0 { + for _, tag := range relation.Tags { + pbfRelation.Keys = append(pbfRelation.Keys, uint32(EncodeString(block, reverseStringTable, tag.Key))) + pbfRelation.Vals = append(pbfRelation.Vals, uint32(EncodeString(block, reverseStringTable, tag.Value))) + } + } + + if len(relation.Members) > 0 { + pbfRelation.Memids = append(pbfRelation.Memids, relation.Members[0].Ref) + pbfRelation.RolesSid = append(pbfRelation.RolesSid, EncodeString(block, reverseStringTable, relation.Members[0].Role)) + switch relation.Members[0].Type { + case osm.TypeNode: + pbfRelation.Types = append(pbfRelation.Types, osmpbf.Relation_NODE) + case osm.TypeWay: + pbfRelation.Types = append(pbfRelation.Types, osmpbf.Relation_WAY) + case osm.TypeRelation: + pbfRelation.Types = append(pbfRelation.Types, osmpbf.Relation_RELATION) + } + for i := 1; i < len(relation.Members); i++ { + pbfRelation.Memids = append(pbfRelation.Memids, relation.Members[i].Ref-relation.Members[i-1].Ref) + pbfRelation.RolesSid = append(pbfRelation.RolesSid, EncodeString(block, reverseStringTable, relation.Members[i].Role)) + switch relation.Members[i].Type { + case osm.TypeNode: + pbfRelation.Types = append(pbfRelation.Types, osmpbf.Relation_NODE) + case osm.TypeWay: + pbfRelation.Types = append(pbfRelation.Types, osmpbf.Relation_WAY) + case osm.TypeRelation: + pbfRelation.Types = append(pbfRelation.Types, osmpbf.Relation_RELATION) + } + } + } + return pbfRelation +} + +func EncodeLatLon(value float64, offset, granularity int64) int64 { + return (int64(value/.000000001) - offset) / granularity +} + +func EncodeTimestamp(timestamp time.Time, dateGranularity int64) int64 { + return timestamp.Unix() / dateGranularity +} + +func EncodeString(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, value string) int32 { + if value == "" { + return 0 + } + + if block.Stringtable == nil { + block.Stringtable = &osmpbf.StringTable{} + block.Stringtable.S = append(block.Stringtable.S, "") + reverseStringTable[""] = 0 + } + + if id, ok := reverseStringTable[value]; ok { + return int32(id) + } + + block.Stringtable.S = append(block.Stringtable.S, value) + id := len(block.Stringtable.S) - 1 + reverseStringTable[value] = id + return int32(id) +} diff --git a/osmpbf/zlib_cgo.go b/osmpbf/zlib_cgo.go index fbcf3f66..b6aaa949 100644 --- a/osmpbf/zlib_cgo.go +++ b/osmpbf/zlib_cgo.go @@ -1,3 +1,4 @@ +//go:build cgo // +build cgo package osmpbf @@ -12,3 +13,7 @@ import ( func zlibReader(data []byte) (io.ReadCloser, error) { return czlib.NewReader(bytes.NewReader(data)) } + +func zlibWriter(w io.Writer) io.WriteCloser { + return czlib.NewWriter(w) +} diff --git a/osmpbf/zlib_go.go b/osmpbf/zlib_go.go index ddae9b86..6b7b657a 100644 --- a/osmpbf/zlib_go.go +++ b/osmpbf/zlib_go.go @@ -1,3 +1,4 @@ +//go:build !cgo // +build !cgo package osmpbf @@ -11,3 +12,7 @@ import ( func zlibReader(data []byte) (io.ReadCloser, error) { return zlib.NewReader(bytes.NewReader(data)) } + +func zlibWriter(w io.Writer) io.WriteCloser { + return zlib.NewWriter(w) +} From c2da18ded1fd55e80eb733340684eda2a6ae32e1 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Fri, 1 Sep 2023 19:45:16 +0200 Subject: [PATCH 02/29] Fix module references --- annotate/change.go | 4 ++-- annotate/change_test.go | 2 +- annotate/datasource.go | 6 +++--- annotate/edgecases_test.go | 2 +- annotate/errors.go | 4 ++-- annotate/errors_test.go | 2 +- annotate/geo.go | 4 ++-- annotate/geo_test.go | 2 +- annotate/internal/core/compute.go | 4 ++-- annotate/internal/core/compute_test.go | 4 ++-- annotate/internal/core/datasourcer_test.go | 2 +- annotate/internal/core/errors.go | 2 +- annotate/internal/core/types.go | 4 ++-- annotate/internal/core/types_test.go | 4 ++-- annotate/options.go | 22 +++++++++++----------- annotate/order.go | 2 +- annotate/order_test.go | 2 +- annotate/relation.go | 6 +++--- annotate/relation_test.go | 2 +- annotate/shared/child.go | 2 +- annotate/way.go | 6 +++--- annotate/way_test.go | 2 +- go.mod | 2 +- internal/mputil/mputil.go | 2 +- internal/mputil/mputil_test.go | 2 +- json.go | 4 ++-- osmapi/changeset.go | 2 +- osmapi/datasource.go | 7 ++++--- osmapi/live_test.go | 2 +- osmapi/map.go | 2 +- osmapi/map_test.go | 2 +- osmapi/node.go | 2 +- osmapi/node_test.go | 2 +- osmapi/note.go | 2 +- osmapi/note_test.go | 2 +- osmapi/relation.go | 2 +- osmapi/relation_test.go | 2 +- osmapi/user.go | 2 +- osmapi/way.go | 2 +- osmapi/way_test.go | 2 +- osmgeojson/benchmarks_test.go | 2 +- osmgeojson/build_polygon.go | 4 ++-- osmgeojson/build_polygon_test.go | 2 +- osmgeojson/convert.go | 4 ++-- osmgeojson/convert_test.go | 2 +- osmgeojson/options_test.go | 2 +- osmpbf/decode.go | 4 ++-- osmpbf/decode_data.go | 6 +++--- osmpbf/decode_test.go | 2 +- osmpbf/encode.go | 4 ++-- osmpbf/example_stats_test.go | 4 ++-- osmpbf/internal/osmpbf/generate.go | 2 +- osmpbf/scanner.go | 2 +- osmpbf/scanner_test.go | 2 +- osmtest/scanner.go | 2 +- osmtest/scanner_test.go | 2 +- osmxml/example_test.go | 4 ++-- osmxml/scanner.go | 3 ++- osmxml/scanner_test.go | 2 +- replication/changesets.go | 4 ++-- replication/interval.go | 2 +- testdata/compare_scanners.go | 6 +++--- 62 files changed, 101 insertions(+), 99 deletions(-) diff --git a/annotate/change.go b/annotate/change.go index a011676e..421988a4 100644 --- a/annotate/change.go +++ b/annotate/change.go @@ -3,8 +3,8 @@ package annotate import ( "context" - "github.com/paulmach/osm" - "github.com/paulmach/osm/annotate/internal/core" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/annotate/internal/core" ) // Change will annotate a change into a diff. It will use the diff --git a/annotate/change_test.go b/annotate/change_test.go index 5706b589..fc7a5e44 100644 --- a/annotate/change_test.go +++ b/annotate/change_test.go @@ -4,7 +4,7 @@ import ( "context" "testing" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) func TestChange_create(t *testing.T) { diff --git a/annotate/datasource.go b/annotate/datasource.go index 41102f9a..0fdff5e6 100644 --- a/annotate/datasource.go +++ b/annotate/datasource.go @@ -3,9 +3,9 @@ package annotate import ( "context" - "github.com/paulmach/osm" - "github.com/paulmach/osm/annotate/internal/core" - "github.com/paulmach/osm/annotate/shared" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/annotate/internal/core" + "github.com/nextmv-io/osm/annotate/shared" "github.com/paulmach/orb" "github.com/paulmach/orb/planar" diff --git a/annotate/edgecases_test.go b/annotate/edgecases_test.go index 09c1310b..b1e7a8a0 100644 --- a/annotate/edgecases_test.go +++ b/annotate/edgecases_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) func TestEdgeCase_childCreatedAfterParent(t *testing.T) { diff --git a/annotate/errors.go b/annotate/errors.go index 13a7d08f..864c8562 100644 --- a/annotate/errors.go +++ b/annotate/errors.go @@ -4,8 +4,8 @@ import ( "fmt" "time" - "github.com/paulmach/osm" - "github.com/paulmach/osm/annotate/internal/core" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/annotate/internal/core" ) // NoHistoryError is returned if there is no entry in the history diff --git a/annotate/errors_test.go b/annotate/errors_test.go index a4ec64c8..e4f69cea 100644 --- a/annotate/errors_test.go +++ b/annotate/errors_test.go @@ -4,7 +4,7 @@ import ( "errors" "testing" - "github.com/paulmach/osm/annotate/internal/core" + "github.com/nextmv-io/osm/annotate/internal/core" ) func TestMapErrors(t *testing.T) { diff --git a/annotate/geo.go b/annotate/geo.go index 91ba7abd..ae1ee69f 100644 --- a/annotate/geo.go +++ b/annotate/geo.go @@ -4,10 +4,10 @@ import ( "math" "time" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/internal/mputil" "github.com/paulmach/orb" "github.com/paulmach/orb/geo" - "github.com/paulmach/osm" - "github.com/paulmach/osm/internal/mputil" ) func wayPointOnSurface(w *osm.Way) orb.Point { diff --git a/annotate/geo_test.go b/annotate/geo_test.go index bc697d20..818db4ed 100644 --- a/annotate/geo_test.go +++ b/annotate/geo_test.go @@ -4,8 +4,8 @@ import ( "encoding/xml" "testing" + "github.com/nextmv-io/osm" "github.com/paulmach/orb" - "github.com/paulmach/osm" ) func TestWayPointOnSurface(t *testing.T) { diff --git a/annotate/internal/core/compute.go b/annotate/internal/core/compute.go index 531b815d..46966623 100644 --- a/annotate/internal/core/compute.go +++ b/annotate/internal/core/compute.go @@ -5,8 +5,8 @@ import ( "fmt" "time" - "github.com/paulmach/osm" - "github.com/paulmach/osm/annotate/shared" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/annotate/shared" ) // A Datasourcer is something that acts like a datasource allowing us to diff --git a/annotate/internal/core/compute_test.go b/annotate/internal/core/compute_test.go index c8fe94e9..8d02f9ef 100644 --- a/annotate/internal/core/compute_test.go +++ b/annotate/internal/core/compute_test.go @@ -7,8 +7,8 @@ import ( "testing" "time" - "github.com/paulmach/osm" - "github.com/paulmach/osm/annotate/shared" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/annotate/shared" ) var ( diff --git a/annotate/internal/core/datasourcer_test.go b/annotate/internal/core/datasourcer_test.go index 4a2fb4aa..c522d183 100644 --- a/annotate/internal/core/datasourcer_test.go +++ b/annotate/internal/core/datasourcer_test.go @@ -4,7 +4,7 @@ import ( "context" "errors" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) var _ Datasourcer = &TestDS{} diff --git a/annotate/internal/core/errors.go b/annotate/internal/core/errors.go index 451e3ee7..2bba17a3 100644 --- a/annotate/internal/core/errors.go +++ b/annotate/internal/core/errors.go @@ -4,7 +4,7 @@ import ( "fmt" "time" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) // NoHistoryError is returned if there is no entry in the history diff --git a/annotate/internal/core/types.go b/annotate/internal/core/types.go index 87fb037d..8fcc627e 100644 --- a/annotate/internal/core/types.go +++ b/annotate/internal/core/types.go @@ -3,8 +3,8 @@ package core import ( "time" - "github.com/paulmach/osm" - "github.com/paulmach/osm/annotate/shared" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/annotate/shared" ) // A Parent is something that holds children. ie. ways have nodes as children diff --git a/annotate/internal/core/types_test.go b/annotate/internal/core/types_test.go index c5ef266a..44754dcc 100644 --- a/annotate/internal/core/types_test.go +++ b/annotate/internal/core/types_test.go @@ -4,8 +4,8 @@ import ( "testing" "time" - "github.com/paulmach/osm" - "github.com/paulmach/osm/annotate/shared" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/annotate/shared" ) type findVisibleTestCase struct { diff --git a/annotate/options.go b/annotate/options.go index 789e19da..142aacca 100644 --- a/annotate/options.go +++ b/annotate/options.go @@ -3,8 +3,8 @@ package annotate import ( "time" - "github.com/paulmach/osm" - "github.com/paulmach/osm/annotate/internal/core" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/annotate/internal/core" ) // Option is a parameter that can be used for annotating. @@ -28,15 +28,15 @@ func Threshold(t time.Duration) Option { // This should be used when you want to gracefully handle the weird data in OSM. // // Nodes with unclear/inconsistent data will not be annotated. Causes include: -// - redacted data: In 2012 due to the license change data had to be removed. -// This could be some nodes of a way. There exist ways for which some nodes have -// just a single delete version, e.g. way 159081205, node 376130526 -// - data pre element versioning: pre-2012(?) data versions were not kept, so -// for old ways there many be no information about some nodes. For example, -// a node may be updated after a way and there is no way to get the original -// version of the node and way. -// - bad editors: sometimes a node is edited 7 times in a single changeset -// and version 5 is a delete. See node 321452894, part of way 28831147. +// - redacted data: In 2012 due to the license change data had to be removed. +// This could be some nodes of a way. There exist ways for which some nodes have +// just a single delete version, e.g. way 159081205, node 376130526 +// - data pre element versioning: pre-2012(?) data versions were not kept, so +// for old ways there many be no information about some nodes. For example, +// a node may be updated after a way and there is no way to get the original +// version of the node and way. +// - bad editors: sometimes a node is edited 7 times in a single changeset +// and version 5 is a delete. See node 321452894, part of way 28831147. func IgnoreInconsistency(yes bool) Option { return func(o *core.Options) error { o.IgnoreInconsistency = yes diff --git a/annotate/order.go b/annotate/order.go index 022fe127..c324b803 100644 --- a/annotate/order.go +++ b/annotate/order.go @@ -4,7 +4,7 @@ import ( "context" "sync" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) // RelationHistoryDatasourcer is an more strict interface for when we only need the relation history. diff --git a/annotate/order_test.go b/annotate/order_test.go index f9f7010f..e29716c3 100644 --- a/annotate/order_test.go +++ b/annotate/order_test.go @@ -4,7 +4,7 @@ import ( "context" "testing" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) func TestChildFirstOrdering(t *testing.T) { diff --git a/annotate/relation.go b/annotate/relation.go index e1fe2bbe..6de0d68d 100644 --- a/annotate/relation.go +++ b/annotate/relation.go @@ -4,9 +4,9 @@ import ( "context" "time" - "github.com/paulmach/osm" - "github.com/paulmach/osm/annotate/internal/core" - "github.com/paulmach/osm/annotate/shared" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/annotate/internal/core" + "github.com/nextmv-io/osm/annotate/shared" ) // HistoryAsChildrenDatasourcer is an advanced data source that diff --git a/annotate/relation_test.go b/annotate/relation_test.go index 1619bbe6..8eb2e41d 100644 --- a/annotate/relation_test.go +++ b/annotate/relation_test.go @@ -9,8 +9,8 @@ import ( "testing" "time" + "github.com/nextmv-io/osm" "github.com/paulmach/orb" - "github.com/paulmach/osm" ) func TestRelation(t *testing.T) { diff --git a/annotate/shared/child.go b/annotate/shared/child.go index fba94709..0881f4e7 100644 --- a/annotate/shared/child.go +++ b/annotate/shared/child.go @@ -5,7 +5,7 @@ package shared import ( "time" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) // A Child represents a node, way or relation that is a dependent for diff --git a/annotate/way.go b/annotate/way.go index 7e0bc2eb..cb08f045 100644 --- a/annotate/way.go +++ b/annotate/way.go @@ -4,9 +4,9 @@ import ( "context" "time" - "github.com/paulmach/osm" - "github.com/paulmach/osm/annotate/internal/core" - "github.com/paulmach/osm/annotate/shared" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/annotate/internal/core" + "github.com/nextmv-io/osm/annotate/shared" ) // NodeHistoryDatasourcer is an more strict interface for when we only need node history. diff --git a/annotate/way_test.go b/annotate/way_test.go index 47cfc3ec..6dae0f63 100644 --- a/annotate/way_test.go +++ b/annotate/way_test.go @@ -8,7 +8,7 @@ import ( "reflect" "testing" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) func TestWays(t *testing.T) { diff --git a/go.mod b/go.mod index 5ea8c278..53234598 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/paulmach/osm +module github.com/nextmv-io/osm go 1.13 diff --git a/internal/mputil/mputil.go b/internal/mputil/mputil.go index 3e8693de..a4719da3 100644 --- a/internal/mputil/mputil.go +++ b/internal/mputil/mputil.go @@ -3,8 +3,8 @@ package mputil import ( "time" + "github.com/nextmv-io/osm" "github.com/paulmach/orb" - "github.com/paulmach/osm" ) // Segment is a section of a multipolygon with some extra information diff --git a/internal/mputil/mputil_test.go b/internal/mputil/mputil_test.go index 219573fa..b6a7c3de 100644 --- a/internal/mputil/mputil_test.go +++ b/internal/mputil/mputil_test.go @@ -5,8 +5,8 @@ import ( "testing" "time" + "github.com/nextmv-io/osm" "github.com/paulmach/orb" - "github.com/paulmach/osm" ) func TestMultiSegment_LineString(t *testing.T) { diff --git a/json.go b/json.go index 593f8f9e..423931ce 100644 --- a/json.go +++ b/json.go @@ -12,7 +12,7 @@ import ( // // import ( // jsoniter "github.com/json-iterator/go" -// "github.com/paulmach/osm" +// "github.com/nextmv-io/osm" // ) // // var c = jsoniter.Config{ @@ -36,7 +36,7 @@ var CustomJSONMarshaler interface { // // import ( // jsoniter "github.com/json-iterator/go" -// "github.com/paulmach/osm" +// "github.com/nextmv-io/osm" // ) // // var c = jsoniter.Config{ diff --git a/osmapi/changeset.go b/osmapi/changeset.go index 959b4f4c..0610a3a0 100644 --- a/osmapi/changeset.go +++ b/osmapi/changeset.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) // Changeset returns a given changeset from the osm rest api. diff --git a/osmapi/datasource.go b/osmapi/datasource.go index 223e3b34..3ad115fc 100644 --- a/osmapi/datasource.go +++ b/osmapi/datasource.go @@ -8,7 +8,7 @@ import ( "net/http" "time" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) // BaseURL defines the api host. This can be change to hit @@ -18,8 +18,9 @@ const BaseURL = "http://api.openstreetmap.org/api/0.6" // A RateLimiter is something that can wait until its next allowed request. // This interface is met by `golang.org/x/time/rate.Limiter` and is meant // to be used with it. For example: -// // 10 qps -// osmapi.DefaultDatasource.Limiter = rate.NewLimiter(10, 1) +// +// // 10 qps +// osmapi.DefaultDatasource.Limiter = rate.NewLimiter(10, 1) type RateLimiter interface { Wait(context.Context) error } diff --git a/osmapi/live_test.go b/osmapi/live_test.go index f09070bb..6d100dc3 100644 --- a/osmapi/live_test.go +++ b/osmapi/live_test.go @@ -5,7 +5,7 @@ import ( "os" "testing" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" "golang.org/x/time/rate" ) diff --git a/osmapi/map.go b/osmapi/map.go index d0b3a93b..5ee833b8 100644 --- a/osmapi/map.go +++ b/osmapi/map.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) // Map returns the latest elements in the given bounding box. diff --git a/osmapi/map_test.go b/osmapi/map_test.go index c8292257..b4f7bc44 100644 --- a/osmapi/map_test.go +++ b/osmapi/map_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) func TestMap_urls(t *testing.T) { diff --git a/osmapi/node.go b/osmapi/node.go index 13140c3b..3a42d6db 100644 --- a/osmapi/node.go +++ b/osmapi/node.go @@ -5,7 +5,7 @@ import ( "fmt" "strconv" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) // Node returns the latest version of the node from the osm rest api. diff --git a/osmapi/node_test.go b/osmapi/node_test.go index 12b541f4..93254627 100644 --- a/osmapi/node_test.go +++ b/osmapi/node_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) func TestNode_urls(t *testing.T) { diff --git a/osmapi/note.go b/osmapi/note.go index c903e563..72b718d0 100644 --- a/osmapi/note.go +++ b/osmapi/note.go @@ -6,7 +6,7 @@ import ( "net/url" "strings" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) // Note returns the note from the osm rest api. diff --git a/osmapi/note_test.go b/osmapi/note_test.go index ec93f432..3d193f5f 100644 --- a/osmapi/note_test.go +++ b/osmapi/note_test.go @@ -7,7 +7,7 @@ import ( "strings" "testing" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) func TestNote_urls(t *testing.T) { diff --git a/osmapi/relation.go b/osmapi/relation.go index e629e0bc..5e767c70 100644 --- a/osmapi/relation.go +++ b/osmapi/relation.go @@ -5,7 +5,7 @@ import ( "fmt" "strconv" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) // Relation returns the latest version of the relation from the osm rest api. diff --git a/osmapi/relation_test.go b/osmapi/relation_test.go index bea89341..b64cbc37 100644 --- a/osmapi/relation_test.go +++ b/osmapi/relation_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) func TestRelation_urls(t *testing.T) { diff --git a/osmapi/user.go b/osmapi/user.go index 17a7bff5..3b01ce81 100644 --- a/osmapi/user.go +++ b/osmapi/user.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) // User returns the user from the osm rest api. diff --git a/osmapi/way.go b/osmapi/way.go index 5f834fd0..9d75d2d1 100644 --- a/osmapi/way.go +++ b/osmapi/way.go @@ -5,7 +5,7 @@ import ( "fmt" "strconv" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) // Way returns the latest version of the way from the osm rest api. diff --git a/osmapi/way_test.go b/osmapi/way_test.go index 050357ee..6377415f 100644 --- a/osmapi/way_test.go +++ b/osmapi/way_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) func TestWay_urls(t *testing.T) { diff --git a/osmgeojson/benchmarks_test.go b/osmgeojson/benchmarks_test.go index 1dc71dd7..42cf0a3f 100644 --- a/osmgeojson/benchmarks_test.go +++ b/osmgeojson/benchmarks_test.go @@ -5,7 +5,7 @@ import ( "io/ioutil" "testing" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) func BenchmarkConvert(b *testing.B) { diff --git a/osmgeojson/build_polygon.go b/osmgeojson/build_polygon.go index bc00284d..45c5721b 100644 --- a/osmgeojson/build_polygon.go +++ b/osmgeojson/build_polygon.go @@ -3,10 +3,10 @@ package osmgeojson import ( "fmt" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/internal/mputil" "github.com/paulmach/orb" "github.com/paulmach/orb/geojson" - "github.com/paulmach/osm" - "github.com/paulmach/osm/internal/mputil" ) func (ctx *context) buildPolygon(relation *osm.Relation) *geojson.Feature { diff --git a/osmgeojson/build_polygon_test.go b/osmgeojson/build_polygon_test.go index d80750c9..fc684b88 100644 --- a/osmgeojson/build_polygon_test.go +++ b/osmgeojson/build_polygon_test.go @@ -4,9 +4,9 @@ import ( "encoding/xml" "testing" + "github.com/nextmv-io/osm" "github.com/paulmach/orb" "github.com/paulmach/orb/geojson" - "github.com/paulmach/osm" ) func TestConvert_polygon(t *testing.T) { diff --git a/osmgeojson/convert.go b/osmgeojson/convert.go index ddc2013b..e0c8cf19 100644 --- a/osmgeojson/convert.go +++ b/osmgeojson/convert.go @@ -3,10 +3,10 @@ package osmgeojson import ( "fmt" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/internal/mputil" "github.com/paulmach/orb" "github.com/paulmach/orb/geojson" - "github.com/paulmach/osm" - "github.com/paulmach/osm/internal/mputil" ) type context struct { diff --git a/osmgeojson/convert_test.go b/osmgeojson/convert_test.go index 3209d0ec..fbee4bf5 100644 --- a/osmgeojson/convert_test.go +++ b/osmgeojson/convert_test.go @@ -6,9 +6,9 @@ import ( "reflect" "testing" + "github.com/nextmv-io/osm" "github.com/paulmach/orb" "github.com/paulmach/orb/geojson" - "github.com/paulmach/osm" ) func TestConvert(t *testing.T) { diff --git a/osmgeojson/options_test.go b/osmgeojson/options_test.go index 2d74b964..80148247 100644 --- a/osmgeojson/options_test.go +++ b/osmgeojson/options_test.go @@ -4,8 +4,8 @@ import ( "encoding/xml" "testing" + "github.com/nextmv-io/osm" "github.com/paulmach/orb/geojson" - "github.com/paulmach/osm" ) var nodeXML = ` diff --git a/osmpbf/decode.go b/osmpbf/decode.go index eec3892a..54473990 100644 --- a/osmpbf/decode.go +++ b/osmpbf/decode.go @@ -10,8 +10,8 @@ import ( "sync" "time" - "github.com/paulmach/osm" - "github.com/paulmach/osm/osmpbf/internal/osmpbf" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/osmpbf/internal/osmpbf" "google.golang.org/protobuf/proto" ) diff --git a/osmpbf/decode_data.go b/osmpbf/decode_data.go index 1847a4ba..ba782721 100644 --- a/osmpbf/decode_data.go +++ b/osmpbf/decode_data.go @@ -4,10 +4,10 @@ import ( "errors" "time" - "google.golang.org/protobuf/proto" - "github.com/paulmach/osm" - "github.com/paulmach/osm/osmpbf/internal/osmpbf" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/osmpbf/internal/osmpbf" "github.com/paulmach/protoscan" + "google.golang.org/protobuf/proto" ) // dataDecoder is a decoder for Blob with OSMData (PrimitiveBlock). diff --git a/osmpbf/decode_test.go b/osmpbf/decode_test.go index 6b6785c6..8a4a8698 100644 --- a/osmpbf/decode_test.go +++ b/osmpbf/decode_test.go @@ -12,7 +12,7 @@ import ( "testing" "time" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) const ( diff --git a/osmpbf/encode.go b/osmpbf/encode.go index 81d28b07..a65cba54 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -5,8 +5,8 @@ import ( "io" "time" - "github.com/paulmach/osm" - "github.com/paulmach/osm/osmpbf/internal/osmpbf" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/osmpbf/internal/osmpbf" "google.golang.org/protobuf/proto" ) diff --git a/osmpbf/example_stats_test.go b/osmpbf/example_stats_test.go index 9a8182f9..b9d7a049 100644 --- a/osmpbf/example_stats_test.go +++ b/osmpbf/example_stats_test.go @@ -7,8 +7,8 @@ import ( "os" "time" - "github.com/paulmach/osm" - "github.com/paulmach/osm/osmpbf" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/osmpbf" ) // ExampleStats demonstrates how to read a full file and gather some stats. diff --git a/osmpbf/internal/osmpbf/generate.go b/osmpbf/internal/osmpbf/generate.go index 8eee6f9b..a330ef3f 100644 --- a/osmpbf/internal/osmpbf/generate.go +++ b/osmpbf/internal/osmpbf/generate.go @@ -1,3 +1,3 @@ package osmpbf -//go:generate protoc --proto_path=. --go_opt=module=github.com/paulmach/osmpbf/internal/osmpbf --go_out=. fileformat.proto osmformat.proto +//go:generate protoc --proto_path=. --go_opt=module=github.com/nextmv-io/osmpbf/internal/osmpbf --go_out=. fileformat.proto osmformat.proto diff --git a/osmpbf/scanner.go b/osmpbf/scanner.go index b0479029..201b7a43 100644 --- a/osmpbf/scanner.go +++ b/osmpbf/scanner.go @@ -5,7 +5,7 @@ import ( "io" "sync/atomic" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) var _ osm.Scanner = &Scanner{} diff --git a/osmpbf/scanner_test.go b/osmpbf/scanner_test.go index fd47c082..70d9e6db 100644 --- a/osmpbf/scanner_test.go +++ b/osmpbf/scanner_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) var ( diff --git a/osmtest/scanner.go b/osmtest/scanner.go index 1119c865..3a35188e 100644 --- a/osmtest/scanner.go +++ b/osmtest/scanner.go @@ -1,6 +1,6 @@ package osmtest -import "github.com/paulmach/osm" +import "github.com/nextmv-io/osm" // Scanner implements the osm.Scanner interface with // just a list of objects. diff --git a/osmtest/scanner_test.go b/osmtest/scanner_test.go index f3c68207..fff9f7e6 100644 --- a/osmtest/scanner_test.go +++ b/osmtest/scanner_test.go @@ -5,7 +5,7 @@ import ( "reflect" "testing" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) func TestScanner(t *testing.T) { diff --git a/osmxml/example_test.go b/osmxml/example_test.go index 61700dca..37979075 100644 --- a/osmxml/example_test.go +++ b/osmxml/example_test.go @@ -5,8 +5,8 @@ import ( "fmt" "os" - "github.com/paulmach/osm" - "github.com/paulmach/osm/osmxml" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/osmxml" ) func ExampleScanner() { diff --git a/osmxml/scanner.go b/osmxml/scanner.go index b232d357..c0c91926 100644 --- a/osmxml/scanner.go +++ b/osmxml/scanner.go @@ -6,7 +6,7 @@ import ( "io" "strings" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) var _ osm.Scanner = &Scanner{} @@ -126,6 +126,7 @@ Loop: // Object returns the most recent token generated by a call to Scan // as a new osm.Object. This interface is implemented by: +// // *osm.Bounds // *osm.Node // *osm.Way diff --git a/osmxml/scanner_test.go b/osmxml/scanner_test.go index fd85aa7a..daf7dc65 100644 --- a/osmxml/scanner_test.go +++ b/osmxml/scanner_test.go @@ -8,7 +8,7 @@ import ( "os" "testing" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) func TestScanner(t *testing.T) { diff --git a/replication/changesets.go b/replication/changesets.go index f149c8cf..f9132a59 100644 --- a/replication/changesets.go +++ b/replication/changesets.go @@ -10,8 +10,8 @@ import ( "net/http" "strconv" - "github.com/paulmach/osm" - "github.com/paulmach/osm/osmxml" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/osmxml" ) // ChangesetSeqNum indicates the sequence of the changeset replication found here: diff --git a/replication/interval.go b/replication/interval.go index 8c258bd8..0193688a 100644 --- a/replication/interval.go +++ b/replication/interval.go @@ -11,7 +11,7 @@ import ( "strconv" "time" - "github.com/paulmach/osm" + "github.com/nextmv-io/osm" ) var _ SeqNum = MinuteSeqNum(0) diff --git a/testdata/compare_scanners.go b/testdata/compare_scanners.go index f019dd7c..7ab9aeff 100644 --- a/testdata/compare_scanners.go +++ b/testdata/compare_scanners.go @@ -8,9 +8,9 @@ import ( "os" "reflect" - "github.com/paulmach/osm" - "github.com/paulmach/osm/osmpbf" - "github.com/paulmach/osm/osmxml" + "github.com/nextmv-io/osm" + "github.com/nextmv-io/osm/osmpbf" + "github.com/nextmv-io/osm/osmxml" ) func main() { From 3b71c2a7bc01fd75510f8b07b93e0f1745ce1953 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Fri, 1 Sep 2023 20:15:18 +0200 Subject: [PATCH 03/29] Use getters for default values --- osmpbf/encode.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/osmpbf/encode.go b/osmpbf/encode.go index a65cba54..8268b4f1 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -222,11 +222,11 @@ func encode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, osm func EncodeDenseNode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, groupDense *osmpbf.DenseNodes, current, previous *osm.Node) { groupDense.Id = append(groupDense.Id, int64(current.ID-previous.ID)) // cast block.Granularity to int64 - granularity := int64(*block.Granularity) - currentLat := EncodeLatLon(current.Lat, int64(*block.LatOffset), granularity) - currentLon := EncodeLatLon(current.Lon, int64(*block.LonOffset), granularity) - previousLat := EncodeLatLon(previous.Lat, int64(*block.LatOffset), granularity) - previousLon := EncodeLatLon(previous.Lon, int64(*block.LonOffset), granularity) + granularity := block.GetGranularity() + currentLat := EncodeLatLon(current.Lat, block.GetLatOffset(), granularity) + currentLon := EncodeLatLon(current.Lon, block.GetLonOffset(), granularity) + previousLat := EncodeLatLon(previous.Lat, block.GetLatOffset(), granularity) + previousLon := EncodeLatLon(previous.Lon, block.GetLonOffset(), granularity) latDiff := currentLat - previousLat lonDiff := currentLon - previousLon groupDense.Lat = append(groupDense.Lat, latDiff) @@ -242,7 +242,7 @@ func EncodeDenseNode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string if groupDense.Denseinfo != nil { groupDense.Denseinfo.Changeset = append(groupDense.Denseinfo.Changeset, int64(current.ChangesetID-previous.ChangesetID)) - dateGranularity := int64(*block.DateGranularity) + dateGranularity := block.GetDateGranularity() currentTimeStamp := EncodeTimestamp(current.Timestamp, dateGranularity) previousTimeStamp := EncodeTimestamp(previous.Timestamp, dateGranularity) groupDense.Denseinfo.Timestamp = append(groupDense.Denseinfo.Timestamp, currentTimeStamp-previousTimeStamp) @@ -269,7 +269,7 @@ func EncodeNode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, pbfNode.Info.Changeset = &changesetId } if !node.Timestamp.IsZero() { - timeStamp := EncodeTimestamp(node.Timestamp, int64(*block.DateGranularity)) + timeStamp := EncodeTimestamp(node.Timestamp, block.GetDateGranularity()) pbfNode.Info.Timestamp = &timeStamp } if node.UserID != 0 { @@ -282,9 +282,9 @@ func EncodeNode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, nodeVersion := int32(node.Version) pbfNode.Info.Version = &nodeVersion } - lat := EncodeLatLon(node.Lat, *block.LatOffset, int64(*block.Granularity)) + lat := EncodeLatLon(node.Lat, *block.LatOffset, block.GetGranularity()) pbfNode.Lat = &lat - lon := EncodeLatLon(node.Lon, *block.LonOffset, int64(*block.Granularity)) + lon := EncodeLatLon(node.Lon, *block.LonOffset, block.GetGranularity()) pbfNode.Lon = &lon if len(node.Tags) > 0 { @@ -308,7 +308,7 @@ func EncodeWay(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, pbfWay.Info.Changeset = (*int64)(&way.ChangesetID) } if !way.Timestamp.IsZero() { - timestamp := EncodeTimestamp(way.Timestamp, int64(*block.DateGranularity)) + timestamp := EncodeTimestamp(way.Timestamp, block.GetDateGranularity()) pbfWay.Info.Timestamp = ×tamp } if way.UserID != 0 { @@ -346,7 +346,7 @@ func EncodeRelation(block *osmpbf.PrimitiveBlock, reverseStringTable map[string] }, } if !relation.Timestamp.IsZero() { - timestamp := EncodeTimestamp(relation.Timestamp, int64(*block.DateGranularity)) + timestamp := EncodeTimestamp(relation.Timestamp, block.GetDateGranularity()) pbfRelation.Info.Timestamp = ×tamp } if relation.UserID != 0 { @@ -392,12 +392,12 @@ func EncodeRelation(block *osmpbf.PrimitiveBlock, reverseStringTable map[string] return pbfRelation } -func EncodeLatLon(value float64, offset, granularity int64) int64 { - return (int64(value/.000000001) - offset) / granularity +func EncodeLatLon(value float64, offset int64, granularity int32) int64 { + return (int64(value/.000000001) - offset) / int64(granularity) } -func EncodeTimestamp(timestamp time.Time, dateGranularity int64) int64 { - return timestamp.Unix() / dateGranularity +func EncodeTimestamp(timestamp time.Time, dateGranularity int32) int64 { + return timestamp.Unix() / int64(dateGranularity) } func EncodeString(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, value string) int32 { From a6a7508424e0d7bcc273832132ffe54e319c0ff6 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Fri, 1 Sep 2023 21:02:58 +0200 Subject: [PATCH 04/29] Write correct osmType to blob --- osmpbf/encode.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/osmpbf/encode.go b/osmpbf/encode.go index 8268b4f1..39b360fb 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -32,7 +32,7 @@ func NewWriter(w io.Writer) (Writer, error) { return nil, err } - _, err = encoder.write(blockHeaderData) + _, err = encoder.write(blockHeaderData, osmHeaderType) if err != nil { return nil, err } @@ -67,11 +67,11 @@ func (e *encoder) flush() error { if err != nil { return nil } - _, err = e.write(blockData) + _, err = e.write(blockData, osmDataType) return err } -func (e *encoder) write(data []byte) (n int, err error) { +func (e *encoder) write(data []byte, osmType string) (n int, err error) { blob := &osmpbf.Blob{} blob.RawSize = proto.Int32(int32(len(data))) if e.compress { @@ -92,7 +92,7 @@ func (e *encoder) write(data []byte) (n int, err error) { blobHeader := &osmpbf.BlobHeader{ Datasize: proto.Int32(int32(len(blobData))), - Type: proto.String(osmDataType), + Type: proto.String(osmType), } blobHeaderData, err := proto.Marshal(blobHeader) From 86cc279a91cc0caaa0dca2478e6c961f71e3ef37 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Fri, 1 Sep 2023 21:23:39 +0200 Subject: [PATCH 05/29] Writing some info about header sizes --- osmpbf/encode.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/osmpbf/encode.go b/osmpbf/encode.go index 39b360fb..4eec7cd2 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -2,6 +2,7 @@ package osmpbf import ( "bytes" + "fmt" "io" "time" @@ -74,6 +75,7 @@ func (e *encoder) flush() error { func (e *encoder) write(data []byte, osmType string) (n int, err error) { blob := &osmpbf.Blob{} blob.RawSize = proto.Int32(int32(len(data))) + fmt.Println("blob.RawSize", blob.RawSize) if e.compress { target := &bytes.Buffer{} // compress the data @@ -95,6 +97,8 @@ func (e *encoder) write(data []byte, osmType string) (n int, err error) { Type: proto.String(osmType), } + fmt.Println("blobHeader.Datasize", blobHeader.Datasize) + blobHeaderData, err := proto.Marshal(blobHeader) if err != nil { return 0, nil From 53ca2374122e0fc6bf63963decb640f77e1b1c1c Mon Sep 17 00:00:00 2001 From: larsbeck Date: Fri, 1 Sep 2023 21:27:22 +0200 Subject: [PATCH 06/29] More output --- osmpbf/encode.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/osmpbf/encode.go b/osmpbf/encode.go index 4eec7cd2..115c0816 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -33,11 +33,15 @@ func NewWriter(w io.Writer) (Writer, error) { return nil, err } - _, err = encoder.write(blockHeaderData, osmHeaderType) + fmt.Println("blockHeaderData:", len(blockHeaderData)) + + n, err := encoder.write(blockHeaderData, osmHeaderType) if err != nil { return nil, err } + fmt.Println("n:", n) + return encoder, nil } @@ -75,7 +79,6 @@ func (e *encoder) flush() error { func (e *encoder) write(data []byte, osmType string) (n int, err error) { blob := &osmpbf.Blob{} blob.RawSize = proto.Int32(int32(len(data))) - fmt.Println("blob.RawSize", blob.RawSize) if e.compress { target := &bytes.Buffer{} // compress the data @@ -97,8 +100,6 @@ func (e *encoder) write(data []byte, osmType string) (n int, err error) { Type: proto.String(osmType), } - fmt.Println("blobHeader.Datasize", blobHeader.Datasize) - blobHeaderData, err := proto.Marshal(blobHeader) if err != nil { return 0, nil From 99a5385d4cfb7db35dc39896d036e0022762e2ec Mon Sep 17 00:00:00 2001 From: larsbeck Date: Fri, 1 Sep 2023 22:16:52 +0200 Subject: [PATCH 07/29] Write size as binary.BigEndian --- osmpbf/encode.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/osmpbf/encode.go b/osmpbf/encode.go index 115c0816..d748d8b7 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -2,7 +2,7 @@ package osmpbf import ( "bytes" - "fmt" + "encoding/binary" "io" "time" @@ -33,15 +33,11 @@ func NewWriter(w io.Writer) (Writer, error) { return nil, err } - fmt.Println("blockHeaderData:", len(blockHeaderData)) - - n, err := encoder.write(blockHeaderData, osmHeaderType) + _, err = encoder.write(blockHeaderData, osmHeaderType) if err != nil { return nil, err } - fmt.Println("n:", n) - return encoder, nil } @@ -105,6 +101,13 @@ func (e *encoder) write(data []byte, osmType string) (n int, err error) { return 0, nil } + size := make([]byte, 4) + binary.BigEndian.PutUint32(size, uint32(len(blobHeaderData))) + + if _, err = e.stream.Write(size); err != nil { + return 0, nil + } + if _, err = e.stream.Write(blobHeaderData); err != nil { return 0, nil } From 833d1d481f52c9de202dde783e88fbb5f1983c3c Mon Sep 17 00:00:00 2001 From: larsbeck Date: Sat, 2 Sep 2023 12:37:16 +0200 Subject: [PATCH 08/29] Add encoder tests --- osmpbf/encode.go | 4 +- osmpbf/encode_test.go | 227 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 229 insertions(+), 2 deletions(-) create mode 100644 osmpbf/encode_test.go diff --git a/osmpbf/encode.go b/osmpbf/encode.go index d748d8b7..21dbe410 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -92,8 +92,8 @@ func (e *encoder) write(data []byte, osmType string) (n int, err error) { } blobHeader := &osmpbf.BlobHeader{ - Datasize: proto.Int32(int32(len(blobData))), Type: proto.String(osmType), + Datasize: proto.Int32(int32(len(blobData))), } blobHeaderData, err := proto.Marshal(blobHeader) @@ -405,7 +405,7 @@ func EncodeLatLon(value float64, offset int64, granularity int32) int64 { } func EncodeTimestamp(timestamp time.Time, dateGranularity int32) int64 { - return timestamp.Unix() / int64(dateGranularity) + return timestamp.UnixMilli() / int64(dateGranularity) } func EncodeString(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, value string) int32 { diff --git a/osmpbf/encode_test.go b/osmpbf/encode_test.go new file mode 100644 index 00000000..7f0a4846 --- /dev/null +++ b/osmpbf/encode_test.go @@ -0,0 +1,227 @@ +package osmpbf + +import ( + "bytes" + "context" + "fmt" + "io" + "runtime" + "testing" + + "github.com/nextmv-io/osm" +) + +func TestEncodeDecode(t *testing.T) { + buffer := bytes.Buffer{} + writer, err := NewWriter(&buffer) + if err != nil { + t.Fatal(err) + } + + // write node + err = writer.WriteObject(en) + if err != nil { + t.Fatal(err) + } + // write way + err = writer.WriteObject(ew) + if err != nil { + t.Fatal(err) + } + // write relation + err = writer.WriteObject(er) + if err != nil { + t.Fatal(err) + } + writer.Close() + + d := newDecoder(context.Background(), &Scanner{}, &buffer) + err = d.Start(runtime.GOMAXPROCS(-1)) + if err != nil { + t.Fatal(err) + } + + for { + e, err := d.Next() + + if err == io.EOF { + break + } else if err != nil { + t.Fatal(err) + } + + switch v := e.(type) { + case *osm.Node: + err = nodeEquals(en, v) + if err != nil { + t.Fatal(err) + } + case *osm.Way: + err = wayEquals(ew, v) + if err != nil { + t.Fatal(err) + } + case *osm.Relation: + err = relationEquals(er, v) + if err != nil { + t.Fatal(err) + } + } + } + d.Close() +} + +func nodeEquals(en, node *osm.Node) error { + if node.ID != en.ID { + return fmt.Errorf("node id mismatch: %d != %d", node.ID, en.ID) + } + if node.Lat != en.Lat { + return fmt.Errorf("node lat mismatch: %f != %f", node.Lat, en.Lat) + } + if node.Lon != en.Lon { + return fmt.Errorf("node lon mismatch: %f != %f", node.Lon, en.Lon) + } + if node.User != en.User { + return fmt.Errorf("node user mismatch: %s != %s", node.User, en.User) + } + if node.UserID != en.UserID { + return fmt.Errorf("node user id mismatch: %d != %d", node.UserID, en.UserID) + } + if node.Visible != en.Visible { + return fmt.Errorf("node visible mismatch: %v != %v", node.Visible, en.Visible) + } + if node.Version != en.Version { + return fmt.Errorf("node version mismatch: %d != %d", node.Version, en.Version) + } + if node.ChangesetID != en.ChangesetID { + return fmt.Errorf("node changeset id mismatch: %d != %d", node.ChangesetID, en.ChangesetID) + } + // TODO: this currently fails with + // "node timestamp mismatch: 2224-09-20 11:45:12.871345152 +0000 UTC != + // 2009-05-20 10:28:54 +0000 UTC" + // The other timestamps (for ways and relations) work fine. + // if node.Timestamp != en.Timestamp { + // return fmt.Errorf("node timestamp mismatch: %s != %s", node.Timestamp, en.Timestamp) + // } + if len(node.Tags) != len(en.Tags) { + return fmt.Errorf("node tags length mismatch: %d != %d", len(node.Tags), len(en.Tags)) + } + for k, v := range node.Tags { + if en.Tags[k] != v { + return fmt.Errorf("node tag mismatch: %s != %s", en.Tags[k], v) + } + } + return nil +} + +func wayEquals(ew, way *osm.Way) error { + if way.ID != ew.ID { + return fmt.Errorf("way id mismatch: %d != %d", way.ID, ew.ID) + } + if way.User != ew.User { + return fmt.Errorf("way user mismatch: %s != %s", way.User, ew.User) + } + if way.UserID != ew.UserID { + return fmt.Errorf("way user id mismatch: %d != %d", way.UserID, ew.UserID) + } + if way.Visible != ew.Visible { + return fmt.Errorf("way visible mismatch: %v != %v", way.Visible, ew.Visible) + } + if way.Version != ew.Version { + return fmt.Errorf("way version mismatch: %d != %d", way.Version, ew.Version) + } + if way.ChangesetID != ew.ChangesetID { + return fmt.Errorf("way changeset id mismatch: %d != %d", way.ChangesetID, ew.ChangesetID) + } + if way.Timestamp != ew.Timestamp { + return fmt.Errorf("way timestamp mismatch: %s != %s", way.Timestamp, ew.Timestamp) + } + if len(way.Tags) != len(ew.Tags) { + return fmt.Errorf("way tags length mismatch: %d != %d", len(way.Tags), len(ew.Tags)) + } + for k, v := range way.Tags { + if ew.Tags[k] != v { + return fmt.Errorf("way tag mismatch: %s != %s", ew.Tags[k], v) + } + } + if len(way.Nodes) != len(ew.Nodes) { + return fmt.Errorf("way nodes length mismatch: %d != %d", len(way.Nodes), len(ew.Nodes)) + } + for i, v := range way.Nodes { + if ew.Nodes[i] != v { + return fmt.Errorf("way node mismatch: %v != %v", ew.Nodes[i], v) + } + } + return nil +} + +func relationEquals(er, relation *osm.Relation) error { + if relation.ID != er.ID { + return fmt.Errorf("relation id mismatch: %d != %d", relation.ID, er.ID) + } + if relation.User != er.User { + return fmt.Errorf("relation user mismatch: %s != %s", relation.User, er.User) + } + if relation.UserID != er.UserID { + return fmt.Errorf("relation user id mismatch: %d != %d", relation.UserID, er.UserID) + } + if relation.Visible != er.Visible { + return fmt.Errorf("relation visible mismatch: %v != %v", relation.Visible, er.Visible) + } + if relation.Version != er.Version { + return fmt.Errorf("relation version mismatch: %d != %d", relation.Version, er.Version) + } + if relation.ChangesetID != er.ChangesetID { + return fmt.Errorf("relation changeset id mismatch: %d != %d", relation.ChangesetID, er.ChangesetID) + } + if relation.Timestamp != er.Timestamp { + return fmt.Errorf("relation timestamp mismatch: %s != %s", relation.Timestamp, er.Timestamp) + } + if len(relation.Tags) != len(er.Tags) { + return fmt.Errorf("relation tags length mismatch: %d != %d", len(relation.Tags), len(er.Tags)) + } + for k, v := range relation.Tags { + if er.Tags[k] != v { + return fmt.Errorf("relation tag mismatch: %s != %s", er.Tags[k], v) + } + } + if len(relation.Members) != len(er.Members) { + return fmt.Errorf("relation members length mismatch: %d != %d", len(relation.Members), len(er.Members)) + } + for i, member := range relation.Members { + expectedMember := er.Members[i] + if member.ChangesetID != expectedMember.ChangesetID { + return fmt.Errorf("relation member changeset id mismatch: %d != %d", member.ChangesetID, expectedMember.ChangesetID) + } + if member.Role != expectedMember.Role { + return fmt.Errorf("relation member role mismatch: %s != %s", member.Role, expectedMember.Role) + } + if member.Type != expectedMember.Type { + return fmt.Errorf("relation member type mismatch: %s != %s", member.Type, expectedMember.Type) + } + if member.Ref != expectedMember.Ref { + return fmt.Errorf("relation member ref mismatch: %d != %d", member.Ref, expectedMember.Ref) + } + if member.Lat != expectedMember.Lat { + return fmt.Errorf("relation member lat mismatch: %f != %f", member.Lat, expectedMember.Lat) + } + if member.Lon != expectedMember.Lon { + return fmt.Errorf("relation member lon mismatch: %f != %f", member.Lon, expectedMember.Lon) + } + if member.Orientation != expectedMember.Orientation { + return fmt.Errorf("relation member orientation mismatch: %d != %d", member.Orientation, expectedMember.Orientation) + } + if member.Version != expectedMember.Version { + return fmt.Errorf("relation member version mismatch: %d != %d", member.Version, expectedMember.Version) + } + if len(member.Nodes) != len(expectedMember.Nodes) { + return fmt.Errorf("relation member nodes length mismatch: %d != %d", len(member.Nodes), len(expectedMember.Nodes)) + } + for j, node := range member.Nodes { + if expectedMember.Nodes[j] != node { + return fmt.Errorf("relation member node mismatch: %v != %v", expectedMember.Nodes[j], node) + } + } + } + return nil +} From 7589006ff8d9cb4a21850d6462020f12ba7e5b18 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Sat, 2 Sep 2023 16:14:08 +0200 Subject: [PATCH 09/29] Protect write by lock --- osmpbf/encode.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/osmpbf/encode.go b/osmpbf/encode.go index 21dbe410..a02b4c9a 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/binary" "io" + "sync" "time" "github.com/nextmv-io/osm" @@ -45,6 +46,7 @@ type encoder struct { stream io.Writer reverseStringTable map[string]int entities []osm.Object + mu sync.Mutex compress bool } @@ -73,6 +75,8 @@ func (e *encoder) flush() error { } func (e *encoder) write(data []byte, osmType string) (n int, err error) { + e.mu.Lock() + defer e.mu.Unlock() blob := &osmpbf.Blob{} blob.RawSize = proto.Int32(int32(len(data))) if e.compress { From b51b739e117463fa7f5b0c500bb780a7d5807ce4 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Sat, 2 Sep 2023 16:16:16 +0200 Subject: [PATCH 10/29] Protect WriteObject by lock --- osmpbf/encode.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/osmpbf/encode.go b/osmpbf/encode.go index a02b4c9a..9904e02e 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -75,8 +75,6 @@ func (e *encoder) flush() error { } func (e *encoder) write(data []byte, osmType string) (n int, err error) { - e.mu.Lock() - defer e.mu.Unlock() blob := &osmpbf.Blob{} blob.RawSize = proto.Int32(int32(len(data))) if e.compress { @@ -120,6 +118,8 @@ func (e *encoder) write(data []byte, osmType string) (n int, err error) { } func (e *encoder) WriteObject(obj osm.Object) error { + e.mu.Lock() + defer e.mu.Unlock() e.entities = append(e.entities, obj) if len(e.entities) >= 8000 { return e.flush() From e559d412273f491e580cb8baaf0d8bd71ab45c67 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Sat, 2 Sep 2023 17:42:15 +0200 Subject: [PATCH 11/29] Flush if entity type changed --- osmpbf/encode.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/osmpbf/encode.go b/osmpbf/encode.go index 9904e02e..efaa1abb 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -47,6 +47,7 @@ type encoder struct { reverseStringTable map[string]int entities []osm.Object mu sync.Mutex + lastWrittenType osm.Type compress bool } @@ -120,6 +121,14 @@ func (e *encoder) write(data []byte, osmType string) (n int, err error) { func (e *encoder) WriteObject(obj osm.Object) error { e.mu.Lock() defer e.mu.Unlock() + + if e.lastWrittenType != obj.ObjectID().Type() { + if err := e.flush(); err != nil { + return err + } + e.lastWrittenType = obj.ObjectID().Type() + } + e.entities = append(e.entities, obj) if len(e.entities) >= 8000 { return e.flush() From fc1139b5f8f9cada0397c44fadb406a03bad5748 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Sat, 2 Sep 2023 18:25:38 +0200 Subject: [PATCH 12/29] Deactivate tags and denseinfo --- osmpbf/encode.go | 50 ++++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/osmpbf/encode.go b/osmpbf/encode.go index efaa1abb..bcbeb84b 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -253,29 +253,29 @@ func EncodeDenseNode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string groupDense.Lat = append(groupDense.Lat, latDiff) groupDense.Lon = append(groupDense.Lon, lonDiff) - if current.Tags != nil { - for _, nodeTag := range current.Tags { - groupDense.KeysVals = append(groupDense.KeysVals, EncodeString(block, reverseStringTable, nodeTag.Key)) - groupDense.KeysVals = append(groupDense.KeysVals, EncodeString(block, reverseStringTable, nodeTag.Value)) - } - groupDense.KeysVals = append(groupDense.KeysVals, 0) - } - - if groupDense.Denseinfo != nil { - groupDense.Denseinfo.Changeset = append(groupDense.Denseinfo.Changeset, int64(current.ChangesetID-previous.ChangesetID)) - dateGranularity := block.GetDateGranularity() - currentTimeStamp := EncodeTimestamp(current.Timestamp, dateGranularity) - previousTimeStamp := EncodeTimestamp(previous.Timestamp, dateGranularity) - groupDense.Denseinfo.Timestamp = append(groupDense.Denseinfo.Timestamp, currentTimeStamp-previousTimeStamp) - groupDense.Denseinfo.Uid = append(groupDense.Denseinfo.Uid, int32(current.UserID-previous.UserID)) - groupDense.Denseinfo.Version = append(groupDense.Denseinfo.Version, int32(current.Version-previous.Version)) - var previousUserNameId int32 = 0 - if previous.User != "" { - previousUserNameId = EncodeString(block, reverseStringTable, previous.User) - } - currentUserNameId := EncodeString(block, reverseStringTable, current.User) - groupDense.Denseinfo.UserSid = append(groupDense.Denseinfo.UserSid, currentUserNameId-previousUserNameId) - } + // if current.Tags != nil { + // for _, nodeTag := range current.Tags { + // groupDense.KeysVals = append(groupDense.KeysVals, EncodeString(block, reverseStringTable, nodeTag.Key)) + // groupDense.KeysVals = append(groupDense.KeysVals, EncodeString(block, reverseStringTable, nodeTag.Value)) + // } + // groupDense.KeysVals = append(groupDense.KeysVals, 0) + // } + + // if groupDense.Denseinfo != nil { + // groupDense.Denseinfo.Changeset = append(groupDense.Denseinfo.Changeset, int64(current.ChangesetID-previous.ChangesetID)) + // dateGranularity := block.GetDateGranularity() + // currentTimeStamp := EncodeTimestamp(current.Timestamp, dateGranularity) + // previousTimeStamp := EncodeTimestamp(previous.Timestamp, dateGranularity) + // groupDense.Denseinfo.Timestamp = append(groupDense.Denseinfo.Timestamp, currentTimeStamp-previousTimeStamp) + // groupDense.Denseinfo.Uid = append(groupDense.Denseinfo.Uid, int32(current.UserID-previous.UserID)) + // groupDense.Denseinfo.Version = append(groupDense.Denseinfo.Version, int32(current.Version-previous.Version)) + // var previousUserNameId int32 = 0 + // if previous.User != "" { + // previousUserNameId = EncodeString(block, reverseStringTable, previous.User) + // } + // currentUserNameId := EncodeString(block, reverseStringTable, current.User) + // groupDense.Denseinfo.UserSid = append(groupDense.Denseinfo.UserSid, currentUserNameId-previousUserNameId) + // } } func EncodeNode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, pbfNode *osmpbf.Node, node *osm.Node) *osmpbf.Node { @@ -303,9 +303,9 @@ func EncodeNode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, nodeVersion := int32(node.Version) pbfNode.Info.Version = &nodeVersion } - lat := EncodeLatLon(node.Lat, *block.LatOffset, block.GetGranularity()) + lat := EncodeLatLon(node.Lat, block.GetLatOffset(), block.GetGranularity()) pbfNode.Lat = &lat - lon := EncodeLatLon(node.Lon, *block.LonOffset, block.GetGranularity()) + lon := EncodeLatLon(node.Lon, block.GetLonOffset(), block.GetGranularity()) pbfNode.Lon = &lon if len(node.Tags) > 0 { From 80feaae93192e39cbe7aa72286c6131681b48039 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Sat, 2 Sep 2023 18:30:15 +0200 Subject: [PATCH 13/29] Readd tags --- osmpbf/encode.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/osmpbf/encode.go b/osmpbf/encode.go index bcbeb84b..5d4f892e 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -253,13 +253,13 @@ func EncodeDenseNode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string groupDense.Lat = append(groupDense.Lat, latDiff) groupDense.Lon = append(groupDense.Lon, lonDiff) - // if current.Tags != nil { - // for _, nodeTag := range current.Tags { - // groupDense.KeysVals = append(groupDense.KeysVals, EncodeString(block, reverseStringTable, nodeTag.Key)) - // groupDense.KeysVals = append(groupDense.KeysVals, EncodeString(block, reverseStringTable, nodeTag.Value)) - // } - // groupDense.KeysVals = append(groupDense.KeysVals, 0) - // } + if current.Tags != nil { + for _, nodeTag := range current.Tags { + groupDense.KeysVals = append(groupDense.KeysVals, EncodeString(block, reverseStringTable, nodeTag.Key)) + groupDense.KeysVals = append(groupDense.KeysVals, EncodeString(block, reverseStringTable, nodeTag.Value)) + } + groupDense.KeysVals = append(groupDense.KeysVals, 0) + } // if groupDense.Denseinfo != nil { // groupDense.Denseinfo.Changeset = append(groupDense.Denseinfo.Changeset, int64(current.ChangesetID-previous.ChangesetID)) From 768d6e38b249a022c23ea120980a4913d2084f1c Mon Sep 17 00:00:00 2001 From: larsbeck Date: Sat, 2 Sep 2023 18:33:03 +0200 Subject: [PATCH 14/29] Fix tags --- osmpbf/encode.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osmpbf/encode.go b/osmpbf/encode.go index 5d4f892e..ab3c247b 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -253,7 +253,7 @@ func EncodeDenseNode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string groupDense.Lat = append(groupDense.Lat, latDiff) groupDense.Lon = append(groupDense.Lon, lonDiff) - if current.Tags != nil { + if len(current.Tags) > 0 { for _, nodeTag := range current.Tags { groupDense.KeysVals = append(groupDense.KeysVals, EncodeString(block, reverseStringTable, nodeTag.Key)) groupDense.KeysVals = append(groupDense.KeysVals, EncodeString(block, reverseStringTable, nodeTag.Value)) From f47a97543f1a864761a2b800a61d411d354c8241 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Sat, 2 Sep 2023 18:33:52 +0200 Subject: [PATCH 15/29] Write 0 in any case --- osmpbf/encode.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osmpbf/encode.go b/osmpbf/encode.go index ab3c247b..217847f1 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -258,8 +258,8 @@ func EncodeDenseNode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string groupDense.KeysVals = append(groupDense.KeysVals, EncodeString(block, reverseStringTable, nodeTag.Key)) groupDense.KeysVals = append(groupDense.KeysVals, EncodeString(block, reverseStringTable, nodeTag.Value)) } - groupDense.KeysVals = append(groupDense.KeysVals, 0) } + groupDense.KeysVals = append(groupDense.KeysVals, 0) // if groupDense.Denseinfo != nil { // groupDense.Denseinfo.Changeset = append(groupDense.Denseinfo.Changeset, int64(current.ChangesetID-previous.ChangesetID)) From 42429ab8c9717fb67a9101ad7d6859b2e7f76e56 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Sat, 2 Sep 2023 18:35:38 +0200 Subject: [PATCH 16/29] Add denseinfo --- osmpbf/encode.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/osmpbf/encode.go b/osmpbf/encode.go index 217847f1..915b35c6 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -261,21 +261,21 @@ func EncodeDenseNode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string } groupDense.KeysVals = append(groupDense.KeysVals, 0) - // if groupDense.Denseinfo != nil { - // groupDense.Denseinfo.Changeset = append(groupDense.Denseinfo.Changeset, int64(current.ChangesetID-previous.ChangesetID)) - // dateGranularity := block.GetDateGranularity() - // currentTimeStamp := EncodeTimestamp(current.Timestamp, dateGranularity) - // previousTimeStamp := EncodeTimestamp(previous.Timestamp, dateGranularity) - // groupDense.Denseinfo.Timestamp = append(groupDense.Denseinfo.Timestamp, currentTimeStamp-previousTimeStamp) - // groupDense.Denseinfo.Uid = append(groupDense.Denseinfo.Uid, int32(current.UserID-previous.UserID)) - // groupDense.Denseinfo.Version = append(groupDense.Denseinfo.Version, int32(current.Version-previous.Version)) - // var previousUserNameId int32 = 0 - // if previous.User != "" { - // previousUserNameId = EncodeString(block, reverseStringTable, previous.User) - // } - // currentUserNameId := EncodeString(block, reverseStringTable, current.User) - // groupDense.Denseinfo.UserSid = append(groupDense.Denseinfo.UserSid, currentUserNameId-previousUserNameId) - // } + if groupDense.Denseinfo != nil { + groupDense.Denseinfo.Changeset = append(groupDense.Denseinfo.Changeset, int64(current.ChangesetID-previous.ChangesetID)) + dateGranularity := block.GetDateGranularity() + currentTimeStamp := EncodeTimestamp(current.Timestamp, dateGranularity) + previousTimeStamp := EncodeTimestamp(previous.Timestamp, dateGranularity) + groupDense.Denseinfo.Timestamp = append(groupDense.Denseinfo.Timestamp, currentTimeStamp-previousTimeStamp) + groupDense.Denseinfo.Uid = append(groupDense.Denseinfo.Uid, int32(current.UserID-previous.UserID)) + groupDense.Denseinfo.Version = append(groupDense.Denseinfo.Version, int32(current.Version-previous.Version)) + var previousUserNameId int32 = 0 + if previous.User != "" { + previousUserNameId = EncodeString(block, reverseStringTable, previous.User) + } + currentUserNameId := EncodeString(block, reverseStringTable, current.User) + groupDense.Denseinfo.UserSid = append(groupDense.Denseinfo.UserSid, currentUserNameId-previousUserNameId) + } } func EncodeNode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, pbfNode *osmpbf.Node, node *osm.Node) *osmpbf.Node { From fad147b7397ce3478ab3caf8a1e3441d86b189c6 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Sat, 2 Sep 2023 18:38:33 +0200 Subject: [PATCH 17/29] Rearrange fields --- osmpbf/encode.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osmpbf/encode.go b/osmpbf/encode.go index 915b35c6..db3de506 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -45,9 +45,9 @@ func NewWriter(w io.Writer) (Writer, error) { type encoder struct { stream io.Writer reverseStringTable map[string]int + lastWrittenType osm.Type entities []osm.Object mu sync.Mutex - lastWrittenType osm.Type compress bool } From 44f1f4ec7cce5f26ebdf8ae8eee8df83583ffd89 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Wed, 6 Sep 2023 16:07:41 +0200 Subject: [PATCH 18/29] Rename writer to encoder --- osmpbf/encode.go | 20 ++++++++++---------- osmpbf/encode_test.go | 8 ++++---- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/osmpbf/encode.go b/osmpbf/encode.go index db3de506..e869dc6e 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -12,16 +12,16 @@ import ( "google.golang.org/protobuf/proto" ) -// Writer is an interface for writing osm data. The format written is the osm +// Encoder is an interface for encoding osm data. The format written is the osm // pbf format. -type Writer interface { +type Encoder interface { io.Closer - WriteObject(obj osm.Object) error + Encode(obj osm.Object) error } -func NewWriter(w io.Writer) (Writer, error) { +func NewEncoder(w io.Writer) (Encoder, error) { encoder := &encoder{ - stream: w, + writer: w, reverseStringTable: make(map[string]int), compress: true, } @@ -43,7 +43,7 @@ func NewWriter(w io.Writer) (Writer, error) { } type encoder struct { - stream io.Writer + writer io.Writer reverseStringTable map[string]int lastWrittenType osm.Type entities []osm.Object @@ -107,18 +107,18 @@ func (e *encoder) write(data []byte, osmType string) (n int, err error) { size := make([]byte, 4) binary.BigEndian.PutUint32(size, uint32(len(blobHeaderData))) - if _, err = e.stream.Write(size); err != nil { + if _, err = e.writer.Write(size); err != nil { return 0, nil } - if _, err = e.stream.Write(blobHeaderData); err != nil { + if _, err = e.writer.Write(blobHeaderData); err != nil { return 0, nil } - return e.stream.Write(blobData) + return e.writer.Write(blobData) } -func (e *encoder) WriteObject(obj osm.Object) error { +func (e *encoder) Encode(obj osm.Object) error { e.mu.Lock() defer e.mu.Unlock() diff --git a/osmpbf/encode_test.go b/osmpbf/encode_test.go index 7f0a4846..c1089d51 100644 --- a/osmpbf/encode_test.go +++ b/osmpbf/encode_test.go @@ -13,23 +13,23 @@ import ( func TestEncodeDecode(t *testing.T) { buffer := bytes.Buffer{} - writer, err := NewWriter(&buffer) + writer, err := NewEncoder(&buffer) if err != nil { t.Fatal(err) } // write node - err = writer.WriteObject(en) + err = writer.Encode(en) if err != nil { t.Fatal(err) } // write way - err = writer.WriteObject(ew) + err = writer.Encode(ew) if err != nil { t.Fatal(err) } // write relation - err = writer.WriteObject(er) + err = writer.Encode(er) if err != nil { t.Fatal(err) } From 9eeab800374c8551126ba4e33fa7f9ff2910a881 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Mon, 13 Nov 2023 10:47:02 +0100 Subject: [PATCH 19/29] Add OSM encoder --- annotate/change.go | 4 ++-- annotate/change_test.go | 2 +- annotate/datasource.go | 6 +++--- annotate/edgecases_test.go | 2 +- annotate/errors.go | 4 ++-- annotate/errors_test.go | 2 +- annotate/geo.go | 4 ++-- annotate/geo_test.go | 2 +- annotate/internal/core/compute.go | 4 ++-- annotate/internal/core/compute_test.go | 4 ++-- annotate/internal/core/datasourcer_test.go | 2 +- annotate/internal/core/errors.go | 2 +- annotate/internal/core/types.go | 4 ++-- annotate/internal/core/types_test.go | 4 ++-- annotate/options.go | 4 ++-- annotate/order.go | 2 +- annotate/order_test.go | 2 +- annotate/relation.go | 6 +++--- annotate/relation_test.go | 2 +- annotate/shared/child.go | 2 +- annotate/way.go | 6 +++--- annotate/way_test.go | 2 +- go.mod | 2 +- internal/mputil/mputil.go | 2 +- internal/mputil/mputil_test.go | 2 +- json.go | 4 ++-- osmapi/changeset.go | 2 +- osmapi/datasource.go | 2 +- osmapi/live_test.go | 2 +- osmapi/map.go | 2 +- osmapi/map_test.go | 2 +- osmapi/node.go | 2 +- osmapi/node_test.go | 2 +- osmapi/note.go | 2 +- osmapi/note_test.go | 2 +- osmapi/relation.go | 2 +- osmapi/relation_test.go | 2 +- osmapi/user.go | 2 +- osmapi/way.go | 2 +- osmapi/way_test.go | 2 +- osmgeojson/benchmarks_test.go | 2 +- osmgeojson/build_polygon.go | 4 ++-- osmgeojson/build_polygon_test.go | 2 +- osmgeojson/convert.go | 4 ++-- osmgeojson/convert_test.go | 2 +- osmgeojson/options_test.go | 2 +- osmpbf/decode.go | 4 ++-- osmpbf/decode_data.go | 4 ++-- osmpbf/decode_test.go | 2 +- osmpbf/encode.go | 4 ++-- osmpbf/encode_test.go | 2 +- osmpbf/example_stats_test.go | 4 ++-- osmpbf/internal/osmpbf/generate.go | 2 +- osmpbf/scanner.go | 2 +- osmpbf/scanner_test.go | 2 +- osmtest/scanner.go | 2 +- osmtest/scanner_test.go | 2 +- osmxml/example_test.go | 4 ++-- osmxml/scanner.go | 2 +- osmxml/scanner_test.go | 2 +- replication/changesets.go | 4 ++-- replication/interval.go | 2 +- testdata/compare_scanners.go | 6 +++--- 63 files changed, 88 insertions(+), 88 deletions(-) diff --git a/annotate/change.go b/annotate/change.go index 421988a4..a011676e 100644 --- a/annotate/change.go +++ b/annotate/change.go @@ -3,8 +3,8 @@ package annotate import ( "context" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/annotate/internal/core" + "github.com/paulmach/osm" + "github.com/paulmach/osm/annotate/internal/core" ) // Change will annotate a change into a diff. It will use the diff --git a/annotate/change_test.go b/annotate/change_test.go index fc7a5e44..5706b589 100644 --- a/annotate/change_test.go +++ b/annotate/change_test.go @@ -4,7 +4,7 @@ import ( "context" "testing" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) func TestChange_create(t *testing.T) { diff --git a/annotate/datasource.go b/annotate/datasource.go index 0fdff5e6..41102f9a 100644 --- a/annotate/datasource.go +++ b/annotate/datasource.go @@ -3,9 +3,9 @@ package annotate import ( "context" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/annotate/internal/core" - "github.com/nextmv-io/osm/annotate/shared" + "github.com/paulmach/osm" + "github.com/paulmach/osm/annotate/internal/core" + "github.com/paulmach/osm/annotate/shared" "github.com/paulmach/orb" "github.com/paulmach/orb/planar" diff --git a/annotate/edgecases_test.go b/annotate/edgecases_test.go index b1e7a8a0..09c1310b 100644 --- a/annotate/edgecases_test.go +++ b/annotate/edgecases_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) func TestEdgeCase_childCreatedAfterParent(t *testing.T) { diff --git a/annotate/errors.go b/annotate/errors.go index 864c8562..13a7d08f 100644 --- a/annotate/errors.go +++ b/annotate/errors.go @@ -4,8 +4,8 @@ import ( "fmt" "time" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/annotate/internal/core" + "github.com/paulmach/osm" + "github.com/paulmach/osm/annotate/internal/core" ) // NoHistoryError is returned if there is no entry in the history diff --git a/annotate/errors_test.go b/annotate/errors_test.go index e4f69cea..a4ec64c8 100644 --- a/annotate/errors_test.go +++ b/annotate/errors_test.go @@ -4,7 +4,7 @@ import ( "errors" "testing" - "github.com/nextmv-io/osm/annotate/internal/core" + "github.com/paulmach/osm/annotate/internal/core" ) func TestMapErrors(t *testing.T) { diff --git a/annotate/geo.go b/annotate/geo.go index ae1ee69f..91ba7abd 100644 --- a/annotate/geo.go +++ b/annotate/geo.go @@ -4,10 +4,10 @@ import ( "math" "time" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/internal/mputil" "github.com/paulmach/orb" "github.com/paulmach/orb/geo" + "github.com/paulmach/osm" + "github.com/paulmach/osm/internal/mputil" ) func wayPointOnSurface(w *osm.Way) orb.Point { diff --git a/annotate/geo_test.go b/annotate/geo_test.go index 818db4ed..bc697d20 100644 --- a/annotate/geo_test.go +++ b/annotate/geo_test.go @@ -4,8 +4,8 @@ import ( "encoding/xml" "testing" - "github.com/nextmv-io/osm" "github.com/paulmach/orb" + "github.com/paulmach/osm" ) func TestWayPointOnSurface(t *testing.T) { diff --git a/annotate/internal/core/compute.go b/annotate/internal/core/compute.go index 46966623..531b815d 100644 --- a/annotate/internal/core/compute.go +++ b/annotate/internal/core/compute.go @@ -5,8 +5,8 @@ import ( "fmt" "time" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/annotate/shared" + "github.com/paulmach/osm" + "github.com/paulmach/osm/annotate/shared" ) // A Datasourcer is something that acts like a datasource allowing us to diff --git a/annotate/internal/core/compute_test.go b/annotate/internal/core/compute_test.go index 8d02f9ef..c8fe94e9 100644 --- a/annotate/internal/core/compute_test.go +++ b/annotate/internal/core/compute_test.go @@ -7,8 +7,8 @@ import ( "testing" "time" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/annotate/shared" + "github.com/paulmach/osm" + "github.com/paulmach/osm/annotate/shared" ) var ( diff --git a/annotate/internal/core/datasourcer_test.go b/annotate/internal/core/datasourcer_test.go index c522d183..4a2fb4aa 100644 --- a/annotate/internal/core/datasourcer_test.go +++ b/annotate/internal/core/datasourcer_test.go @@ -4,7 +4,7 @@ import ( "context" "errors" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) var _ Datasourcer = &TestDS{} diff --git a/annotate/internal/core/errors.go b/annotate/internal/core/errors.go index 2bba17a3..451e3ee7 100644 --- a/annotate/internal/core/errors.go +++ b/annotate/internal/core/errors.go @@ -4,7 +4,7 @@ import ( "fmt" "time" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) // NoHistoryError is returned if there is no entry in the history diff --git a/annotate/internal/core/types.go b/annotate/internal/core/types.go index 8fcc627e..87fb037d 100644 --- a/annotate/internal/core/types.go +++ b/annotate/internal/core/types.go @@ -3,8 +3,8 @@ package core import ( "time" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/annotate/shared" + "github.com/paulmach/osm" + "github.com/paulmach/osm/annotate/shared" ) // A Parent is something that holds children. ie. ways have nodes as children diff --git a/annotate/internal/core/types_test.go b/annotate/internal/core/types_test.go index 44754dcc..c5ef266a 100644 --- a/annotate/internal/core/types_test.go +++ b/annotate/internal/core/types_test.go @@ -4,8 +4,8 @@ import ( "testing" "time" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/annotate/shared" + "github.com/paulmach/osm" + "github.com/paulmach/osm/annotate/shared" ) type findVisibleTestCase struct { diff --git a/annotate/options.go b/annotate/options.go index 142aacca..4251de96 100644 --- a/annotate/options.go +++ b/annotate/options.go @@ -3,8 +3,8 @@ package annotate import ( "time" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/annotate/internal/core" + "github.com/paulmach/osm" + "github.com/paulmach/osm/annotate/internal/core" ) // Option is a parameter that can be used for annotating. diff --git a/annotate/order.go b/annotate/order.go index c324b803..022fe127 100644 --- a/annotate/order.go +++ b/annotate/order.go @@ -4,7 +4,7 @@ import ( "context" "sync" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) // RelationHistoryDatasourcer is an more strict interface for when we only need the relation history. diff --git a/annotate/order_test.go b/annotate/order_test.go index e29716c3..f9f7010f 100644 --- a/annotate/order_test.go +++ b/annotate/order_test.go @@ -4,7 +4,7 @@ import ( "context" "testing" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) func TestChildFirstOrdering(t *testing.T) { diff --git a/annotate/relation.go b/annotate/relation.go index 6de0d68d..e1fe2bbe 100644 --- a/annotate/relation.go +++ b/annotate/relation.go @@ -4,9 +4,9 @@ import ( "context" "time" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/annotate/internal/core" - "github.com/nextmv-io/osm/annotate/shared" + "github.com/paulmach/osm" + "github.com/paulmach/osm/annotate/internal/core" + "github.com/paulmach/osm/annotate/shared" ) // HistoryAsChildrenDatasourcer is an advanced data source that diff --git a/annotate/relation_test.go b/annotate/relation_test.go index 8eb2e41d..1619bbe6 100644 --- a/annotate/relation_test.go +++ b/annotate/relation_test.go @@ -9,8 +9,8 @@ import ( "testing" "time" - "github.com/nextmv-io/osm" "github.com/paulmach/orb" + "github.com/paulmach/osm" ) func TestRelation(t *testing.T) { diff --git a/annotate/shared/child.go b/annotate/shared/child.go index 0881f4e7..fba94709 100644 --- a/annotate/shared/child.go +++ b/annotate/shared/child.go @@ -5,7 +5,7 @@ package shared import ( "time" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) // A Child represents a node, way or relation that is a dependent for diff --git a/annotate/way.go b/annotate/way.go index cb08f045..7e0bc2eb 100644 --- a/annotate/way.go +++ b/annotate/way.go @@ -4,9 +4,9 @@ import ( "context" "time" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/annotate/internal/core" - "github.com/nextmv-io/osm/annotate/shared" + "github.com/paulmach/osm" + "github.com/paulmach/osm/annotate/internal/core" + "github.com/paulmach/osm/annotate/shared" ) // NodeHistoryDatasourcer is an more strict interface for when we only need node history. diff --git a/annotate/way_test.go b/annotate/way_test.go index 6dae0f63..47cfc3ec 100644 --- a/annotate/way_test.go +++ b/annotate/way_test.go @@ -8,7 +8,7 @@ import ( "reflect" "testing" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) func TestWays(t *testing.T) { diff --git a/go.mod b/go.mod index 53234598..5ea8c278 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/nextmv-io/osm +module github.com/paulmach/osm go 1.13 diff --git a/internal/mputil/mputil.go b/internal/mputil/mputil.go index a4719da3..3e8693de 100644 --- a/internal/mputil/mputil.go +++ b/internal/mputil/mputil.go @@ -3,8 +3,8 @@ package mputil import ( "time" - "github.com/nextmv-io/osm" "github.com/paulmach/orb" + "github.com/paulmach/osm" ) // Segment is a section of a multipolygon with some extra information diff --git a/internal/mputil/mputil_test.go b/internal/mputil/mputil_test.go index b6a7c3de..219573fa 100644 --- a/internal/mputil/mputil_test.go +++ b/internal/mputil/mputil_test.go @@ -5,8 +5,8 @@ import ( "testing" "time" - "github.com/nextmv-io/osm" "github.com/paulmach/orb" + "github.com/paulmach/osm" ) func TestMultiSegment_LineString(t *testing.T) { diff --git a/json.go b/json.go index 423931ce..593f8f9e 100644 --- a/json.go +++ b/json.go @@ -12,7 +12,7 @@ import ( // // import ( // jsoniter "github.com/json-iterator/go" -// "github.com/nextmv-io/osm" +// "github.com/paulmach/osm" // ) // // var c = jsoniter.Config{ @@ -36,7 +36,7 @@ var CustomJSONMarshaler interface { // // import ( // jsoniter "github.com/json-iterator/go" -// "github.com/nextmv-io/osm" +// "github.com/paulmach/osm" // ) // // var c = jsoniter.Config{ diff --git a/osmapi/changeset.go b/osmapi/changeset.go index 0610a3a0..959b4f4c 100644 --- a/osmapi/changeset.go +++ b/osmapi/changeset.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) // Changeset returns a given changeset from the osm rest api. diff --git a/osmapi/datasource.go b/osmapi/datasource.go index 3ad115fc..cc06a252 100644 --- a/osmapi/datasource.go +++ b/osmapi/datasource.go @@ -8,7 +8,7 @@ import ( "net/http" "time" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) // BaseURL defines the api host. This can be change to hit diff --git a/osmapi/live_test.go b/osmapi/live_test.go index 6d100dc3..f09070bb 100644 --- a/osmapi/live_test.go +++ b/osmapi/live_test.go @@ -5,7 +5,7 @@ import ( "os" "testing" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" "golang.org/x/time/rate" ) diff --git a/osmapi/map.go b/osmapi/map.go index 5ee833b8..d0b3a93b 100644 --- a/osmapi/map.go +++ b/osmapi/map.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) // Map returns the latest elements in the given bounding box. diff --git a/osmapi/map_test.go b/osmapi/map_test.go index b4f7bc44..c8292257 100644 --- a/osmapi/map_test.go +++ b/osmapi/map_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) func TestMap_urls(t *testing.T) { diff --git a/osmapi/node.go b/osmapi/node.go index 3a42d6db..13140c3b 100644 --- a/osmapi/node.go +++ b/osmapi/node.go @@ -5,7 +5,7 @@ import ( "fmt" "strconv" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) // Node returns the latest version of the node from the osm rest api. diff --git a/osmapi/node_test.go b/osmapi/node_test.go index 93254627..12b541f4 100644 --- a/osmapi/node_test.go +++ b/osmapi/node_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) func TestNode_urls(t *testing.T) { diff --git a/osmapi/note.go b/osmapi/note.go index 72b718d0..c903e563 100644 --- a/osmapi/note.go +++ b/osmapi/note.go @@ -6,7 +6,7 @@ import ( "net/url" "strings" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) // Note returns the note from the osm rest api. diff --git a/osmapi/note_test.go b/osmapi/note_test.go index 3d193f5f..ec93f432 100644 --- a/osmapi/note_test.go +++ b/osmapi/note_test.go @@ -7,7 +7,7 @@ import ( "strings" "testing" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) func TestNote_urls(t *testing.T) { diff --git a/osmapi/relation.go b/osmapi/relation.go index 5e767c70..e629e0bc 100644 --- a/osmapi/relation.go +++ b/osmapi/relation.go @@ -5,7 +5,7 @@ import ( "fmt" "strconv" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) // Relation returns the latest version of the relation from the osm rest api. diff --git a/osmapi/relation_test.go b/osmapi/relation_test.go index b64cbc37..bea89341 100644 --- a/osmapi/relation_test.go +++ b/osmapi/relation_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) func TestRelation_urls(t *testing.T) { diff --git a/osmapi/user.go b/osmapi/user.go index 3b01ce81..17a7bff5 100644 --- a/osmapi/user.go +++ b/osmapi/user.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) // User returns the user from the osm rest api. diff --git a/osmapi/way.go b/osmapi/way.go index 9d75d2d1..5f834fd0 100644 --- a/osmapi/way.go +++ b/osmapi/way.go @@ -5,7 +5,7 @@ import ( "fmt" "strconv" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) // Way returns the latest version of the way from the osm rest api. diff --git a/osmapi/way_test.go b/osmapi/way_test.go index 6377415f..050357ee 100644 --- a/osmapi/way_test.go +++ b/osmapi/way_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) func TestWay_urls(t *testing.T) { diff --git a/osmgeojson/benchmarks_test.go b/osmgeojson/benchmarks_test.go index 42cf0a3f..1dc71dd7 100644 --- a/osmgeojson/benchmarks_test.go +++ b/osmgeojson/benchmarks_test.go @@ -5,7 +5,7 @@ import ( "io/ioutil" "testing" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) func BenchmarkConvert(b *testing.B) { diff --git a/osmgeojson/build_polygon.go b/osmgeojson/build_polygon.go index 45c5721b..bc00284d 100644 --- a/osmgeojson/build_polygon.go +++ b/osmgeojson/build_polygon.go @@ -3,10 +3,10 @@ package osmgeojson import ( "fmt" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/internal/mputil" "github.com/paulmach/orb" "github.com/paulmach/orb/geojson" + "github.com/paulmach/osm" + "github.com/paulmach/osm/internal/mputil" ) func (ctx *context) buildPolygon(relation *osm.Relation) *geojson.Feature { diff --git a/osmgeojson/build_polygon_test.go b/osmgeojson/build_polygon_test.go index fc684b88..d80750c9 100644 --- a/osmgeojson/build_polygon_test.go +++ b/osmgeojson/build_polygon_test.go @@ -4,9 +4,9 @@ import ( "encoding/xml" "testing" - "github.com/nextmv-io/osm" "github.com/paulmach/orb" "github.com/paulmach/orb/geojson" + "github.com/paulmach/osm" ) func TestConvert_polygon(t *testing.T) { diff --git a/osmgeojson/convert.go b/osmgeojson/convert.go index e0c8cf19..ddc2013b 100644 --- a/osmgeojson/convert.go +++ b/osmgeojson/convert.go @@ -3,10 +3,10 @@ package osmgeojson import ( "fmt" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/internal/mputil" "github.com/paulmach/orb" "github.com/paulmach/orb/geojson" + "github.com/paulmach/osm" + "github.com/paulmach/osm/internal/mputil" ) type context struct { diff --git a/osmgeojson/convert_test.go b/osmgeojson/convert_test.go index fbee4bf5..3209d0ec 100644 --- a/osmgeojson/convert_test.go +++ b/osmgeojson/convert_test.go @@ -6,9 +6,9 @@ import ( "reflect" "testing" - "github.com/nextmv-io/osm" "github.com/paulmach/orb" "github.com/paulmach/orb/geojson" + "github.com/paulmach/osm" ) func TestConvert(t *testing.T) { diff --git a/osmgeojson/options_test.go b/osmgeojson/options_test.go index 80148247..2d74b964 100644 --- a/osmgeojson/options_test.go +++ b/osmgeojson/options_test.go @@ -4,8 +4,8 @@ import ( "encoding/xml" "testing" - "github.com/nextmv-io/osm" "github.com/paulmach/orb/geojson" + "github.com/paulmach/osm" ) var nodeXML = ` diff --git a/osmpbf/decode.go b/osmpbf/decode.go index 54473990..eec3892a 100644 --- a/osmpbf/decode.go +++ b/osmpbf/decode.go @@ -10,8 +10,8 @@ import ( "sync" "time" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/osmpbf/internal/osmpbf" + "github.com/paulmach/osm" + "github.com/paulmach/osm/osmpbf/internal/osmpbf" "google.golang.org/protobuf/proto" ) diff --git a/osmpbf/decode_data.go b/osmpbf/decode_data.go index ba782721..1d652b38 100644 --- a/osmpbf/decode_data.go +++ b/osmpbf/decode_data.go @@ -4,8 +4,8 @@ import ( "errors" "time" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/osmpbf/internal/osmpbf" + "github.com/paulmach/osm" + "github.com/paulmach/osm/osmpbf/internal/osmpbf" "github.com/paulmach/protoscan" "google.golang.org/protobuf/proto" ) diff --git a/osmpbf/decode_test.go b/osmpbf/decode_test.go index 8a4a8698..6b6785c6 100644 --- a/osmpbf/decode_test.go +++ b/osmpbf/decode_test.go @@ -12,7 +12,7 @@ import ( "testing" "time" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) const ( diff --git a/osmpbf/encode.go b/osmpbf/encode.go index e869dc6e..9a32ea37 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -7,8 +7,8 @@ import ( "sync" "time" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/osmpbf/internal/osmpbf" + "github.com/paulmach/osm" + "github.com/paulmach/osm/osmpbf/internal/osmpbf" "google.golang.org/protobuf/proto" ) diff --git a/osmpbf/encode_test.go b/osmpbf/encode_test.go index c1089d51..8f1e86a4 100644 --- a/osmpbf/encode_test.go +++ b/osmpbf/encode_test.go @@ -8,7 +8,7 @@ import ( "runtime" "testing" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) func TestEncodeDecode(t *testing.T) { diff --git a/osmpbf/example_stats_test.go b/osmpbf/example_stats_test.go index b9d7a049..9a8182f9 100644 --- a/osmpbf/example_stats_test.go +++ b/osmpbf/example_stats_test.go @@ -7,8 +7,8 @@ import ( "os" "time" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/osmpbf" + "github.com/paulmach/osm" + "github.com/paulmach/osm/osmpbf" ) // ExampleStats demonstrates how to read a full file and gather some stats. diff --git a/osmpbf/internal/osmpbf/generate.go b/osmpbf/internal/osmpbf/generate.go index a330ef3f..8eee6f9b 100644 --- a/osmpbf/internal/osmpbf/generate.go +++ b/osmpbf/internal/osmpbf/generate.go @@ -1,3 +1,3 @@ package osmpbf -//go:generate protoc --proto_path=. --go_opt=module=github.com/nextmv-io/osmpbf/internal/osmpbf --go_out=. fileformat.proto osmformat.proto +//go:generate protoc --proto_path=. --go_opt=module=github.com/paulmach/osmpbf/internal/osmpbf --go_out=. fileformat.proto osmformat.proto diff --git a/osmpbf/scanner.go b/osmpbf/scanner.go index 201b7a43..b0479029 100644 --- a/osmpbf/scanner.go +++ b/osmpbf/scanner.go @@ -5,7 +5,7 @@ import ( "io" "sync/atomic" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) var _ osm.Scanner = &Scanner{} diff --git a/osmpbf/scanner_test.go b/osmpbf/scanner_test.go index 70d9e6db..fd47c082 100644 --- a/osmpbf/scanner_test.go +++ b/osmpbf/scanner_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) var ( diff --git a/osmtest/scanner.go b/osmtest/scanner.go index 3a35188e..1119c865 100644 --- a/osmtest/scanner.go +++ b/osmtest/scanner.go @@ -1,6 +1,6 @@ package osmtest -import "github.com/nextmv-io/osm" +import "github.com/paulmach/osm" // Scanner implements the osm.Scanner interface with // just a list of objects. diff --git a/osmtest/scanner_test.go b/osmtest/scanner_test.go index fff9f7e6..f3c68207 100644 --- a/osmtest/scanner_test.go +++ b/osmtest/scanner_test.go @@ -5,7 +5,7 @@ import ( "reflect" "testing" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) func TestScanner(t *testing.T) { diff --git a/osmxml/example_test.go b/osmxml/example_test.go index 37979075..61700dca 100644 --- a/osmxml/example_test.go +++ b/osmxml/example_test.go @@ -5,8 +5,8 @@ import ( "fmt" "os" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/osmxml" + "github.com/paulmach/osm" + "github.com/paulmach/osm/osmxml" ) func ExampleScanner() { diff --git a/osmxml/scanner.go b/osmxml/scanner.go index c0c91926..bdd0efe3 100644 --- a/osmxml/scanner.go +++ b/osmxml/scanner.go @@ -6,7 +6,7 @@ import ( "io" "strings" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) var _ osm.Scanner = &Scanner{} diff --git a/osmxml/scanner_test.go b/osmxml/scanner_test.go index daf7dc65..fd85aa7a 100644 --- a/osmxml/scanner_test.go +++ b/osmxml/scanner_test.go @@ -8,7 +8,7 @@ import ( "os" "testing" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) func TestScanner(t *testing.T) { diff --git a/replication/changesets.go b/replication/changesets.go index f9132a59..f149c8cf 100644 --- a/replication/changesets.go +++ b/replication/changesets.go @@ -10,8 +10,8 @@ import ( "net/http" "strconv" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/osmxml" + "github.com/paulmach/osm" + "github.com/paulmach/osm/osmxml" ) // ChangesetSeqNum indicates the sequence of the changeset replication found here: diff --git a/replication/interval.go b/replication/interval.go index 0193688a..8c258bd8 100644 --- a/replication/interval.go +++ b/replication/interval.go @@ -11,7 +11,7 @@ import ( "strconv" "time" - "github.com/nextmv-io/osm" + "github.com/paulmach/osm" ) var _ SeqNum = MinuteSeqNum(0) diff --git a/testdata/compare_scanners.go b/testdata/compare_scanners.go index 7ab9aeff..f019dd7c 100644 --- a/testdata/compare_scanners.go +++ b/testdata/compare_scanners.go @@ -8,9 +8,9 @@ import ( "os" "reflect" - "github.com/nextmv-io/osm" - "github.com/nextmv-io/osm/osmpbf" - "github.com/nextmv-io/osm/osmxml" + "github.com/paulmach/osm" + "github.com/paulmach/osm/osmpbf" + "github.com/paulmach/osm/osmxml" ) func main() { From 48c0be54b812cd4365231b500dab94449ed5fc42 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Mon, 13 Nov 2023 10:52:23 +0100 Subject: [PATCH 20/29] Revert auto-formatting related changes --- annotate/options.go | 18 +++++++++--------- osmapi/datasource.go | 4 ++-- osmpbf/decode_data.go | 2 +- osmxml/scanner.go | 1 - 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/annotate/options.go b/annotate/options.go index 4251de96..789e19da 100644 --- a/annotate/options.go +++ b/annotate/options.go @@ -28,15 +28,15 @@ func Threshold(t time.Duration) Option { // This should be used when you want to gracefully handle the weird data in OSM. // // Nodes with unclear/inconsistent data will not be annotated. Causes include: -// - redacted data: In 2012 due to the license change data had to be removed. -// This could be some nodes of a way. There exist ways for which some nodes have -// just a single delete version, e.g. way 159081205, node 376130526 -// - data pre element versioning: pre-2012(?) data versions were not kept, so -// for old ways there many be no information about some nodes. For example, -// a node may be updated after a way and there is no way to get the original -// version of the node and way. -// - bad editors: sometimes a node is edited 7 times in a single changeset -// and version 5 is a delete. See node 321452894, part of way 28831147. +// - redacted data: In 2012 due to the license change data had to be removed. +// This could be some nodes of a way. There exist ways for which some nodes have +// just a single delete version, e.g. way 159081205, node 376130526 +// - data pre element versioning: pre-2012(?) data versions were not kept, so +// for old ways there many be no information about some nodes. For example, +// a node may be updated after a way and there is no way to get the original +// version of the node and way. +// - bad editors: sometimes a node is edited 7 times in a single changeset +// and version 5 is a delete. See node 321452894, part of way 28831147. func IgnoreInconsistency(yes bool) Option { return func(o *core.Options) error { o.IgnoreInconsistency = yes diff --git a/osmapi/datasource.go b/osmapi/datasource.go index cc06a252..28685832 100644 --- a/osmapi/datasource.go +++ b/osmapi/datasource.go @@ -19,8 +19,8 @@ const BaseURL = "http://api.openstreetmap.org/api/0.6" // This interface is met by `golang.org/x/time/rate.Limiter` and is meant // to be used with it. For example: // -// // 10 qps -// osmapi.DefaultDatasource.Limiter = rate.NewLimiter(10, 1) +// // 10 qps +// osmapi.DefaultDatasource.Limiter = rate.NewLimiter(10, 1) type RateLimiter interface { Wait(context.Context) error } diff --git a/osmpbf/decode_data.go b/osmpbf/decode_data.go index 1d652b38..1847a4ba 100644 --- a/osmpbf/decode_data.go +++ b/osmpbf/decode_data.go @@ -4,10 +4,10 @@ import ( "errors" "time" + "google.golang.org/protobuf/proto" "github.com/paulmach/osm" "github.com/paulmach/osm/osmpbf/internal/osmpbf" "github.com/paulmach/protoscan" - "google.golang.org/protobuf/proto" ) // dataDecoder is a decoder for Blob with OSMData (PrimitiveBlock). diff --git a/osmxml/scanner.go b/osmxml/scanner.go index bdd0efe3..b232d357 100644 --- a/osmxml/scanner.go +++ b/osmxml/scanner.go @@ -126,7 +126,6 @@ Loop: // Object returns the most recent token generated by a call to Scan // as a new osm.Object. This interface is implemented by: -// // *osm.Bounds // *osm.Node // *osm.Way From 73659be68233ded0ca0b7b65735acb45692f3d5b Mon Sep 17 00:00:00 2001 From: larsbeck Date: Mon, 13 Nov 2023 10:53:20 +0100 Subject: [PATCH 21/29] Remove auto-formatting newline comment --- osmapi/datasource.go | 1 - 1 file changed, 1 deletion(-) diff --git a/osmapi/datasource.go b/osmapi/datasource.go index 28685832..223e3b34 100644 --- a/osmapi/datasource.go +++ b/osmapi/datasource.go @@ -18,7 +18,6 @@ const BaseURL = "http://api.openstreetmap.org/api/0.6" // A RateLimiter is something that can wait until its next allowed request. // This interface is met by `golang.org/x/time/rate.Limiter` and is meant // to be used with it. For example: -// // // 10 qps // osmapi.DefaultDatasource.Limiter = rate.NewLimiter(10, 1) type RateLimiter interface { From 656c1278e821ed01e950794e0af9296fc6b52b0f Mon Sep 17 00:00:00 2001 From: Paul Mach Date: Wed, 27 Dec 2023 14:31:24 -0800 Subject: [PATCH 22/29] update github actions, add linter action --- .github/workflows/golangci-lint.yml | 24 ++++++++++++++++++++++++ .github/workflows/main.yml | 25 +++++++++---------------- 2 files changed, 33 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/golangci-lint.yml diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml new file mode 100644 index 00000000..3f541892 --- /dev/null +++ b/.github/workflows/golangci-lint.yml @@ -0,0 +1,24 @@ +name: golangci-lint +on: + push: + branches: + - master + pull_request: + +permissions: + contents: read + +jobs: + golangci: + name: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version: '1.21' + + - name: golangci-lint + uses: golangci/golangci-lint-action@v3 + with: + version: v1.55 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 70c7be37..17825701 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -4,35 +4,28 @@ on: push: branches: [ master ] pull_request: - branches: [ master ] jobs: build-and-test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - + - uses: actions/checkout@v3 + - name: Setup Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: - go-version: '1.15.3' - - - name: Install dependencies - run: | - go version - go get -u golang.org/x/lint/golint + go-version: '1.21' - name: Run build - run: go build . - - - name: Run vet & lint + run: go build . + + - name: Run vet run: | go vet . - golint . - + - name: Run tests run: go test -v -coverprofile=profile.cov ./... - + - name: codecov uses: codecov/codecov-action@v1 with: From 89f4b826a7c27cb4008694a77c616c05a4ab7b96 Mon Sep 17 00:00:00 2001 From: Paul Mach Date: Wed, 27 Dec 2023 16:21:15 -0800 Subject: [PATCH 23/29] fix golangci-lint errors --- .golangci.yml | 5 +++++ README.md | 6 ++++-- annotate/relation_test.go | 7 ++++-- annotate/way_test.go | 6 +++++- change_test.go | 17 --------------- diff_test.go | 10 +++++++-- osm_test.go | 40 ++--------------------------------- osmapi/options.go | 2 -- osmgeojson/benchmarks_test.go | 30 ++++++++++++++++++++------ osmpbf/decode_test.go | 33 ++++++++++++++--------------- 10 files changed, 69 insertions(+), 87 deletions(-) create mode 100644 .golangci.yml diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 00000000..8e5e857f --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,5 @@ +issues: + exclude-rules: + - path: '(.+)_test\.go' + linters: + - errcheck diff --git a/README.md b/README.md index b8e29f65..9d65d09c 100644 --- a/README.md +++ b/README.md @@ -113,13 +113,15 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/paulmach/osm" ) + var c = jsoniter.Config{ EscapeHTML: true, SortMapKeys: false, MarshalFloatWith6Digits: true, }.Froze() -CustomJSONMarshaler = c -CustomJSONUnmarshaler = c + +osmm.CustomJSONMarshaler = c +osm.CustomJSONUnmarshaler = c ``` The above change can have dramatic performance implications, see the benchmarks below diff --git a/annotate/relation_test.go b/annotate/relation_test.go index 1619bbe6..015c1441 100644 --- a/annotate/relation_test.go +++ b/annotate/relation_test.go @@ -4,7 +4,7 @@ import ( "context" "encoding/xml" "fmt" - "io/ioutil" + "os" "reflect" "testing" "time" @@ -42,7 +42,10 @@ func TestRelation(t *testing.T) { t.Errorf("expected relations not equal, file saved to %s", filename) data, _ := xml.MarshalIndent(&osm.OSM{Relations: relations}, "", " ") - ioutil.WriteFile(filename, data, 0644) + err := os.WriteFile(filename, data, 0644) + if err != nil { + t.Fatalf("write error: %v", err) + } } } } diff --git a/annotate/way_test.go b/annotate/way_test.go index 47cfc3ec..42d35ce4 100644 --- a/annotate/way_test.go +++ b/annotate/way_test.go @@ -5,6 +5,7 @@ import ( "encoding/xml" "fmt" "io/ioutil" + "os" "reflect" "testing" @@ -36,7 +37,10 @@ func TestWays(t *testing.T) { t.Errorf("expected way not equal, file saved to %s", filename) data, _ := xml.MarshalIndent(&osm.OSM{Ways: o.Ways}, "", " ") - ioutil.WriteFile(filename, data, 0644) + err := os.WriteFile(filename, data, 0644) + if err != nil { + t.Fatalf("write error: %v", err) + } } } } diff --git a/change_test.go b/change_test.go index 73c86a1e..444d754f 100644 --- a/change_test.go +++ b/change_test.go @@ -236,23 +236,6 @@ func TestChange_HistoryDatasource(t *testing.T) { } } -func cleanXMLNameFromChange(c *Change) { - c.Version = "" - c.Generator = "" - c.Copyright = "" - c.Attribution = "" - c.License = "" - if c.Create != nil { - cleanXMLNameFromOSM(c.Create) - } - if c.Modify != nil { - cleanXMLNameFromOSM(c.Modify) - } - if c.Delete != nil { - cleanXMLNameFromOSM(c.Delete) - } -} - func BenchmarkChange_MarshalXML(b *testing.B) { filename := "testdata/changeset_38162206.osc" data := readFile(b, filename) diff --git a/diff_test.go b/diff_test.go index 517b9cc1..0955747a 100644 --- a/diff_test.go +++ b/diff_test.go @@ -148,7 +148,10 @@ func BenchmarkDiff_Marshal(b *testing.B) { b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { - xml.Marshal(diff) + _, err := xml.Marshal(diff) + if err != nil { + b.Fatalf("marshal error: %v", err) + } } } @@ -159,6 +162,9 @@ func BenchmarkDiff_Unmarshal(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { diff := &Diff{} - xml.Unmarshal(data, &diff) + err := xml.Unmarshal(data, &diff) + if err != nil { + b.Fatalf("unmarshal error: %v", err) + } } } diff --git a/osm_test.go b/osm_test.go index e9500e03..f615e236 100644 --- a/osm_test.go +++ b/osm_test.go @@ -4,7 +4,7 @@ import ( "bytes" "encoding/json" "encoding/xml" - "io/ioutil" + "io" "os" "reflect" "testing" @@ -280,42 +280,6 @@ func TestOSM_MarshalXML(t *testing.T) { } } -func flattenOSM(c *Change) *OSM { - o := c.Create - if o == nil { - o = &OSM{} - } - - if c.Modify != nil { - o.Nodes = append(o.Nodes, c.Modify.Nodes...) - o.Ways = append(o.Ways, c.Modify.Ways...) - o.Relations = append(o.Relations, c.Modify.Relations...) - } - - if c.Delete != nil { - o.Nodes = append(o.Nodes, c.Delete.Nodes...) - o.Ways = append(o.Ways, c.Delete.Ways...) - o.Relations = append(o.Relations, c.Delete.Relations...) - } - - return o -} - -func cleanXMLNameFromOSM(o *OSM) { - for _, n := range o.Nodes { - n.XMLName = xmlNameJSONTypeNode{} - } - - for _, w := range o.Ways { - w.XMLName = xmlNameJSONTypeWay{} - } - - for _, r := range o.Relations { - // r.XMLName = xml.Name{} - r.XMLName = xmlNameJSONTypeRel{} - } -} - func readFile(t testing.TB, filename string) []byte { f, err := os.Open(filename) if err != nil { @@ -323,7 +287,7 @@ func readFile(t testing.TB, filename string) []byte { } defer f.Close() - data, err := ioutil.ReadAll(f) + data, err := io.ReadAll(f) if err != nil { t.Fatalf("unable to read file %s: %v", filename, err) } diff --git a/osmapi/options.go b/osmapi/options.go index c54d2b60..744a19b3 100644 --- a/osmapi/options.go +++ b/osmapi/options.go @@ -24,8 +24,6 @@ func (o *at) applyFeature(p []string) ([]string, error) { return append(p, "at="+o.t.UTC().Format("2006-01-02T15:04:05Z")), nil } -func (o *at) feature() {} - // NotesOption defines a valid option for the osmapi.Notes by bounding box api. type NotesOption interface { applyNotes([]string) ([]string, error) diff --git a/osmgeojson/benchmarks_test.go b/osmgeojson/benchmarks_test.go index 1dc71dd7..48771877 100644 --- a/osmgeojson/benchmarks_test.go +++ b/osmgeojson/benchmarks_test.go @@ -14,7 +14,10 @@ func BenchmarkConvert(b *testing.B) { b.ReportAllocs() b.ResetTimer() for n := 0; n < b.N; n++ { - Convert(o) + _, err := Convert(o) + if err != nil { + b.Fatalf("convert error: %v", err) + } } } @@ -25,7 +28,10 @@ func BenchmarkConvertAnnotated(b *testing.B) { b.ReportAllocs() b.ResetTimer() for n := 0; n < b.N; n++ { - Convert(o) + _, err := Convert(o) + if err != nil { + b.Fatalf("convert error: %v", err) + } } } @@ -35,7 +41,10 @@ func BenchmarkConvert_NoID(b *testing.B) { b.ReportAllocs() b.ResetTimer() for n := 0; n < b.N; n++ { - Convert(o, NoID(true)) + _, err := Convert(o, NoID(true)) + if err != nil { + b.Fatalf("convert error: %v", err) + } } } @@ -45,7 +54,10 @@ func BenchmarkConvert_NoMeta(b *testing.B) { b.ReportAllocs() b.ResetTimer() for n := 0; n < b.N; n++ { - Convert(o, NoMeta(true)) + _, err := Convert(o, NoMeta(true)) + if err != nil { + b.Fatalf("convert error: %v", err) + } } } @@ -55,7 +67,10 @@ func BenchmarkConvert_NoRelationMembership(b *testing.B) { b.ReportAllocs() b.ResetTimer() for n := 0; n < b.N; n++ { - Convert(o, NoRelationMembership(true)) + _, err := Convert(o, NoRelationMembership(true)) + if err != nil { + b.Fatalf("convert error: %v", err) + } } } @@ -65,7 +80,10 @@ func BenchmarkConvert_NoIDsMetaMembership(b *testing.B) { b.ReportAllocs() b.ResetTimer() for n := 0; n < b.N; n++ { - Convert(o, NoID(true), NoMeta(true), NoRelationMembership(true)) + _, err := Convert(o, NoID(true), NoMeta(true), NoRelationMembership(true)) + if err != nil { + b.Fatalf("convert error: %v", err) + } } } diff --git a/osmpbf/decode_test.go b/osmpbf/decode_test.go index 6b6785c6..5d7340df 100644 --- a/osmpbf/decode_test.go +++ b/osmpbf/decode_test.go @@ -175,25 +175,25 @@ func (ft *OSMFileTest) downloadTestOSMFile() { if _, err := os.Stat(ft.FileName); os.IsNotExist(err) { out, err := os.Create(ft.FileName) if err != nil { - ft.T.Fatal(err) + ft.Fatal(err) } defer out.Close() resp, err := http.Get(ft.FileURL) if err != nil { - ft.T.Fatal(err) + ft.Fatal(err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - ft.T.Fatalf("test status code invalid: %v", resp.StatusCode) + ft.Fatalf("test status code invalid: %v", resp.StatusCode) } if _, err := io.Copy(out, resp.Body); err != nil { - ft.T.Fatal(err) + ft.Fatal(err) } } else if err != nil { - ft.T.Fatal(err) + ft.Fatal(err) } } @@ -202,21 +202,20 @@ func (ft *OSMFileTest) testDecode() { f, err := os.Open(ft.FileName) if err != nil { - ft.T.Fatal(err) + ft.Fatal(err) } defer f.Close() d := newDecoder(context.Background(), &Scanner{}, f) err = d.Start(runtime.GOMAXPROCS(-1)) if err != nil { - ft.T.Fatal(err) + ft.Fatal(err) } var n *osm.Node var w *osm.Way var r *osm.Relation var nc, wc, rc uint64 - var id string idsOrder := make([]string, 0, len(IDsExpectedOrder)) for { e, err := d.Next() @@ -224,7 +223,7 @@ func (ft *OSMFileTest) testDecode() { if err == io.EOF { break } else if err != nil { - ft.T.Fatal(err) + ft.Fatal(err) } switch v := e.(type) { @@ -233,7 +232,7 @@ func (ft *OSMFileTest) testDecode() { if v.ID == ft.ExpNode.ID { n = v } - id = fmt.Sprintf("node/%d", v.ID) + id := fmt.Sprintf("node/%d", v.ID) if _, ok := IDs[id]; ok { idsOrder = append(idsOrder, id) } @@ -242,7 +241,7 @@ func (ft *OSMFileTest) testDecode() { if v.ID == ft.ExpWay.ID { w = v } - id = fmt.Sprintf("way/%d", v.ID) + id := fmt.Sprintf("way/%d", v.ID) if _, ok := IDs[id]; ok { idsOrder = append(idsOrder, id) } @@ -251,7 +250,7 @@ func (ft *OSMFileTest) testDecode() { if v.ID == ft.ExpRel.ID { r = v } - id = fmt.Sprintf("relation/%d", v.ID) + id := fmt.Sprintf("relation/%d", v.ID) if _, ok := IDs[id]; ok { idsOrder = append(idsOrder, id) } @@ -260,21 +259,21 @@ func (ft *OSMFileTest) testDecode() { d.Close() if !reflect.DeepEqual(ft.ExpNode, n) { - ft.T.Errorf("\nExpected: %#v\nActual: %#v", ft.ExpNode, n) + ft.Errorf("\nExpected: %#v\nActual: %#v", ft.ExpNode, n) } roundCoordinates(w) if !reflect.DeepEqual(ft.ExpWay, w) { - ft.T.Errorf("\nExpected: %#v\nActual: %#v", ft.ExpWay, w) + ft.Errorf("\nExpected: %#v\nActual: %#v", ft.ExpWay, w) } if !reflect.DeepEqual(ft.ExpRel, r) { - ft.T.Errorf("\nExpected: %#v\nActual: %#v", ft.ExpRel, r) + ft.Errorf("\nExpected: %#v\nActual: %#v", ft.ExpRel, r) } if ft.ExpNodeCount != nc || ft.ExpWayCount != wc || ft.ExpRelCount != rc { - ft.T.Errorf("\nExpected %7d nodes, %7d ways, %7d relations\nGot %7d nodes, %7d ways, %7d relations.", + ft.Errorf("\nExpected %7d nodes, %7d ways, %7d relations\nGot %7d nodes, %7d ways, %7d relations.", ft.ExpNodeCount, ft.ExpWayCount, ft.ExpRelCount, nc, wc, rc) } if !reflect.DeepEqual(ft.IDsExpOrder, idsOrder) { - ft.T.Errorf("\nExpected: %v\nGot: %v", ft.IDsExpOrder, idsOrder) + ft.Errorf("\nExpected: %v\nGot: %v", ft.IDsExpOrder, idsOrder) } } From b3aa927531c2a3f89fa4873442f40233baf33d98 Mon Sep 17 00:00:00 2001 From: Paul Mach Date: Wed, 27 Dec 2023 14:14:12 -0800 Subject: [PATCH 24/29] correctly unmarshal elements with a type tag --- osm.go | 19 +++++++++++++------ osm_test.go | 24 +++++++++++++++++++++++- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/osm.go b/osm.go index 76c3532e..ada25c2f 100644 --- a/osm.go +++ b/osm.go @@ -3,7 +3,6 @@ package osm import ( "encoding/xml" "fmt" - "regexp" ) // These values should be returned if the osm data is actual @@ -367,13 +366,21 @@ func (o *OSM) UnmarshalJSON(data []byte) error { return nil } -var jsonTypeRegexp = regexp.MustCompile(`"type"\s*:\s*"([^"]*)"`) +type typeStruct struct { + Type string `json:"type"` +} func findType(index int, data []byte) (string, error) { - matches := jsonTypeRegexp.FindAllSubmatch(data, 1) - if len(matches) > 0 { - return string(matches[0][1]), nil + ts := typeStruct{} + err := unmarshalJSON(data, &ts) + if err != nil { + // should not happened due to previous decoding succeeded + return "", err + } + + if ts.Type == "" { + return "", fmt.Errorf("could not find type in element index %d", index) } - return "", fmt.Errorf("could not find type in element index %d", index) + return ts.Type, nil } diff --git a/osm_test.go b/osm_test.go index f615e236..a80c8072 100644 --- a/osm_test.go +++ b/osm_test.go @@ -183,7 +183,10 @@ func TestOSM_UnmarshalJSON(t *testing.T) { {"type":"way","id":456,"visible":false,"timestamp":"0001-01-01T00:00:00Z","nodes":[]}, {"type":"relation","id":789,"visible":false,"timestamp":"0001-01-01T00:00:00Z","members":[]}, {"type":"changeset","id":10,"created_at":"0001-01-01T00:00:00Z","closed_at":"0001-01-01T00:00:00Z","open":false}, - {"type":"user","id":16,"name":"","img":{"href":""},"changesets":{"count":0},"traces":{"count":0},"home":{"lat":0,"lon":0,"zoom":0},"languages":null,"blocks":{"received":{"count":0,"active":0}},"messages":{"received":{"count":0,"unread":0},"sent":{"count":0}},"created_at":"0001-01-01T00:00:00Z"}, + {"type":"user","id":16,"name":"","img":{"href":""}, + "changesets":{"count":0},"traces":{"count":0},"home":{"lat":0,"lon":0,"zoom":0},"languages":null, + "blocks":{"received":{"count":0,"active":0}},"messages":{"received":{"count":0,"unread":0},"sent":{"count":0}}, + "created_at":"0001-01-01T00:00:00Z"}, {"type":"note","id":15,"lat":0,"lon":0,"date_created":null,"date_closed":null,"comments":null} ]}`) @@ -244,6 +247,25 @@ func TestOSM_UnmarshalJSON_Version(t *testing.T) { } } +func TestOSM_UnmarshalJSON_Type(t *testing.T) { + data := []byte(`{ + "version":0.6,"generator":"osm-go", + "elements":[ + {"type":"relation","id":120,"timestamp":"0001-01-01T00:00:00Z","tags":{"type":"route"}}, + {"tags":{"type":"route","other":"asdf"},"type":"relation","id":121,"timestamp":"0001-01-01T00:00:00Z"} + ]}`) + + o := &OSM{} + err := json.Unmarshal(data, &o) + if err != nil { + t.Fatalf("unmarshal error: %v", err) + } + + if l := len(o.Relations); l != 2 { + t.Errorf("incorrect number of relations: %v", l) + } +} + func TestOSM_MarshalXML(t *testing.T) { o := &OSM{ Version: "0.7", From f9fed71d03f550357f2619d5e8bf620fe6a618f9 Mon Sep 17 00:00:00 2001 From: Paul Mach Date: Wed, 27 Dec 2023 22:57:39 -0800 Subject: [PATCH 25/29] fix lint errors: remove ioutil references and other fixes --- README.md | 2 +- annotate/geo.go | 2 +- annotate/way_test.go | 3 +-- change_test.go | 20 ++++++++++++-------- diff_test.go | 20 +++++++++++++++----- go.mod | 2 +- osm_test.go | 17 ----------------- osmgeojson/benchmarks_test.go | 4 ++-- osmgeojson/build_polygon.go | 2 +- osmgeojson/convert_test.go | 2 +- osmgeojson/options.go | 2 +- relation_test.go | 6 +++++- replication/changesets.go | 3 +-- replication/interval.go | 4 ++-- way_test.go | 6 +++++- 15 files changed, 49 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index 9d65d09c..594da6a0 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,7 @@ var c = jsoniter.Config{ MarshalFloatWith6Digits: true, }.Froze() -osmm.CustomJSONMarshaler = c +osm.CustomJSONMarshaler = c osm.CustomJSONUnmarshaler = c ``` diff --git a/annotate/geo.go b/annotate/geo.go index 91ba7abd..a17a3baf 100644 --- a/annotate/geo.go +++ b/annotate/geo.go @@ -52,7 +52,7 @@ func wayCentroid(w *osm.Way) orb.Point { return point } -// orientation will annotate the the orientation of multipolygon relation members. +// orientation will annotate the orientation of multipolygon relation members. // This makes it possible to reconstruct relations with partial data in the right direction. // Return value indicates if the result is 'tainted', e.g. not all way members were present. func orientation(members osm.Members, ways map[osm.WayID]*osm.Way, at time.Time) bool { diff --git a/annotate/way_test.go b/annotate/way_test.go index 42d35ce4..01bc9334 100644 --- a/annotate/way_test.go +++ b/annotate/way_test.go @@ -4,7 +4,6 @@ import ( "context" "encoding/xml" "fmt" - "io/ioutil" "os" "reflect" "testing" @@ -132,7 +131,7 @@ func BenchmarkWays(b *testing.B) { } func loadTestdata(tb testing.TB, filename string) *osm.OSM { - data, err := ioutil.ReadFile(filename) + data, err := os.ReadFile(filename) if err != nil { tb.Fatalf("unable to open file: %v", err) } diff --git a/change_test.go b/change_test.go index 444d754f..085f60a6 100644 --- a/change_test.go +++ b/change_test.go @@ -5,7 +5,7 @@ import ( "context" "encoding/json" "encoding/xml" - "io/ioutil" + "os" "testing" ) @@ -237,11 +237,13 @@ func TestChange_HistoryDatasource(t *testing.T) { } func BenchmarkChange_MarshalXML(b *testing.B) { - filename := "testdata/changeset_38162206.osc" - data := readFile(b, filename) + data, err := os.ReadFile("testdata/changeset_38162206.osc") + if err != nil { + b.Fatalf("unable to read file: %v", err) + } c := &Change{} - err := xml.Unmarshal(data, c) + err = xml.Unmarshal(data, c) if err != nil { b.Fatalf("unable to unmarshal: %v", err) } @@ -270,7 +272,7 @@ func BenchmarkChange_MarshalXML(b *testing.B) { // } func BenchmarkChange_MarshalJSON(b *testing.B) { - data, err := ioutil.ReadFile("testdata/minute_871.osc") + data, err := os.ReadFile("testdata/minute_871.osc") if err != nil { b.Fatalf("could not read file: %v", err) } @@ -292,7 +294,7 @@ func BenchmarkChange_MarshalJSON(b *testing.B) { } func BenchmarkChange_UnmarshalJSON(b *testing.B) { - data, err := ioutil.ReadFile("testdata/minute_871.osc") + data, err := os.ReadFile("testdata/minute_871.osc") if err != nil { b.Fatalf("could not read file: %v", err) } @@ -320,8 +322,10 @@ func BenchmarkChange_UnmarshalJSON(b *testing.B) { } func BenchmarkChangeset_UnmarshalXML(b *testing.B) { - filename := "testdata/changeset_38162206.osc" - data := readFile(b, filename) + data, err := os.ReadFile("testdata/changeset_38162206.osc") + if err != nil { + b.Fatalf("unable to read file: %v", err) + } b.ReportAllocs() b.ResetTimer() diff --git a/diff_test.go b/diff_test.go index 0955747a..6ff329ba 100644 --- a/diff_test.go +++ b/diff_test.go @@ -2,6 +2,7 @@ package osm import ( "encoding/xml" + "os" "reflect" "testing" ) @@ -72,10 +73,13 @@ func TestDiff_MarshalXML(t *testing.T) { } func TestDiff(t *testing.T) { - data := readFile(t, "testdata/annotated_diff.xml") + data, err := os.ReadFile("testdata/annotated_diff.xml") + if err != nil { + t.Fatalf("unable to read file: %v", err) + } diff := &Diff{} - err := xml.Unmarshal(data, &diff) + err = xml.Unmarshal(data, &diff) if err != nil { t.Errorf("unable to unmarshal: %v", err) } @@ -137,10 +141,13 @@ func TestDiff(t *testing.T) { } func BenchmarkDiff_Marshal(b *testing.B) { - data := readFile(b, "testdata/annotated_diff.xml") + data, err := os.ReadFile("testdata/annotated_diff.xml") + if err != nil { + b.Fatalf("unable to read file: %v", err) + } diff := &Diff{} - err := xml.Unmarshal(data, &diff) + err = xml.Unmarshal(data, &diff) if err != nil { b.Fatalf("unmarshal error: %v", err) } @@ -156,7 +163,10 @@ func BenchmarkDiff_Marshal(b *testing.B) { } func BenchmarkDiff_Unmarshal(b *testing.B) { - data := readFile(b, "testdata/annotated_diff.xml") + data, err := os.ReadFile("testdata/annotated_diff.xml") + if err != nil { + b.Fatalf("unable to read file: %v", err) + } b.ReportAllocs() b.ResetTimer() diff --git a/go.mod b/go.mod index 5ea8c278..19eb1b3b 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/paulmach/osm -go 1.13 +go 1.16 require ( github.com/datadog/czlib v0.0.0-20160811164712-4bc9a24e37f2 diff --git a/osm_test.go b/osm_test.go index a80c8072..7f57c8a0 100644 --- a/osm_test.go +++ b/osm_test.go @@ -4,8 +4,6 @@ import ( "bytes" "encoding/json" "encoding/xml" - "io" - "os" "reflect" "testing" ) @@ -301,18 +299,3 @@ func TestOSM_MarshalXML(t *testing.T) { t.Errorf("incorrect marshal, got: %s", string(data)) } } - -func readFile(t testing.TB, filename string) []byte { - f, err := os.Open(filename) - if err != nil { - t.Fatalf("unable to open %s: %v", filename, err) - } - defer f.Close() - - data, err := io.ReadAll(f) - if err != nil { - t.Fatalf("unable to read file %s: %v", filename, err) - } - - return data -} diff --git a/osmgeojson/benchmarks_test.go b/osmgeojson/benchmarks_test.go index 48771877..3d25259b 100644 --- a/osmgeojson/benchmarks_test.go +++ b/osmgeojson/benchmarks_test.go @@ -2,7 +2,7 @@ package osmgeojson import ( "encoding/xml" - "io/ioutil" + "os" "testing" "github.com/paulmach/osm" @@ -88,7 +88,7 @@ func BenchmarkConvert_NoIDsMetaMembership(b *testing.B) { } func parseFile(t testing.TB, filename string) *osm.OSM { - data, err := ioutil.ReadFile(filename) + data, err := os.ReadFile(filename) if err != nil { t.Fatalf("could not read file: %v", err) } diff --git a/osmgeojson/build_polygon.go b/osmgeojson/build_polygon.go index bc00284d..e78c4d30 100644 --- a/osmgeojson/build_polygon.go +++ b/osmgeojson/build_polygon.go @@ -61,7 +61,7 @@ func (ctx *context) buildPolygon(relation *osm.Relation) *geojson.Feature { } if len(ls) == 0 { - // we have the way but none the the node members + // we have the way but none of the node members continue } diff --git a/osmgeojson/convert_test.go b/osmgeojson/convert_test.go index 3209d0ec..3c59e833 100644 --- a/osmgeojson/convert_test.go +++ b/osmgeojson/convert_test.go @@ -674,7 +674,7 @@ type rawFC struct { func jsonLoop(t *testing.T, fc *geojson.FeatureCollection) *rawFC { data, err := json.Marshal(fc) if err != nil { - t.Fatalf("unabled to marshal fc: %v", err) + t.Fatalf("unable to marshal fc: %v", err) } result := &rawFC{} diff --git a/osmgeojson/options.go b/osmgeojson/options.go index dd8321d5..ac838be7 100644 --- a/osmgeojson/options.go +++ b/osmgeojson/options.go @@ -20,7 +20,7 @@ func NoMeta(yes bool) Option { } } -// NoRelationMembership will omit the the list of relations +// NoRelationMembership will omit the list of relations // an element is a member of from the output geojson features. func NoRelationMembership(yes bool) Option { return func(ctx *context) error { diff --git a/relation_test.go b/relation_test.go index bb19e92d..63b4b5e1 100644 --- a/relation_test.go +++ b/relation_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "encoding/xml" + "os" "reflect" "testing" "time" @@ -199,7 +200,10 @@ func TestRelation_MarshalXML(t *testing.T) { } // blanket xml test - data = readFile(t, "testdata/relation-updates.osm") + data, err = os.ReadFile("testdata/relation-updates.osm") + if err != nil { + t.Fatalf("could not read file: %v", err) + } osm := &OSM{} err = xml.Unmarshal(data, &osm) diff --git a/replication/changesets.go b/replication/changesets.go index f149c8cf..65eb2730 100644 --- a/replication/changesets.go +++ b/replication/changesets.go @@ -6,7 +6,6 @@ import ( "context" "fmt" "io" - "io/ioutil" "net/http" "strconv" @@ -88,7 +87,7 @@ func (ds *Datasource) fetchChangesetState(ctx context.Context, n ChangesetSeqNum } } - data, err := ioutil.ReadAll(resp.Body) + data, err := io.ReadAll(resp.Body) if err != nil { return nil, err } diff --git a/replication/interval.go b/replication/interval.go index 8c258bd8..d322d855 100644 --- a/replication/interval.go +++ b/replication/interval.go @@ -6,7 +6,7 @@ import ( "context" "encoding/xml" "fmt" - "io/ioutil" + "io" "net/http" "strconv" "time" @@ -219,7 +219,7 @@ func (ds *Datasource) fetchState(ctx context.Context, n SeqNum) (*State, error) } } - data, err := ioutil.ReadAll(resp.Body) + data, err := io.ReadAll(resp.Body) if err != nil { return nil, err } diff --git a/way_test.go b/way_test.go index 59d7ab38..d669bbd8 100644 --- a/way_test.go +++ b/way_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "encoding/xml" + "os" "reflect" "testing" "time" @@ -284,7 +285,10 @@ func TestWay_MarshalXML(t *testing.T) { } // blanket xml test - data = readFile(t, "testdata/way-updates.osm") + data, err = os.ReadFile("testdata/way-updates.osm") + if err != nil { + t.Fatalf("unable to read file: %v", err) + } osm := &OSM{} err = xml.Unmarshal(data, &osm) From 63649b347bfd6f46879cd134b6579af3fff96fe0 Mon Sep 17 00:00:00 2001 From: Paul Mach Date: Mon, 8 Jan 2024 11:33:51 -0800 Subject: [PATCH 26/29] update changelog --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f551a6b2..9a7aa0b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ All notable changes to this project will be documented in this file. +## [v0.8.0](https://github.com/paulmach/osm/compare/v0.7.1...v0.8.0) - 2024-01-08 + +### Changed + +- go 1.16 is required, updated usages of `ioutil` for similar functions in `io` and `os` + +### Fixed + +- correctly JSON unmarshal elements with a type tag by [@paulmach](https://github.com/paulmach) in https://github.com/paulmach/osm/pull/53 + ## [v0.7.1](https://github.com/paulmach/osm/compare/v0.7.0...v0.7.1) - 2022-11-29 ### Added From fbd68419d3abc9ea851cb003159ea5fb396757b6 Mon Sep 17 00:00:00 2001 From: Brandon Liu Date: Wed, 10 Jan 2024 12:37:23 +0800 Subject: [PATCH 27/29] osmpbf examples: fix small typos --- osmpbf/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/osmpbf/README.md b/osmpbf/README.md index cdcdf61f..8c993771 100644 --- a/osmpbf/README.md +++ b/osmpbf/README.md @@ -10,19 +10,21 @@ file, err := os.Open("./delaware-latest.osm.pbf") if err != nil { panic(err) } -defer f.Close() +defer file.Close() // The third parameter is the number of parallel decoders to use. scanner := osmpbf.New(context.Background(), file, runtime.GOMAXPROCS(-1)) defer scanner.Close() for scanner.Scan() { - switch o := scanner.Object().(type) + switch o := scanner.Object().(type) { case *osm.Node: - + case *osm.Way: case *osm.Relation: + + } } if err := scanner.Err(); err != nil { From db9d8ebc69cde0241f4989a74ea94ca9e7066e12 Mon Sep 17 00:00:00 2001 From: Brandon Liu Date: Wed, 10 Jan 2024 12:38:38 +0800 Subject: [PATCH 28/29] fix whitespace --- osmpbf/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osmpbf/README.md b/osmpbf/README.md index 8c993771..57a79991 100644 --- a/osmpbf/README.md +++ b/osmpbf/README.md @@ -19,7 +19,7 @@ defer scanner.Close() for scanner.Scan() { switch o := scanner.Object().(type) { case *osm.Node: - + case *osm.Way: case *osm.Relation: From af1fec1a55b1e303f94aec27164b15100dde4d16 Mon Sep 17 00:00:00 2001 From: larsbeck Date: Mon, 6 May 2024 10:58:25 +0200 Subject: [PATCH 29/29] Fix delta encoding of a node's timestamp on DenseInfo field --- osmpbf/encode.go | 18 ++++++++++++++---- osmpbf/encode_test.go | 10 +++------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/osmpbf/encode.go b/osmpbf/encode.go index 9a32ea37..796213b7 100644 --- a/osmpbf/encode.go +++ b/osmpbf/encode.go @@ -264,9 +264,15 @@ func EncodeDenseNode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string if groupDense.Denseinfo != nil { groupDense.Denseinfo.Changeset = append(groupDense.Denseinfo.Changeset, int64(current.ChangesetID-previous.ChangesetID)) dateGranularity := block.GetDateGranularity() - currentTimeStamp := EncodeTimestamp(current.Timestamp, dateGranularity) - previousTimeStamp := EncodeTimestamp(previous.Timestamp, dateGranularity) - groupDense.Denseinfo.Timestamp = append(groupDense.Denseinfo.Timestamp, currentTimeStamp-previousTimeStamp) + if !current.Timestamp.IsZero() { + currentTimeStamp := EncodeTimestamp(current.Timestamp, dateGranularity) + if !previous.Timestamp.IsZero() { + previousTimeStamp := EncodeTimestamp(previous.Timestamp, dateGranularity) + groupDense.Denseinfo.Timestamp = append(groupDense.Denseinfo.Timestamp, currentTimeStamp-previousTimeStamp) + } else { + groupDense.Denseinfo.Timestamp = append(groupDense.Denseinfo.Timestamp, currentTimeStamp) + } + } groupDense.Denseinfo.Uid = append(groupDense.Denseinfo.Uid, int32(current.UserID-previous.UserID)) groupDense.Denseinfo.Version = append(groupDense.Denseinfo.Version, int32(current.Version-previous.Version)) var previousUserNameId int32 = 0 @@ -418,7 +424,11 @@ func EncodeLatLon(value float64, offset int64, granularity int32) int64 { } func EncodeTimestamp(timestamp time.Time, dateGranularity int32) int64 { - return timestamp.UnixMilli() / int64(dateGranularity) + ts := timestamp.UnixMilli() / int64(dateGranularity) + if ts < 0 { + return 0 + } + return ts } func EncodeString(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, value string) int32 { diff --git a/osmpbf/encode_test.go b/osmpbf/encode_test.go index 8f1e86a4..578944a7 100644 --- a/osmpbf/encode_test.go +++ b/osmpbf/encode_test.go @@ -96,13 +96,9 @@ func nodeEquals(en, node *osm.Node) error { if node.ChangesetID != en.ChangesetID { return fmt.Errorf("node changeset id mismatch: %d != %d", node.ChangesetID, en.ChangesetID) } - // TODO: this currently fails with - // "node timestamp mismatch: 2224-09-20 11:45:12.871345152 +0000 UTC != - // 2009-05-20 10:28:54 +0000 UTC" - // The other timestamps (for ways and relations) work fine. - // if node.Timestamp != en.Timestamp { - // return fmt.Errorf("node timestamp mismatch: %s != %s", node.Timestamp, en.Timestamp) - // } + if node.Timestamp != en.Timestamp { + return fmt.Errorf("node timestamp mismatch: %s != %s", node.Timestamp, en.Timestamp) + } if len(node.Tags) != len(en.Tags) { return fmt.Errorf("node tags length mismatch: %d != %d", len(node.Tags), len(en.Tags)) }