diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml new file mode 100644 index 00000000..3f541892 --- /dev/null +++ b/.github/workflows/golangci-lint.yml @@ -0,0 +1,24 @@ +name: golangci-lint +on: + push: + branches: + - master + pull_request: + +permissions: + contents: read + +jobs: + golangci: + name: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version: '1.21' + + - name: golangci-lint + uses: golangci/golangci-lint-action@v3 + with: + version: v1.55 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 70c7be37..17825701 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -4,35 +4,28 @@ on: push: branches: [ master ] pull_request: - branches: [ master ] jobs: build-and-test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - + - uses: actions/checkout@v3 + - name: Setup Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: - go-version: '1.15.3' - - - name: Install dependencies - run: | - go version - go get -u golang.org/x/lint/golint + go-version: '1.21' - name: Run build - run: go build . - - - name: Run vet & lint + run: go build . + + - name: Run vet run: | go vet . - golint . - + - name: Run tests run: go test -v -coverprofile=profile.cov ./... - + - name: codecov uses: codecov/codecov-action@v1 with: diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 00000000..8e5e857f --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,5 @@ +issues: + exclude-rules: + - path: '(.+)_test\.go' + linters: + - errcheck diff --git a/CHANGELOG.md b/CHANGELOG.md index f551a6b2..9a7aa0b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ All notable changes to this project will be documented in this file. +## [v0.8.0](https://github.com/paulmach/osm/compare/v0.7.1...v0.8.0) - 2024-01-08 + +### Changed + +- go 1.16 is required, updated usages of `ioutil` for similar functions in `io` and `os` + +### Fixed + +- correctly JSON unmarshal elements with a type tag by [@paulmach](https://github.com/paulmach) in https://github.com/paulmach/osm/pull/53 + ## [v0.7.1](https://github.com/paulmach/osm/compare/v0.7.0...v0.7.1) - 2022-11-29 ### Added diff --git a/README.md b/README.md index b8e29f65..594da6a0 100644 --- a/README.md +++ b/README.md @@ -113,13 +113,15 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/paulmach/osm" ) + var c = jsoniter.Config{ EscapeHTML: true, SortMapKeys: false, MarshalFloatWith6Digits: true, }.Froze() -CustomJSONMarshaler = c -CustomJSONUnmarshaler = c + +osm.CustomJSONMarshaler = c +osm.CustomJSONUnmarshaler = c ``` The above change can have dramatic performance implications, see the benchmarks below diff --git a/annotate/geo.go b/annotate/geo.go index 91ba7abd..a17a3baf 100644 --- a/annotate/geo.go +++ b/annotate/geo.go @@ -52,7 +52,7 @@ func wayCentroid(w *osm.Way) orb.Point { return point } -// orientation will annotate the the orientation of multipolygon relation members. +// orientation will annotate the orientation of multipolygon relation members. // This makes it possible to reconstruct relations with partial data in the right direction. // Return value indicates if the result is 'tainted', e.g. not all way members were present. func orientation(members osm.Members, ways map[osm.WayID]*osm.Way, at time.Time) bool { diff --git a/annotate/relation_test.go b/annotate/relation_test.go index 1619bbe6..015c1441 100644 --- a/annotate/relation_test.go +++ b/annotate/relation_test.go @@ -4,7 +4,7 @@ import ( "context" "encoding/xml" "fmt" - "io/ioutil" + "os" "reflect" "testing" "time" @@ -42,7 +42,10 @@ func TestRelation(t *testing.T) { t.Errorf("expected relations not equal, file saved to %s", filename) data, _ := xml.MarshalIndent(&osm.OSM{Relations: relations}, "", " ") - ioutil.WriteFile(filename, data, 0644) + err := os.WriteFile(filename, data, 0644) + if err != nil { + t.Fatalf("write error: %v", err) + } } } } diff --git a/annotate/way_test.go b/annotate/way_test.go index 47cfc3ec..01bc9334 100644 --- a/annotate/way_test.go +++ b/annotate/way_test.go @@ -4,7 +4,7 @@ import ( "context" "encoding/xml" "fmt" - "io/ioutil" + "os" "reflect" "testing" @@ -36,7 +36,10 @@ func TestWays(t *testing.T) { t.Errorf("expected way not equal, file saved to %s", filename) data, _ := xml.MarshalIndent(&osm.OSM{Ways: o.Ways}, "", " ") - ioutil.WriteFile(filename, data, 0644) + err := os.WriteFile(filename, data, 0644) + if err != nil { + t.Fatalf("write error: %v", err) + } } } } @@ -128,7 +131,7 @@ func BenchmarkWays(b *testing.B) { } func loadTestdata(tb testing.TB, filename string) *osm.OSM { - data, err := ioutil.ReadFile(filename) + data, err := os.ReadFile(filename) if err != nil { tb.Fatalf("unable to open file: %v", err) } diff --git a/change_test.go b/change_test.go index 73c86a1e..085f60a6 100644 --- a/change_test.go +++ b/change_test.go @@ -5,7 +5,7 @@ import ( "context" "encoding/json" "encoding/xml" - "io/ioutil" + "os" "testing" ) @@ -236,29 +236,14 @@ func TestChange_HistoryDatasource(t *testing.T) { } } -func cleanXMLNameFromChange(c *Change) { - c.Version = "" - c.Generator = "" - c.Copyright = "" - c.Attribution = "" - c.License = "" - if c.Create != nil { - cleanXMLNameFromOSM(c.Create) - } - if c.Modify != nil { - cleanXMLNameFromOSM(c.Modify) - } - if c.Delete != nil { - cleanXMLNameFromOSM(c.Delete) - } -} - func BenchmarkChange_MarshalXML(b *testing.B) { - filename := "testdata/changeset_38162206.osc" - data := readFile(b, filename) + data, err := os.ReadFile("testdata/changeset_38162206.osc") + if err != nil { + b.Fatalf("unable to read file: %v", err) + } c := &Change{} - err := xml.Unmarshal(data, c) + err = xml.Unmarshal(data, c) if err != nil { b.Fatalf("unable to unmarshal: %v", err) } @@ -287,7 +272,7 @@ func BenchmarkChange_MarshalXML(b *testing.B) { // } func BenchmarkChange_MarshalJSON(b *testing.B) { - data, err := ioutil.ReadFile("testdata/minute_871.osc") + data, err := os.ReadFile("testdata/minute_871.osc") if err != nil { b.Fatalf("could not read file: %v", err) } @@ -309,7 +294,7 @@ func BenchmarkChange_MarshalJSON(b *testing.B) { } func BenchmarkChange_UnmarshalJSON(b *testing.B) { - data, err := ioutil.ReadFile("testdata/minute_871.osc") + data, err := os.ReadFile("testdata/minute_871.osc") if err != nil { b.Fatalf("could not read file: %v", err) } @@ -337,8 +322,10 @@ func BenchmarkChange_UnmarshalJSON(b *testing.B) { } func BenchmarkChangeset_UnmarshalXML(b *testing.B) { - filename := "testdata/changeset_38162206.osc" - data := readFile(b, filename) + data, err := os.ReadFile("testdata/changeset_38162206.osc") + if err != nil { + b.Fatalf("unable to read file: %v", err) + } b.ReportAllocs() b.ResetTimer() diff --git a/diff_test.go b/diff_test.go index 517b9cc1..6ff329ba 100644 --- a/diff_test.go +++ b/diff_test.go @@ -2,6 +2,7 @@ package osm import ( "encoding/xml" + "os" "reflect" "testing" ) @@ -72,10 +73,13 @@ func TestDiff_MarshalXML(t *testing.T) { } func TestDiff(t *testing.T) { - data := readFile(t, "testdata/annotated_diff.xml") + data, err := os.ReadFile("testdata/annotated_diff.xml") + if err != nil { + t.Fatalf("unable to read file: %v", err) + } diff := &Diff{} - err := xml.Unmarshal(data, &diff) + err = xml.Unmarshal(data, &diff) if err != nil { t.Errorf("unable to unmarshal: %v", err) } @@ -137,10 +141,13 @@ func TestDiff(t *testing.T) { } func BenchmarkDiff_Marshal(b *testing.B) { - data := readFile(b, "testdata/annotated_diff.xml") + data, err := os.ReadFile("testdata/annotated_diff.xml") + if err != nil { + b.Fatalf("unable to read file: %v", err) + } diff := &Diff{} - err := xml.Unmarshal(data, &diff) + err = xml.Unmarshal(data, &diff) if err != nil { b.Fatalf("unmarshal error: %v", err) } @@ -148,17 +155,26 @@ func BenchmarkDiff_Marshal(b *testing.B) { b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { - xml.Marshal(diff) + _, err := xml.Marshal(diff) + if err != nil { + b.Fatalf("marshal error: %v", err) + } } } func BenchmarkDiff_Unmarshal(b *testing.B) { - data := readFile(b, "testdata/annotated_diff.xml") + data, err := os.ReadFile("testdata/annotated_diff.xml") + if err != nil { + b.Fatalf("unable to read file: %v", err) + } b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { diff := &Diff{} - xml.Unmarshal(data, &diff) + err := xml.Unmarshal(data, &diff) + if err != nil { + b.Fatalf("unmarshal error: %v", err) + } } } diff --git a/go.mod b/go.mod index 5ea8c278..19eb1b3b 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/paulmach/osm -go 1.13 +go 1.16 require ( github.com/datadog/czlib v0.0.0-20160811164712-4bc9a24e37f2 diff --git a/osm.go b/osm.go index 76c3532e..ada25c2f 100644 --- a/osm.go +++ b/osm.go @@ -3,7 +3,6 @@ package osm import ( "encoding/xml" "fmt" - "regexp" ) // These values should be returned if the osm data is actual @@ -367,13 +366,21 @@ func (o *OSM) UnmarshalJSON(data []byte) error { return nil } -var jsonTypeRegexp = regexp.MustCompile(`"type"\s*:\s*"([^"]*)"`) +type typeStruct struct { + Type string `json:"type"` +} func findType(index int, data []byte) (string, error) { - matches := jsonTypeRegexp.FindAllSubmatch(data, 1) - if len(matches) > 0 { - return string(matches[0][1]), nil + ts := typeStruct{} + err := unmarshalJSON(data, &ts) + if err != nil { + // should not happened due to previous decoding succeeded + return "", err + } + + if ts.Type == "" { + return "", fmt.Errorf("could not find type in element index %d", index) } - return "", fmt.Errorf("could not find type in element index %d", index) + return ts.Type, nil } diff --git a/osm_test.go b/osm_test.go index e9500e03..7f57c8a0 100644 --- a/osm_test.go +++ b/osm_test.go @@ -4,8 +4,6 @@ import ( "bytes" "encoding/json" "encoding/xml" - "io/ioutil" - "os" "reflect" "testing" ) @@ -183,7 +181,10 @@ func TestOSM_UnmarshalJSON(t *testing.T) { {"type":"way","id":456,"visible":false,"timestamp":"0001-01-01T00:00:00Z","nodes":[]}, {"type":"relation","id":789,"visible":false,"timestamp":"0001-01-01T00:00:00Z","members":[]}, {"type":"changeset","id":10,"created_at":"0001-01-01T00:00:00Z","closed_at":"0001-01-01T00:00:00Z","open":false}, - {"type":"user","id":16,"name":"","img":{"href":""},"changesets":{"count":0},"traces":{"count":0},"home":{"lat":0,"lon":0,"zoom":0},"languages":null,"blocks":{"received":{"count":0,"active":0}},"messages":{"received":{"count":0,"unread":0},"sent":{"count":0}},"created_at":"0001-01-01T00:00:00Z"}, + {"type":"user","id":16,"name":"","img":{"href":""}, + "changesets":{"count":0},"traces":{"count":0},"home":{"lat":0,"lon":0,"zoom":0},"languages":null, + "blocks":{"received":{"count":0,"active":0}},"messages":{"received":{"count":0,"unread":0},"sent":{"count":0}}, + "created_at":"0001-01-01T00:00:00Z"}, {"type":"note","id":15,"lat":0,"lon":0,"date_created":null,"date_closed":null,"comments":null} ]}`) @@ -244,6 +245,25 @@ func TestOSM_UnmarshalJSON_Version(t *testing.T) { } } +func TestOSM_UnmarshalJSON_Type(t *testing.T) { + data := []byte(`{ + "version":0.6,"generator":"osm-go", + "elements":[ + {"type":"relation","id":120,"timestamp":"0001-01-01T00:00:00Z","tags":{"type":"route"}}, + {"tags":{"type":"route","other":"asdf"},"type":"relation","id":121,"timestamp":"0001-01-01T00:00:00Z"} + ]}`) + + o := &OSM{} + err := json.Unmarshal(data, &o) + if err != nil { + t.Fatalf("unmarshal error: %v", err) + } + + if l := len(o.Relations); l != 2 { + t.Errorf("incorrect number of relations: %v", l) + } +} + func TestOSM_MarshalXML(t *testing.T) { o := &OSM{ Version: "0.7", @@ -279,54 +299,3 @@ func TestOSM_MarshalXML(t *testing.T) { t.Errorf("incorrect marshal, got: %s", string(data)) } } - -func flattenOSM(c *Change) *OSM { - o := c.Create - if o == nil { - o = &OSM{} - } - - if c.Modify != nil { - o.Nodes = append(o.Nodes, c.Modify.Nodes...) - o.Ways = append(o.Ways, c.Modify.Ways...) - o.Relations = append(o.Relations, c.Modify.Relations...) - } - - if c.Delete != nil { - o.Nodes = append(o.Nodes, c.Delete.Nodes...) - o.Ways = append(o.Ways, c.Delete.Ways...) - o.Relations = append(o.Relations, c.Delete.Relations...) - } - - return o -} - -func cleanXMLNameFromOSM(o *OSM) { - for _, n := range o.Nodes { - n.XMLName = xmlNameJSONTypeNode{} - } - - for _, w := range o.Ways { - w.XMLName = xmlNameJSONTypeWay{} - } - - for _, r := range o.Relations { - // r.XMLName = xml.Name{} - r.XMLName = xmlNameJSONTypeRel{} - } -} - -func readFile(t testing.TB, filename string) []byte { - f, err := os.Open(filename) - if err != nil { - t.Fatalf("unable to open %s: %v", filename, err) - } - defer f.Close() - - data, err := ioutil.ReadAll(f) - if err != nil { - t.Fatalf("unable to read file %s: %v", filename, err) - } - - return data -} diff --git a/osmapi/options.go b/osmapi/options.go index c54d2b60..744a19b3 100644 --- a/osmapi/options.go +++ b/osmapi/options.go @@ -24,8 +24,6 @@ func (o *at) applyFeature(p []string) ([]string, error) { return append(p, "at="+o.t.UTC().Format("2006-01-02T15:04:05Z")), nil } -func (o *at) feature() {} - // NotesOption defines a valid option for the osmapi.Notes by bounding box api. type NotesOption interface { applyNotes([]string) ([]string, error) diff --git a/osmgeojson/benchmarks_test.go b/osmgeojson/benchmarks_test.go index 1dc71dd7..3d25259b 100644 --- a/osmgeojson/benchmarks_test.go +++ b/osmgeojson/benchmarks_test.go @@ -2,7 +2,7 @@ package osmgeojson import ( "encoding/xml" - "io/ioutil" + "os" "testing" "github.com/paulmach/osm" @@ -14,7 +14,10 @@ func BenchmarkConvert(b *testing.B) { b.ReportAllocs() b.ResetTimer() for n := 0; n < b.N; n++ { - Convert(o) + _, err := Convert(o) + if err != nil { + b.Fatalf("convert error: %v", err) + } } } @@ -25,7 +28,10 @@ func BenchmarkConvertAnnotated(b *testing.B) { b.ReportAllocs() b.ResetTimer() for n := 0; n < b.N; n++ { - Convert(o) + _, err := Convert(o) + if err != nil { + b.Fatalf("convert error: %v", err) + } } } @@ -35,7 +41,10 @@ func BenchmarkConvert_NoID(b *testing.B) { b.ReportAllocs() b.ResetTimer() for n := 0; n < b.N; n++ { - Convert(o, NoID(true)) + _, err := Convert(o, NoID(true)) + if err != nil { + b.Fatalf("convert error: %v", err) + } } } @@ -45,7 +54,10 @@ func BenchmarkConvert_NoMeta(b *testing.B) { b.ReportAllocs() b.ResetTimer() for n := 0; n < b.N; n++ { - Convert(o, NoMeta(true)) + _, err := Convert(o, NoMeta(true)) + if err != nil { + b.Fatalf("convert error: %v", err) + } } } @@ -55,7 +67,10 @@ func BenchmarkConvert_NoRelationMembership(b *testing.B) { b.ReportAllocs() b.ResetTimer() for n := 0; n < b.N; n++ { - Convert(o, NoRelationMembership(true)) + _, err := Convert(o, NoRelationMembership(true)) + if err != nil { + b.Fatalf("convert error: %v", err) + } } } @@ -65,12 +80,15 @@ func BenchmarkConvert_NoIDsMetaMembership(b *testing.B) { b.ReportAllocs() b.ResetTimer() for n := 0; n < b.N; n++ { - Convert(o, NoID(true), NoMeta(true), NoRelationMembership(true)) + _, err := Convert(o, NoID(true), NoMeta(true), NoRelationMembership(true)) + if err != nil { + b.Fatalf("convert error: %v", err) + } } } func parseFile(t testing.TB, filename string) *osm.OSM { - data, err := ioutil.ReadFile(filename) + data, err := os.ReadFile(filename) if err != nil { t.Fatalf("could not read file: %v", err) } diff --git a/osmgeojson/build_polygon.go b/osmgeojson/build_polygon.go index bc00284d..e78c4d30 100644 --- a/osmgeojson/build_polygon.go +++ b/osmgeojson/build_polygon.go @@ -61,7 +61,7 @@ func (ctx *context) buildPolygon(relation *osm.Relation) *geojson.Feature { } if len(ls) == 0 { - // we have the way but none the the node members + // we have the way but none of the node members continue } diff --git a/osmgeojson/convert_test.go b/osmgeojson/convert_test.go index 3209d0ec..3c59e833 100644 --- a/osmgeojson/convert_test.go +++ b/osmgeojson/convert_test.go @@ -674,7 +674,7 @@ type rawFC struct { func jsonLoop(t *testing.T, fc *geojson.FeatureCollection) *rawFC { data, err := json.Marshal(fc) if err != nil { - t.Fatalf("unabled to marshal fc: %v", err) + t.Fatalf("unable to marshal fc: %v", err) } result := &rawFC{} diff --git a/osmgeojson/options.go b/osmgeojson/options.go index dd8321d5..ac838be7 100644 --- a/osmgeojson/options.go +++ b/osmgeojson/options.go @@ -20,7 +20,7 @@ func NoMeta(yes bool) Option { } } -// NoRelationMembership will omit the the list of relations +// NoRelationMembership will omit the list of relations // an element is a member of from the output geojson features. func NoRelationMembership(yes bool) Option { return func(ctx *context) error { diff --git a/osmpbf/README.md b/osmpbf/README.md index cdcdf61f..57a79991 100644 --- a/osmpbf/README.md +++ b/osmpbf/README.md @@ -10,19 +10,21 @@ file, err := os.Open("./delaware-latest.osm.pbf") if err != nil { panic(err) } -defer f.Close() +defer file.Close() // The third parameter is the number of parallel decoders to use. scanner := osmpbf.New(context.Background(), file, runtime.GOMAXPROCS(-1)) defer scanner.Close() for scanner.Scan() { - switch o := scanner.Object().(type) + switch o := scanner.Object().(type) { case *osm.Node: case *osm.Way: case *osm.Relation: + + } } if err := scanner.Err(); err != nil { diff --git a/osmpbf/decode_test.go b/osmpbf/decode_test.go index 6b6785c6..5d7340df 100644 --- a/osmpbf/decode_test.go +++ b/osmpbf/decode_test.go @@ -175,25 +175,25 @@ func (ft *OSMFileTest) downloadTestOSMFile() { if _, err := os.Stat(ft.FileName); os.IsNotExist(err) { out, err := os.Create(ft.FileName) if err != nil { - ft.T.Fatal(err) + ft.Fatal(err) } defer out.Close() resp, err := http.Get(ft.FileURL) if err != nil { - ft.T.Fatal(err) + ft.Fatal(err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - ft.T.Fatalf("test status code invalid: %v", resp.StatusCode) + ft.Fatalf("test status code invalid: %v", resp.StatusCode) } if _, err := io.Copy(out, resp.Body); err != nil { - ft.T.Fatal(err) + ft.Fatal(err) } } else if err != nil { - ft.T.Fatal(err) + ft.Fatal(err) } } @@ -202,21 +202,20 @@ func (ft *OSMFileTest) testDecode() { f, err := os.Open(ft.FileName) if err != nil { - ft.T.Fatal(err) + ft.Fatal(err) } defer f.Close() d := newDecoder(context.Background(), &Scanner{}, f) err = d.Start(runtime.GOMAXPROCS(-1)) if err != nil { - ft.T.Fatal(err) + ft.Fatal(err) } var n *osm.Node var w *osm.Way var r *osm.Relation var nc, wc, rc uint64 - var id string idsOrder := make([]string, 0, len(IDsExpectedOrder)) for { e, err := d.Next() @@ -224,7 +223,7 @@ func (ft *OSMFileTest) testDecode() { if err == io.EOF { break } else if err != nil { - ft.T.Fatal(err) + ft.Fatal(err) } switch v := e.(type) { @@ -233,7 +232,7 @@ func (ft *OSMFileTest) testDecode() { if v.ID == ft.ExpNode.ID { n = v } - id = fmt.Sprintf("node/%d", v.ID) + id := fmt.Sprintf("node/%d", v.ID) if _, ok := IDs[id]; ok { idsOrder = append(idsOrder, id) } @@ -242,7 +241,7 @@ func (ft *OSMFileTest) testDecode() { if v.ID == ft.ExpWay.ID { w = v } - id = fmt.Sprintf("way/%d", v.ID) + id := fmt.Sprintf("way/%d", v.ID) if _, ok := IDs[id]; ok { idsOrder = append(idsOrder, id) } @@ -251,7 +250,7 @@ func (ft *OSMFileTest) testDecode() { if v.ID == ft.ExpRel.ID { r = v } - id = fmt.Sprintf("relation/%d", v.ID) + id := fmt.Sprintf("relation/%d", v.ID) if _, ok := IDs[id]; ok { idsOrder = append(idsOrder, id) } @@ -260,21 +259,21 @@ func (ft *OSMFileTest) testDecode() { d.Close() if !reflect.DeepEqual(ft.ExpNode, n) { - ft.T.Errorf("\nExpected: %#v\nActual: %#v", ft.ExpNode, n) + ft.Errorf("\nExpected: %#v\nActual: %#v", ft.ExpNode, n) } roundCoordinates(w) if !reflect.DeepEqual(ft.ExpWay, w) { - ft.T.Errorf("\nExpected: %#v\nActual: %#v", ft.ExpWay, w) + ft.Errorf("\nExpected: %#v\nActual: %#v", ft.ExpWay, w) } if !reflect.DeepEqual(ft.ExpRel, r) { - ft.T.Errorf("\nExpected: %#v\nActual: %#v", ft.ExpRel, r) + ft.Errorf("\nExpected: %#v\nActual: %#v", ft.ExpRel, r) } if ft.ExpNodeCount != nc || ft.ExpWayCount != wc || ft.ExpRelCount != rc { - ft.T.Errorf("\nExpected %7d nodes, %7d ways, %7d relations\nGot %7d nodes, %7d ways, %7d relations.", + ft.Errorf("\nExpected %7d nodes, %7d ways, %7d relations\nGot %7d nodes, %7d ways, %7d relations.", ft.ExpNodeCount, ft.ExpWayCount, ft.ExpRelCount, nc, wc, rc) } if !reflect.DeepEqual(ft.IDsExpOrder, idsOrder) { - ft.T.Errorf("\nExpected: %v\nGot: %v", ft.IDsExpOrder, idsOrder) + ft.Errorf("\nExpected: %v\nGot: %v", ft.IDsExpOrder, idsOrder) } } diff --git a/osmpbf/encode.go b/osmpbf/encode.go new file mode 100644 index 00000000..796213b7 --- /dev/null +++ b/osmpbf/encode.go @@ -0,0 +1,453 @@ +package osmpbf + +import ( + "bytes" + "encoding/binary" + "io" + "sync" + "time" + + "github.com/paulmach/osm" + "github.com/paulmach/osm/osmpbf/internal/osmpbf" + "google.golang.org/protobuf/proto" +) + +// Encoder is an interface for encoding osm data. The format written is the osm +// pbf format. +type Encoder interface { + io.Closer + Encode(obj osm.Object) error +} + +func NewEncoder(w io.Writer) (Encoder, error) { + encoder := &encoder{ + writer: w, + reverseStringTable: make(map[string]int), + compress: true, + } + + blockHeader := &osmpbf.HeaderBlock{ + RequiredFeatures: []string{"OsmSchema-V0.6", "DenseNodes"}, + } + blockHeaderData, err := proto.Marshal(blockHeader) + if err != nil { + return nil, err + } + + _, err = encoder.write(blockHeaderData, osmHeaderType) + if err != nil { + return nil, err + } + + return encoder, nil +} + +type encoder struct { + writer io.Writer + reverseStringTable map[string]int + lastWrittenType osm.Type + entities []osm.Object + mu sync.Mutex + compress bool +} + +func (e *encoder) Close() error { + return e.flush() +} + +func (e *encoder) flush() error { + if len(e.entities) == 0 { + return nil + } + + block := &osmpbf.PrimitiveBlock{} + encode(block, e.reverseStringTable, e.entities, e.compress) + e.entities = e.entities[:0] + for k := range e.reverseStringTable { + delete(e.reverseStringTable, k) + } + + blockData, err := proto.Marshal(block) + if err != nil { + return nil + } + _, err = e.write(blockData, osmDataType) + return err +} + +func (e *encoder) write(data []byte, osmType string) (n int, err error) { + blob := &osmpbf.Blob{} + blob.RawSize = proto.Int32(int32(len(data))) + if e.compress { + target := &bytes.Buffer{} + // compress the data + writer := zlibWriter(target) + writer.Write(data) + writer.Close() + blob.ZlibData = target.Bytes() + } else { + blob.Raw = data + } + + blobData, err := proto.Marshal(blob) + if err != nil { + return 0, nil + } + + blobHeader := &osmpbf.BlobHeader{ + Type: proto.String(osmType), + Datasize: proto.Int32(int32(len(blobData))), + } + + blobHeaderData, err := proto.Marshal(blobHeader) + if err != nil { + return 0, nil + } + + size := make([]byte, 4) + binary.BigEndian.PutUint32(size, uint32(len(blobHeaderData))) + + if _, err = e.writer.Write(size); err != nil { + return 0, nil + } + + if _, err = e.writer.Write(blobHeaderData); err != nil { + return 0, nil + } + + return e.writer.Write(blobData) +} + +func (e *encoder) Encode(obj osm.Object) error { + e.mu.Lock() + defer e.mu.Unlock() + + if e.lastWrittenType != obj.ObjectID().Type() { + if err := e.flush(); err != nil { + return err + } + e.lastWrittenType = obj.ObjectID().Type() + } + + e.entities = append(e.entities, obj) + if len(e.entities) >= 8000 { + return e.flush() + } + return nil +} + +func encode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, osmGeos []osm.Object, compressed bool) { + groupIdx := 0 + i := 0 + + if block.Stringtable != nil && block.Stringtable.S != nil { + block.Stringtable.S = nil + } + + for i < len(osmGeos) { + var group *osmpbf.PrimitiveGroup = nil + if groupIdx < len(block.Primitivegroup) { + group = block.Primitivegroup[groupIdx] + if group == nil { + group = &osmpbf.PrimitiveGroup{} + } + if group.Dense != nil { + if group.Dense.Denseinfo != nil { + if group.Dense.Denseinfo.Changeset != nil { + group.Dense.Denseinfo.Changeset = nil + } + if group.Dense.Denseinfo.Timestamp != nil { + group.Dense.Denseinfo.Timestamp = nil + } + if group.Dense.Denseinfo.Uid != nil { + group.Dense.Denseinfo.Uid = nil + } + if group.Dense.Denseinfo.UserSid != nil { + group.Dense.Denseinfo.UserSid = nil + } + if group.Dense.Denseinfo.Version != nil { + group.Dense.Denseinfo.Version = nil + } + } + if group.Dense.Id != nil { + group.Dense.Id = nil + } + if group.Dense.KeysVals != nil { + group.Dense.KeysVals = nil + } + if group.Dense.Lat != nil { + group.Dense.Lat = nil + } + if group.Dense.Lon != nil { + group.Dense.Lon = nil + } + } + if group.Changesets != nil { + group.Changesets = nil + } + if group.Ways != nil { + group.Ways = nil + } + if group.Relations != nil { + group.Relations = nil + } + } else { + group = &osmpbf.PrimitiveGroup{} + block.Primitivegroup = append(block.Primitivegroup, group) + } + + currentNodeCount := 0 + groupType := osmGeos[i].ObjectID().Type() + previousNode := &osm.Node{} + if groupType == osm.TypeNode && compressed && group.Dense == nil { + group.Dense = &osmpbf.DenseNodes{Denseinfo: &osmpbf.DenseInfo{}} + } + for i < len(osmGeos) && osmGeos[i].ObjectID().Type() == groupType { + switch groupType { + case osm.TypeNode: + if compressed { + currentNode := osmGeos[i].(*osm.Node) + EncodeDenseNode(block, reverseStringTable, group.Dense, currentNode, previousNode) + previousNode = currentNode + } else { + if currentNodeCount < len(group.Nodes) { + EncodeNode(block, reverseStringTable, group.Nodes[currentNodeCount], osmGeos[i].(*osm.Node)) + } else { + pbfNode := &osmpbf.Node{} + group.Nodes = append(group.Nodes, EncodeNode(block, reverseStringTable, pbfNode, osmGeos[i].(*osm.Node))) + } + currentNodeCount++ + } + case osm.TypeWay: + group.Ways = append(group.Ways, EncodeWay(block, reverseStringTable, osmGeos[i].(*osm.Way))) + case osm.TypeRelation: + group.Relations = append(group.Relations, EncodeRelation(block, reverseStringTable, osmGeos[i].(*osm.Relation))) + } + i++ + } + + if group.Nodes != nil { + for currentNodeCount < len(group.Nodes) { + group.Nodes = group.Nodes[:len(group.Nodes)-1] + } + } + + groupIdx++ + } + + if groupIdx < len(block.Primitivegroup) { + block.Primitivegroup = block.Primitivegroup[:groupIdx] + } +} + +func EncodeDenseNode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, groupDense *osmpbf.DenseNodes, current, previous *osm.Node) { + groupDense.Id = append(groupDense.Id, int64(current.ID-previous.ID)) + // cast block.Granularity to int64 + granularity := block.GetGranularity() + currentLat := EncodeLatLon(current.Lat, block.GetLatOffset(), granularity) + currentLon := EncodeLatLon(current.Lon, block.GetLonOffset(), granularity) + previousLat := EncodeLatLon(previous.Lat, block.GetLatOffset(), granularity) + previousLon := EncodeLatLon(previous.Lon, block.GetLonOffset(), granularity) + latDiff := currentLat - previousLat + lonDiff := currentLon - previousLon + groupDense.Lat = append(groupDense.Lat, latDiff) + groupDense.Lon = append(groupDense.Lon, lonDiff) + + if len(current.Tags) > 0 { + for _, nodeTag := range current.Tags { + groupDense.KeysVals = append(groupDense.KeysVals, EncodeString(block, reverseStringTable, nodeTag.Key)) + groupDense.KeysVals = append(groupDense.KeysVals, EncodeString(block, reverseStringTable, nodeTag.Value)) + } + } + groupDense.KeysVals = append(groupDense.KeysVals, 0) + + if groupDense.Denseinfo != nil { + groupDense.Denseinfo.Changeset = append(groupDense.Denseinfo.Changeset, int64(current.ChangesetID-previous.ChangesetID)) + dateGranularity := block.GetDateGranularity() + if !current.Timestamp.IsZero() { + currentTimeStamp := EncodeTimestamp(current.Timestamp, dateGranularity) + if !previous.Timestamp.IsZero() { + previousTimeStamp := EncodeTimestamp(previous.Timestamp, dateGranularity) + groupDense.Denseinfo.Timestamp = append(groupDense.Denseinfo.Timestamp, currentTimeStamp-previousTimeStamp) + } else { + groupDense.Denseinfo.Timestamp = append(groupDense.Denseinfo.Timestamp, currentTimeStamp) + } + } + groupDense.Denseinfo.Uid = append(groupDense.Denseinfo.Uid, int32(current.UserID-previous.UserID)) + groupDense.Denseinfo.Version = append(groupDense.Denseinfo.Version, int32(current.Version-previous.Version)) + var previousUserNameId int32 = 0 + if previous.User != "" { + previousUserNameId = EncodeString(block, reverseStringTable, previous.User) + } + currentUserNameId := EncodeString(block, reverseStringTable, current.User) + groupDense.Denseinfo.UserSid = append(groupDense.Denseinfo.UserSid, currentUserNameId-previousUserNameId) + } +} + +func EncodeNode(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, pbfNode *osmpbf.Node, node *osm.Node) *osmpbf.Node { + castId := int64(node.ID) + pbfNode.Id = &castId + zero := int32(0) + pbfNode.Info = &osmpbf.Info{ + Version: &zero, + } + if node.ChangesetID != 0 { + changesetId := int64(node.ChangesetID) + pbfNode.Info.Changeset = &changesetId + } + if !node.Timestamp.IsZero() { + timeStamp := EncodeTimestamp(node.Timestamp, block.GetDateGranularity()) + pbfNode.Info.Timestamp = &timeStamp + } + if node.UserID != 0 { + userId := int32(node.UserID) + pbfNode.Info.Uid = &userId + } + userSid := uint32(EncodeString(block, reverseStringTable, node.User)) + pbfNode.Info.UserSid = &userSid + if node.Version != 0 { + nodeVersion := int32(node.Version) + pbfNode.Info.Version = &nodeVersion + } + lat := EncodeLatLon(node.Lat, block.GetLatOffset(), block.GetGranularity()) + pbfNode.Lat = &lat + lon := EncodeLatLon(node.Lon, block.GetLonOffset(), block.GetGranularity()) + pbfNode.Lon = &lon + + if len(node.Tags) > 0 { + for _, tag := range node.Tags { + pbfNode.Keys = append(pbfNode.Keys, uint32(EncodeString(block, reverseStringTable, tag.Key))) + pbfNode.Vals = append(pbfNode.Vals, uint32(EncodeString(block, reverseStringTable, tag.Value))) + } + } else { + pbfNode.Keys = nil + pbfNode.Vals = nil + } + return pbfNode +} + +func EncodeWay(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, way *osm.Way) *osmpbf.Way { + pbfWay := &osmpbf.Way{ + Id: (*int64)(&way.ID), + Info: &osmpbf.Info{}, + } + if way.ChangesetID != 0 { + pbfWay.Info.Changeset = (*int64)(&way.ChangesetID) + } + if !way.Timestamp.IsZero() { + timestamp := EncodeTimestamp(way.Timestamp, block.GetDateGranularity()) + pbfWay.Info.Timestamp = ×tamp + } + if way.UserID != 0 { + userId := int32(way.UserID) + pbfWay.Info.Uid = &userId + } + userId := uint32(EncodeString(block, reverseStringTable, way.User)) + pbfWay.Info.UserSid = &userId + if way.Version != 0 { + wayVersion := int32(way.Version) + pbfWay.Info.Version = &wayVersion + } + + if len(way.Tags) > 0 { + for _, tag := range way.Tags { + pbfWay.Keys = append(pbfWay.Keys, uint32(EncodeString(block, reverseStringTable, tag.Key))) + pbfWay.Vals = append(pbfWay.Vals, uint32(EncodeString(block, reverseStringTable, tag.Value))) + } + } + + if len(way.Nodes) > 0 { + pbfWay.Refs = append(pbfWay.Refs, int64(way.Nodes[0].ID)) + for i := 1; i < len(way.Nodes); i++ { + pbfWay.Refs = append(pbfWay.Refs, int64(way.Nodes[i].ID)-int64(way.Nodes[i-1].ID)) + } + } + return pbfWay +} + +func EncodeRelation(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, relation *osm.Relation) *osmpbf.Relation { + pbfRelation := &osmpbf.Relation{ + Id: (*int64)(&relation.ID), + Info: &osmpbf.Info{ + Changeset: (*int64)(&relation.ChangesetID), + }, + } + if !relation.Timestamp.IsZero() { + timestamp := EncodeTimestamp(relation.Timestamp, block.GetDateGranularity()) + pbfRelation.Info.Timestamp = ×tamp + } + if relation.UserID != 0 { + userId := int32(relation.UserID) + pbfRelation.Info.Uid = &userId + } + userSid := uint32(EncodeString(block, reverseStringTable, relation.User)) + pbfRelation.Info.UserSid = &userSid + relationVersion := int32(relation.Version) + pbfRelation.Info.Version = &relationVersion + + if len(relation.Tags) > 0 { + for _, tag := range relation.Tags { + pbfRelation.Keys = append(pbfRelation.Keys, uint32(EncodeString(block, reverseStringTable, tag.Key))) + pbfRelation.Vals = append(pbfRelation.Vals, uint32(EncodeString(block, reverseStringTable, tag.Value))) + } + } + + if len(relation.Members) > 0 { + pbfRelation.Memids = append(pbfRelation.Memids, relation.Members[0].Ref) + pbfRelation.RolesSid = append(pbfRelation.RolesSid, EncodeString(block, reverseStringTable, relation.Members[0].Role)) + switch relation.Members[0].Type { + case osm.TypeNode: + pbfRelation.Types = append(pbfRelation.Types, osmpbf.Relation_NODE) + case osm.TypeWay: + pbfRelation.Types = append(pbfRelation.Types, osmpbf.Relation_WAY) + case osm.TypeRelation: + pbfRelation.Types = append(pbfRelation.Types, osmpbf.Relation_RELATION) + } + for i := 1; i < len(relation.Members); i++ { + pbfRelation.Memids = append(pbfRelation.Memids, relation.Members[i].Ref-relation.Members[i-1].Ref) + pbfRelation.RolesSid = append(pbfRelation.RolesSid, EncodeString(block, reverseStringTable, relation.Members[i].Role)) + switch relation.Members[i].Type { + case osm.TypeNode: + pbfRelation.Types = append(pbfRelation.Types, osmpbf.Relation_NODE) + case osm.TypeWay: + pbfRelation.Types = append(pbfRelation.Types, osmpbf.Relation_WAY) + case osm.TypeRelation: + pbfRelation.Types = append(pbfRelation.Types, osmpbf.Relation_RELATION) + } + } + } + return pbfRelation +} + +func EncodeLatLon(value float64, offset int64, granularity int32) int64 { + return (int64(value/.000000001) - offset) / int64(granularity) +} + +func EncodeTimestamp(timestamp time.Time, dateGranularity int32) int64 { + ts := timestamp.UnixMilli() / int64(dateGranularity) + if ts < 0 { + return 0 + } + return ts +} + +func EncodeString(block *osmpbf.PrimitiveBlock, reverseStringTable map[string]int, value string) int32 { + if value == "" { + return 0 + } + + if block.Stringtable == nil { + block.Stringtable = &osmpbf.StringTable{} + block.Stringtable.S = append(block.Stringtable.S, "") + reverseStringTable[""] = 0 + } + + if id, ok := reverseStringTable[value]; ok { + return int32(id) + } + + block.Stringtable.S = append(block.Stringtable.S, value) + id := len(block.Stringtable.S) - 1 + reverseStringTable[value] = id + return int32(id) +} diff --git a/osmpbf/encode_test.go b/osmpbf/encode_test.go new file mode 100644 index 00000000..578944a7 --- /dev/null +++ b/osmpbf/encode_test.go @@ -0,0 +1,223 @@ +package osmpbf + +import ( + "bytes" + "context" + "fmt" + "io" + "runtime" + "testing" + + "github.com/paulmach/osm" +) + +func TestEncodeDecode(t *testing.T) { + buffer := bytes.Buffer{} + writer, err := NewEncoder(&buffer) + if err != nil { + t.Fatal(err) + } + + // write node + err = writer.Encode(en) + if err != nil { + t.Fatal(err) + } + // write way + err = writer.Encode(ew) + if err != nil { + t.Fatal(err) + } + // write relation + err = writer.Encode(er) + if err != nil { + t.Fatal(err) + } + writer.Close() + + d := newDecoder(context.Background(), &Scanner{}, &buffer) + err = d.Start(runtime.GOMAXPROCS(-1)) + if err != nil { + t.Fatal(err) + } + + for { + e, err := d.Next() + + if err == io.EOF { + break + } else if err != nil { + t.Fatal(err) + } + + switch v := e.(type) { + case *osm.Node: + err = nodeEquals(en, v) + if err != nil { + t.Fatal(err) + } + case *osm.Way: + err = wayEquals(ew, v) + if err != nil { + t.Fatal(err) + } + case *osm.Relation: + err = relationEquals(er, v) + if err != nil { + t.Fatal(err) + } + } + } + d.Close() +} + +func nodeEquals(en, node *osm.Node) error { + if node.ID != en.ID { + return fmt.Errorf("node id mismatch: %d != %d", node.ID, en.ID) + } + if node.Lat != en.Lat { + return fmt.Errorf("node lat mismatch: %f != %f", node.Lat, en.Lat) + } + if node.Lon != en.Lon { + return fmt.Errorf("node lon mismatch: %f != %f", node.Lon, en.Lon) + } + if node.User != en.User { + return fmt.Errorf("node user mismatch: %s != %s", node.User, en.User) + } + if node.UserID != en.UserID { + return fmt.Errorf("node user id mismatch: %d != %d", node.UserID, en.UserID) + } + if node.Visible != en.Visible { + return fmt.Errorf("node visible mismatch: %v != %v", node.Visible, en.Visible) + } + if node.Version != en.Version { + return fmt.Errorf("node version mismatch: %d != %d", node.Version, en.Version) + } + if node.ChangesetID != en.ChangesetID { + return fmt.Errorf("node changeset id mismatch: %d != %d", node.ChangesetID, en.ChangesetID) + } + if node.Timestamp != en.Timestamp { + return fmt.Errorf("node timestamp mismatch: %s != %s", node.Timestamp, en.Timestamp) + } + if len(node.Tags) != len(en.Tags) { + return fmt.Errorf("node tags length mismatch: %d != %d", len(node.Tags), len(en.Tags)) + } + for k, v := range node.Tags { + if en.Tags[k] != v { + return fmt.Errorf("node tag mismatch: %s != %s", en.Tags[k], v) + } + } + return nil +} + +func wayEquals(ew, way *osm.Way) error { + if way.ID != ew.ID { + return fmt.Errorf("way id mismatch: %d != %d", way.ID, ew.ID) + } + if way.User != ew.User { + return fmt.Errorf("way user mismatch: %s != %s", way.User, ew.User) + } + if way.UserID != ew.UserID { + return fmt.Errorf("way user id mismatch: %d != %d", way.UserID, ew.UserID) + } + if way.Visible != ew.Visible { + return fmt.Errorf("way visible mismatch: %v != %v", way.Visible, ew.Visible) + } + if way.Version != ew.Version { + return fmt.Errorf("way version mismatch: %d != %d", way.Version, ew.Version) + } + if way.ChangesetID != ew.ChangesetID { + return fmt.Errorf("way changeset id mismatch: %d != %d", way.ChangesetID, ew.ChangesetID) + } + if way.Timestamp != ew.Timestamp { + return fmt.Errorf("way timestamp mismatch: %s != %s", way.Timestamp, ew.Timestamp) + } + if len(way.Tags) != len(ew.Tags) { + return fmt.Errorf("way tags length mismatch: %d != %d", len(way.Tags), len(ew.Tags)) + } + for k, v := range way.Tags { + if ew.Tags[k] != v { + return fmt.Errorf("way tag mismatch: %s != %s", ew.Tags[k], v) + } + } + if len(way.Nodes) != len(ew.Nodes) { + return fmt.Errorf("way nodes length mismatch: %d != %d", len(way.Nodes), len(ew.Nodes)) + } + for i, v := range way.Nodes { + if ew.Nodes[i] != v { + return fmt.Errorf("way node mismatch: %v != %v", ew.Nodes[i], v) + } + } + return nil +} + +func relationEquals(er, relation *osm.Relation) error { + if relation.ID != er.ID { + return fmt.Errorf("relation id mismatch: %d != %d", relation.ID, er.ID) + } + if relation.User != er.User { + return fmt.Errorf("relation user mismatch: %s != %s", relation.User, er.User) + } + if relation.UserID != er.UserID { + return fmt.Errorf("relation user id mismatch: %d != %d", relation.UserID, er.UserID) + } + if relation.Visible != er.Visible { + return fmt.Errorf("relation visible mismatch: %v != %v", relation.Visible, er.Visible) + } + if relation.Version != er.Version { + return fmt.Errorf("relation version mismatch: %d != %d", relation.Version, er.Version) + } + if relation.ChangesetID != er.ChangesetID { + return fmt.Errorf("relation changeset id mismatch: %d != %d", relation.ChangesetID, er.ChangesetID) + } + if relation.Timestamp != er.Timestamp { + return fmt.Errorf("relation timestamp mismatch: %s != %s", relation.Timestamp, er.Timestamp) + } + if len(relation.Tags) != len(er.Tags) { + return fmt.Errorf("relation tags length mismatch: %d != %d", len(relation.Tags), len(er.Tags)) + } + for k, v := range relation.Tags { + if er.Tags[k] != v { + return fmt.Errorf("relation tag mismatch: %s != %s", er.Tags[k], v) + } + } + if len(relation.Members) != len(er.Members) { + return fmt.Errorf("relation members length mismatch: %d != %d", len(relation.Members), len(er.Members)) + } + for i, member := range relation.Members { + expectedMember := er.Members[i] + if member.ChangesetID != expectedMember.ChangesetID { + return fmt.Errorf("relation member changeset id mismatch: %d != %d", member.ChangesetID, expectedMember.ChangesetID) + } + if member.Role != expectedMember.Role { + return fmt.Errorf("relation member role mismatch: %s != %s", member.Role, expectedMember.Role) + } + if member.Type != expectedMember.Type { + return fmt.Errorf("relation member type mismatch: %s != %s", member.Type, expectedMember.Type) + } + if member.Ref != expectedMember.Ref { + return fmt.Errorf("relation member ref mismatch: %d != %d", member.Ref, expectedMember.Ref) + } + if member.Lat != expectedMember.Lat { + return fmt.Errorf("relation member lat mismatch: %f != %f", member.Lat, expectedMember.Lat) + } + if member.Lon != expectedMember.Lon { + return fmt.Errorf("relation member lon mismatch: %f != %f", member.Lon, expectedMember.Lon) + } + if member.Orientation != expectedMember.Orientation { + return fmt.Errorf("relation member orientation mismatch: %d != %d", member.Orientation, expectedMember.Orientation) + } + if member.Version != expectedMember.Version { + return fmt.Errorf("relation member version mismatch: %d != %d", member.Version, expectedMember.Version) + } + if len(member.Nodes) != len(expectedMember.Nodes) { + return fmt.Errorf("relation member nodes length mismatch: %d != %d", len(member.Nodes), len(expectedMember.Nodes)) + } + for j, node := range member.Nodes { + if expectedMember.Nodes[j] != node { + return fmt.Errorf("relation member node mismatch: %v != %v", expectedMember.Nodes[j], node) + } + } + } + return nil +} diff --git a/osmpbf/zlib_cgo.go b/osmpbf/zlib_cgo.go index fbcf3f66..b6aaa949 100644 --- a/osmpbf/zlib_cgo.go +++ b/osmpbf/zlib_cgo.go @@ -1,3 +1,4 @@ +//go:build cgo // +build cgo package osmpbf @@ -12,3 +13,7 @@ import ( func zlibReader(data []byte) (io.ReadCloser, error) { return czlib.NewReader(bytes.NewReader(data)) } + +func zlibWriter(w io.Writer) io.WriteCloser { + return czlib.NewWriter(w) +} diff --git a/osmpbf/zlib_go.go b/osmpbf/zlib_go.go index ddae9b86..6b7b657a 100644 --- a/osmpbf/zlib_go.go +++ b/osmpbf/zlib_go.go @@ -1,3 +1,4 @@ +//go:build !cgo // +build !cgo package osmpbf @@ -11,3 +12,7 @@ import ( func zlibReader(data []byte) (io.ReadCloser, error) { return zlib.NewReader(bytes.NewReader(data)) } + +func zlibWriter(w io.Writer) io.WriteCloser { + return zlib.NewWriter(w) +} diff --git a/relation_test.go b/relation_test.go index bb19e92d..63b4b5e1 100644 --- a/relation_test.go +++ b/relation_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "encoding/xml" + "os" "reflect" "testing" "time" @@ -199,7 +200,10 @@ func TestRelation_MarshalXML(t *testing.T) { } // blanket xml test - data = readFile(t, "testdata/relation-updates.osm") + data, err = os.ReadFile("testdata/relation-updates.osm") + if err != nil { + t.Fatalf("could not read file: %v", err) + } osm := &OSM{} err = xml.Unmarshal(data, &osm) diff --git a/replication/changesets.go b/replication/changesets.go index f149c8cf..65eb2730 100644 --- a/replication/changesets.go +++ b/replication/changesets.go @@ -6,7 +6,6 @@ import ( "context" "fmt" "io" - "io/ioutil" "net/http" "strconv" @@ -88,7 +87,7 @@ func (ds *Datasource) fetchChangesetState(ctx context.Context, n ChangesetSeqNum } } - data, err := ioutil.ReadAll(resp.Body) + data, err := io.ReadAll(resp.Body) if err != nil { return nil, err } diff --git a/replication/interval.go b/replication/interval.go index 8c258bd8..d322d855 100644 --- a/replication/interval.go +++ b/replication/interval.go @@ -6,7 +6,7 @@ import ( "context" "encoding/xml" "fmt" - "io/ioutil" + "io" "net/http" "strconv" "time" @@ -219,7 +219,7 @@ func (ds *Datasource) fetchState(ctx context.Context, n SeqNum) (*State, error) } } - data, err := ioutil.ReadAll(resp.Body) + data, err := io.ReadAll(resp.Body) if err != nil { return nil, err } diff --git a/way_test.go b/way_test.go index 59d7ab38..d669bbd8 100644 --- a/way_test.go +++ b/way_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "encoding/xml" + "os" "reflect" "testing" "time" @@ -284,7 +285,10 @@ func TestWay_MarshalXML(t *testing.T) { } // blanket xml test - data = readFile(t, "testdata/way-updates.osm") + data, err = os.ReadFile("testdata/way-updates.osm") + if err != nil { + t.Fatalf("unable to read file: %v", err) + } osm := &OSM{} err = xml.Unmarshal(data, &osm)