diff options
Diffstat (limited to 'mountain_pass')
| -rw-r--r-- | mountain_pass/extract.go | 32 | ||||
| -rw-r--r-- | mountain_pass/extract_test.go | 21 |
2 files changed, 51 insertions, 2 deletions
diff --git a/mountain_pass/extract.go b/mountain_pass/extract.go index 705dbd3..9643603 100644 --- a/mountain_pass/extract.go +++ b/mountain_pass/extract.go @@ -24,14 +24,28 @@ const insertStatement = ` longitude = excluded.longitude ` -func ExtractMountainPasses(ctx context.Context, pbfPath string, db *sql.DB) (int, error) { +// ExtractMountainPasses scans pbfPath for mountain pass nodes and upserts them +// into the database. An optional reporter is invoked as scanning progresses +// with bytes scanned and the total file size, so callers (e.g. the CLI) can +// render progress. +func ExtractMountainPasses(ctx context.Context, pbfPath string, db *sql.DB, reporters ...func(current, total int64)) (int, error) { file, err := os.Open(pbfPath) if err != nil { return 0, fmt.Errorf("failed to open pbf file: %w", err) } defer file.Close() - scanner := osmpbf.New(ctx, file, 4) + info, err := file.Stat() + if err != nil { + return 0, fmt.Errorf("failed to stat pbf file: %w", err) + } + total := info.Size() + report := func(int64, int64) {} + if len(reporters) > 0 { + report = reporters[0] + } + reader := &countingReader{r: file} + scanner := osmpbf.New(ctx, reader, 4) scanner.SkipWays = true scanner.SkipRelations = true scanner.FilterNode = isMountainPass @@ -50,6 +64,7 @@ func ExtractMountainPasses(ctx context.Context, pbfPath string, db *sql.DB) (int count := 0 for scanner.Scan() { + report(reader.n, total) node, ok := scanner.Object().(*osm.Node) if !ok { continue @@ -74,6 +89,19 @@ func ExtractMountainPasses(ctx context.Context, pbfPath string, db *sql.DB) (int return count, nil } +// countingReader tracks the number of bytes read from an underlying reader, +// used to report scan progress against the PBF file size. +type countingReader struct { + r io.Reader + n int64 +} + +func (c *countingReader) Read(p []byte) (int, error) { + n, err := c.r.Read(p) + c.n += int64(n) + return n, err +} + func isMountainPass(node *osm.Node) bool { if node.Tags.Find("mountain_pass") == "yes" { return true diff --git a/mountain_pass/extract_test.go b/mountain_pass/extract_test.go index 636372c..7eae33b 100644 --- a/mountain_pass/extract_test.go +++ b/mountain_pass/extract_test.go @@ -1,10 +1,13 @@ package mountain_pass import ( + "io" + "strings" "testing" "github.com/paulmach/osm" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestIsMountainPass(t *testing.T) { @@ -47,3 +50,21 @@ func TestParseElevation(t *testing.T) { assert.Equal(t, c.ok, ok) } } + +func TestCountingReaderTracksBytes(t *testing.T) { + const content = "0123456789abcdef" + reader := &countingReader{r: strings.NewReader(content)} + + buf := make([]byte, 4) + var total int64 + for { + n, err := reader.Read(buf) + total += int64(n) + require.Equal(t, total, reader.n) + if err == io.EOF { + break + } + require.NoError(t, err) + } + assert.Equal(t, int64(len(content)), reader.n) +} |
