From 75ab60f609eeaf2d4df9f24175993612319d25c1 Mon Sep 17 00:00:00 2001 From: Richard Hull Date: Sun, 22 Jun 2025 20:31:04 +0100 Subject: [PATCH 1/4] Reprocess postcode units (truncate to 6dp) --- .dockerignore | 8 ++ .editorconfig | 57 +++++++++++ .github/dependabot.yml | 10 ++ .github/workflows/build.yml | 131 ++++++++++++++++++++++++ .gitignore | 2 + .vscode/launch.json | 16 +++ ATTRIBUTION.md | 7 ++ Dockerfile | 41 ++++++++ README.md | 11 +- extraction/postcode_compressor.go | 161 ++++++++++++++++++++++++++++++ go.mod | 15 +++ go.sum | 89 +++++++++++++++++ main.go | 9 ++ 13 files changed, 556 insertions(+), 1 deletion(-) create mode 100644 .dockerignore create mode 100644 .editorconfig create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/build.yml create mode 100644 .vscode/launch.json create mode 100644 ATTRIBUTION.md create mode 100644 Dockerfile create mode 100644 extraction/postcode_compressor.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..77f42bd0 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +.github +.vscode +data/ +LICENSE.md +README.md +ATTRIBUTION.md +.env* +docker-compose.yml diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..130b7717 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,57 @@ +# EditorConfig is awesome: https://EditorConfig.org + +# Top-most EditorConfig file +root = true + +# Default settings for all files +[*] +charset = utf-8 +end_of_line = lf +# insert_final_newline = true +trim_trailing_whitespace = true +indent_style = space +indent_size = 4 + +# Python-specific settings +[*.{py,pyw}] +# Conform to PEP 8 recommendations +indent_style = space +indent_size = 4 +max_line_length = 120 + +# Docstring settings +[*.{py,pyw}] +# Ensure consistent docstring formatting +max_line_length = 72 + +# Test files may have different conventions +[test_*.py] +max_line_length = 120 + +# Configuration and requirement files +[{pyproject.toml,setup.cfg,requirements*.txt}] +indent_style = space +indent_size = 2 + +# YAML files often use 2-space indentation +[*.{yaml,yml}] +indent_style = space +indent_size = 2 + +[*.{json}] +indent_style = space +indent_size = 2 + +# Makefiles require tabs +[Makefile] +indent_style = tab + +# Ignore certain file types or paths +[{*.min.py,__pycache__/**}] +insert_final_newline = false +trim_trailing_whitespace = false + +[*.go] +indent_style = tab +tab_width = 4 +max_line_length = 100 diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..2fccaad8 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: gomod + directory: / + schedule: + interval: daily + - package-ecosystem: github-actions + directory: / + schedule: + interval: daily diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..00bb317e --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,131 @@ +name: Build and publish Docker image + +on: + push: + branches: ["main"] + tags: ["v*.*.*"] + pull_request: + branches: ["main"] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-test: + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + checks: write + + services: + docker: + image: docker:dind + options: --privileged + + steps: + - uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: "1.24" + + - name: Install testcontainers dependencies + run: docker --version + + - name: Cache Docker images + uses: actions/cache@v4 + with: + path: /var/lib/docker + key: ${{ runner.os }}-docker-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-docker- + + - name: Test + run: | + go install gotest.tools/gotestsum@latest + mkdir -p ./test-reports/ + gotestsum --junitfile=./test-reports/junit.xml --format github-actions -- -v -coverprofile=profile.cov -coverpkg=./... ./... + + - name: Collect test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-reports + path: ./test-reports/ + + - name: Publish test results + uses: mikepenz/action-junit-report@v5 + if: always() + with: + report_paths: "./test-reports/junit.xml" + comment: true + include_passed: true + detailed_summary: true + + - name: Upload coverage to Coveralls + uses: shogo82148/actions-goveralls@v1 + with: + path-to-profile: profile.cov + + - name: Lint + uses: golangci/golangci-lint-action@v8 + + docker-publish: + concurrency: + group: "publishing" + needs: + - build-and-test + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository with full history + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: "Log into registry: ${{ env.REGISTRY }}" + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=semver,pattern=v{{version}} + type=semver,pattern=v{{major}}.{{minor}} + type=semver,pattern=v{{major}},enable=${{ !startsWith(github.ref, 'refs/tags/v0.') }} + type=sha + labels: | + maintainer=rm-hull + org.opencontainers.image.description=REST API for GPS routes + org.opencontainers.image.licenses=MIT + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/amd64 #,linux/arm64 + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.gitignore b/.gitignore index aaadf736..5bf28ae8 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,5 @@ go.work.sum # Editor/IDE # .idea/ # .vscode/ + +data/ \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..5bd10df0 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,16 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Launch HTTP server", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "main.go", + "args": [] + } + ] +} diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md new file mode 100644 index 00000000..5c50810c --- /dev/null +++ b/ATTRIBUTION.md @@ -0,0 +1,7 @@ +# Attribution + +You may use this data under the Open Government License v3.0, and must include the following copyright notices: + +- Contains OS data © Crown copyright and database right 2020 +- Contains Royal Mail data © Royal Mail copyright and database right 2020 +- Source: Office for National Statistics licensed under the Open Government Licence v.3.0 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..ff541e2e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,41 @@ +FROM golang:1.24-alpine AS build + +RUN apk update && \ + apk add --no-cache ca-certificates tzdata git build-base && \ + update-ca-certificates + +RUN adduser -D -g '' appuser + +WORKDIR /app + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . + +ENV CGO_ENABLED=1 +ENV GOOS=linux + +RUN go build -ldflags="-w -s" -o postcode-polygons . + +FROM alpine:latest AS runtime +ENV GIN_MODE=release +ENV TZ=UTC + +RUN apk --no-cache add curl ca-certificates tzdata && \ + update-ca-certificates + +RUN adduser -D -g '' appuser +WORKDIR /app + +COPY --from=build /app/postcode-polygons . +COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +COPY --from=build /usr/share/zoneinfo /usr/share/zoneinfo + +USER appuser +EXPOSE 8080/tcp + +HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ + CMD curl -f http://localhost:8080/healthz || exit 1 + +ENTRYPOINT ["./postcode-polygons"] diff --git a/README.md b/README.md index 835047df..3d56e57c 100644 --- a/README.md +++ b/README.md @@ -1 +1,10 @@ -# postcode-polygons \ No newline at end of file +# Postcode Polygons + +## Data extraction and reprocessing + +First download (or otherwise generate) the **gb-postcodes-v5.tar.bz2** data file. See https://longair.net/blog/2021/08/23/open-data-gb-postcode-unit-boundaries/. + +```console +$ curl https://postcodes-mapit-static.s3.eu-west-2.amazonaws.com/data/gb-postcodes-v5.tar.bz2 -O data/gb-postcodes-v5.tar.bz2 +$ go run main.go +``` diff --git a/extraction/postcode_compressor.go b/extraction/postcode_compressor.go new file mode 100644 index 00000000..f407d398 --- /dev/null +++ b/extraction/postcode_compressor.go @@ -0,0 +1,161 @@ +package extraction + +import ( + "archive/tar" + "fmt" + "io" + "log" + "os" + "path/filepath" + "strings" + + "github.com/dsnet/compress/bzip2" + "github.com/dustin/go-humanize" + "github.com/fatih/color" + "github.com/paulmach/orb" + "github.com/paulmach/orb/geojson" +) + +func Extract(tarFile string) { + + f, err := os.Open(tarFile) + if err != nil { + log.Fatalf("Error opening file: %v", err) + } + defer func() { + if err := f.Close(); err != nil { + log.Printf("Error closing file: %v", err) + } + }() + + bz2Reader, err := bzip2.NewReader(f, &bzip2.ReaderConfig{}) + if err != nil { + log.Fatalf("Error creating bzip2 reader: %v", err) + } + tarReader := tar.NewReader(bz2Reader) + + skipped := color.New(color.FgBlue).SprintFunc() + successful := color.New(color.FgGreen).SprintFunc() + + for { + header, err := tarReader.Next() + if err != nil { + break + } + + if header.Typeflag == tar.TypeReg && strings.HasPrefix(header.Name, "gb-postcodes-v5/units/") { + + filename := filepath.Base(header.Name) + if exists, err := os.Stat("./data/postcodes/" + filename + ".bz2"); err == nil && !exists.IsDir() { + log.Printf("Skipping file %s (already exists)", skipped(filename)) + continue + } + + content := make([]byte, header.Size) + _, err := io.ReadFull(tarReader, content) + if err != nil { + log.Fatalf("Error reading file %s: %v", header.Name, err) + } + + processed, err := reprocessFile(content) + if err != nil { + log.Fatalf("Error processing file %s: %v", header.Name, err) + } + + outputFile := "./data/postcodes/" + filename + ".bz2" + newSize, err := compressFile(outputFile, processed) + if err != nil { + log.Fatalf("Error compressing file %s: %v", outputFile, err) + } + + log.Printf("Processed %s: original size %s -> %s (%0.2f%% reduction)\n", + successful(filename), + humanize.Bytes(uint64(header.Size)), + humanize.Bytes(uint64(newSize)), + 100-float64(newSize)/float64(header.Size)*100) + + + } else { + log.Printf("Skipping: %v\n", skipped(header.Name)) + } + } +} + +func compressFile(filename string, content []byte) (int, error) { + f, err := os.Create(filename) + if err != nil { + return 0, fmt.Errorf("error creating output file: %w", err) + } + defer func() { + if err := f.Close(); err != nil { + log.Printf("Error closing file %s: %v", filename, err) + } + }() + + w, err := bzip2.NewWriter(f, &bzip2.WriterConfig{Level: bzip2.BestCompression}) + if err != nil { + return 0, fmt.Errorf("error creating bzip2 writer: %w", err) + } + defer func() { + if err := w.Close(); err != nil { + log.Printf("Error closing bzip2 writer for file %s: %v", filename, err) + } + }() + + _, err = w.Write(content) + if err != nil { + return 0, fmt.Errorf("error writing bzip2 file: %w", err) + } + + err = w.Close() + if err != nil { + return 0, fmt.Errorf("error closing bzip2 writer: %w", err) + } + + return int(w.OutputOffset), nil +} + +func reprocessFile(content []byte) ([]byte, error) { + fc, err := geojson.UnmarshalFeatureCollection(content) + if err != nil { + return nil, fmt.Errorf("error unmarshalling GeoJSON: %w", err) + } + + for _, feature := range fc.Features { + postcode := feature.Properties["postcodes"].(string) + + truncateCoordinates(feature) + delete(feature.Properties, "mapit_code") + delete(feature.Properties, "postcodes") + feature.Properties["postcode"] = postcode + } + + return fc.MarshalJSON() +} + +func truncateCoordinates(feature *geojson.Feature) { + if polygon, ok := feature.Geometry.(orb.Polygon); ok { + truncatePolygon(&polygon) + } else if multiPolygon, ok := feature.Geometry.(orb.MultiPolygon); ok { + for i := range multiPolygon { + truncatePolygon(&multiPolygon[i]) + } + } else { + log.Fatalf("Geometry type %T not supported for truncation", feature.Geometry.GeoJSONType()) + } +} + +func truncatePolygon(polygon *orb.Polygon) { + for i := range *polygon { + ring := (*polygon)[i] + for j := range ring { + point := &ring[j] + point[0] = truncate(point.X()) + point[1] = truncate(point.Y()) + } + } +} + +func truncate(value float64) float64 { + return float64(int(value*1e6)) / 1e6 +} diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..17fdcc21 --- /dev/null +++ b/go.mod @@ -0,0 +1,15 @@ +module postcode-polygons + +go 1.24.4 + +require github.com/paulmach/orb v0.11.1 + +require ( + github.com/dsnet/compress v0.0.1 + github.com/dustin/go-humanize v1.0.1 + github.com/fatih/color v1.18.0 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + go.mongodb.org/mongo-driver v1.11.4 // indirect + golang.org/x/sys v0.25.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..666ec6fd --- /dev/null +++ b/go.sum @@ -0,0 +1,89 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= +github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= +github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU= +github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= +github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= +github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.mongodb.org/mongo-driver v1.11.4 h1:4ayjakA013OdpGyL2K3ZqylTac/rMjrJOMZ1EHizXas= +go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go new file mode 100644 index 00000000..9c52a062 --- /dev/null +++ b/main.go @@ -0,0 +1,9 @@ +package main + +import ( + "postcode-polygons/extraction" +) + +func main() { + extraction.Extract("./data/gb-postcodes-v5.tar.bz2") +} From 8a7c474f622180f949d3290d17d9a35d57d1c95e Mon Sep 17 00:00:00 2001 From: Richard Hull Date: Sun, 22 Jun 2025 20:38:10 +0100 Subject: [PATCH 2/4] PR comments addressed --- Dockerfile | 3 +-- extraction/postcode_compressor.go | 13 +++++-------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index ff541e2e..278894dc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM golang:1.24-alpine AS build RUN apk update && \ - apk add --no-cache ca-certificates tzdata git build-base && \ + apk add --no-cache ca-certificates tzdata git && \ update-ca-certificates RUN adduser -D -g '' appuser @@ -13,7 +13,6 @@ RUN go mod download COPY . . -ENV CGO_ENABLED=1 ENV GOOS=linux RUN go build -ldflags="-w -s" -o postcode-polygons . diff --git a/extraction/postcode_compressor.go b/extraction/postcode_compressor.go index f407d398..39331baa 100644 --- a/extraction/postcode_compressor.go +++ b/extraction/postcode_compressor.go @@ -40,6 +40,9 @@ func Extract(tarFile string) { for { header, err := tarReader.Next() if err != nil { + if err != io.EOF { + log.Printf("Error reading from tar archive: %v", err) + } break } @@ -68,13 +71,12 @@ func Extract(tarFile string) { log.Fatalf("Error compressing file %s: %v", outputFile, err) } - log.Printf("Processed %s: original size %s -> %s (%0.2f%% reduction)\n", + log.Printf("Processed file %s: original size %s -> %s (%0.2f%% reduction)\n", successful(filename), humanize.Bytes(uint64(header.Size)), humanize.Bytes(uint64(newSize)), 100-float64(newSize)/float64(header.Size)*100) - } else { log.Printf("Skipping: %v\n", skipped(header.Name)) } @@ -96,18 +98,13 @@ func compressFile(filename string, content []byte) (int, error) { if err != nil { return 0, fmt.Errorf("error creating bzip2 writer: %w", err) } - defer func() { - if err := w.Close(); err != nil { - log.Printf("Error closing bzip2 writer for file %s: %v", filename, err) - } - }() _, err = w.Write(content) if err != nil { return 0, fmt.Errorf("error writing bzip2 file: %w", err) } - err = w.Close() + err = w.Close() // Ensure to close the writer to flush the data, so that the output offset is correct if err != nil { return 0, fmt.Errorf("error closing bzip2 writer: %w", err) } From 0639be935bfffe3a0469c4f9305bc07cc8634036 Mon Sep 17 00:00:00 2001 From: Richard Hull Date: Sun, 22 Jun 2025 20:44:04 +0100 Subject: [PATCH 3/4] PR comments addressed --- extraction/postcode_compressor.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/extraction/postcode_compressor.go b/extraction/postcode_compressor.go index 39331baa..2534beaa 100644 --- a/extraction/postcode_compressor.go +++ b/extraction/postcode_compressor.go @@ -119,7 +119,10 @@ func reprocessFile(content []byte) ([]byte, error) { } for _, feature := range fc.Features { - postcode := feature.Properties["postcodes"].(string) + postcode, ok := feature.Properties["postcodes"].(string) + if !ok { + return nil, fmt.Errorf("missing or invalid 'postcodes' property in feature %s", feature.ID) + } truncateCoordinates(feature) delete(feature.Properties, "mapit_code") From 30a173bfbf075303e9a4729dd5301aa2ad141b71 Mon Sep 17 00:00:00 2001 From: Richard Hull Date: Sun, 22 Jun 2025 20:48:35 +0100 Subject: [PATCH 4/4] Tweak editor config --- .editorconfig | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/.editorconfig b/.editorconfig index 130b7717..46dbffb4 100644 --- a/.editorconfig +++ b/.editorconfig @@ -12,27 +12,6 @@ trim_trailing_whitespace = true indent_style = space indent_size = 4 -# Python-specific settings -[*.{py,pyw}] -# Conform to PEP 8 recommendations -indent_style = space -indent_size = 4 -max_line_length = 120 - -# Docstring settings -[*.{py,pyw}] -# Ensure consistent docstring formatting -max_line_length = 72 - -# Test files may have different conventions -[test_*.py] -max_line_length = 120 - -# Configuration and requirement files -[{pyproject.toml,setup.cfg,requirements*.txt}] -indent_style = space -indent_size = 2 - # YAML files often use 2-space indentation [*.{yaml,yml}] indent_style = space @@ -46,11 +25,6 @@ indent_size = 2 [Makefile] indent_style = tab -# Ignore certain file types or paths -[{*.min.py,__pycache__/**}] -insert_final_newline = false -trim_trailing_whitespace = false - [*.go] indent_style = tab tab_width = 4