From 558f7c3ba1a815c7ceb3f00e07e5923c349b6421 Mon Sep 17 00:00:00 2001 From: Richard Hull Date: Thu, 26 Jun 2025 00:41:44 +0100 Subject: [PATCH 1/3] Code reorg --- .vscode/launch.json | 36 ++++--- README.md | 11 ++- cmd/api_server.go | 68 +++++++++++++ .../extract_data.go | 57 +++-------- internal/file_operations.go | 68 +++++++++++++ main.go | 95 +++++-------------- routes/search.go | 34 +------ 7 files changed, 212 insertions(+), 157 deletions(-) create mode 100644 cmd/api_server.go rename extraction/postcode_compressor.go => cmd/extract_data.go (66%) create mode 100644 internal/file_operations.go diff --git a/.vscode/launch.json b/.vscode/launch.json index 5bd10df0..33e423e4 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,16 +1,24 @@ { - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "name": "Launch HTTP server", - "type": "go", - "request": "launch", - "mode": "auto", - "program": "main.go", - "args": [] - } - ] + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Launch HTTP server", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "main.go", + "args": ["api-server"] + }, + { + "name": "Launch data extraction", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "main.go", + "args": ["extract-data"] + } + ] } diff --git a/README.md b/README.md index 3d56e57c..26127a15 100644 --- a/README.md +++ b/README.md @@ -6,5 +6,14 @@ First download (or otherwise generate) the **gb-postcodes-v5.tar.bz2** data file ```console $ curl https://postcodes-mapit-static.s3.eu-west-2.amazonaws.com/data/gb-postcodes-v5.tar.bz2 -O data/gb-postcodes-v5.tar.bz2 -$ go run main.go +$ go run main.go extract-data +``` + +This will regenerate the (checked-in) data files under `./data/postcodes`. There is typically no need to run this unless +the NSUL datafile got updated + +## Starting the API server + +```console +$ go run main.go api-server ``` diff --git a/cmd/api_server.go b/cmd/api_server.go new file mode 100644 index 00000000..ff5700e5 --- /dev/null +++ b/cmd/api_server.go @@ -0,0 +1,68 @@ +package cmd + +import ( + "fmt" + "log" + "os" + "postcode-polygons/routes" + spatialindex "postcode-polygons/spatial-index" + + "github.com/Depado/ginprom" + "github.com/aurowora/compress" + "github.com/gin-contrib/cors" + "github.com/gin-contrib/pprof" + "github.com/gin-gonic/gin" + "github.com/tavsec/gin-healthcheck/checks" + cachecontrol "go.eigsys.de/gin-cachecontrol/v2" + + healthcheck "github.com/tavsec/gin-healthcheck" + hc_config "github.com/tavsec/gin-healthcheck/config" +) + +func ApiServer(zipFile string, port int, debug bool) { + if _, err := os.Stat(zipFile); os.IsNotExist(err) { + log.Fatalf("CodePoint zip file does not exist: %s", zipFile) + } + + log.Printf("Loading CodePoint data from: %s", zipFile) + spatialIndex, err := spatialindex.NewCodePointSpatialIndex(zipFile) + if err != nil { + log.Fatalf("failed to create spatial index: %v", err) + } + log.Printf("CodePoint spatial index created with %d entries", spatialIndex.Len()) + + r := gin.New() + + prometheus := ginprom.New( + ginprom.Engine(r), + ginprom.Path("/metrics"), + ginprom.Ignore("/healthz"), + ) + + r.Use( + gin.Recovery(), + gin.LoggerWithWriter(gin.DefaultWriter, "/healthz", "/metrics"), + prometheus.Instrument(), + compress.Compress(), + cachecontrol.New(cachecontrol.CacheAssetsForeverPreset), + cors.Default(), + ) + + if debug { + log.Println("WARNING: pprof endpoints are enabled and exposed. Do not run with this flag in production.") + pprof.Register(r) + } + + err = healthcheck.New(r, hc_config.DefaultConfig(), []checks.Check{}) + if err != nil { + log.Fatalf("failed to initialize healthcheck: %v", err) + } + + r.GET("/v1/postcode/codepoints", routes.CodePointSearch(spatialIndex)) + r.GET("/v1/postcode/polygons", routes.PolygonSearch(spatialIndex)) + + addr := fmt.Sprintf(":%d", port) + log.Printf("Starting HTTP API Server on port %d...", port) + err = r.Run(addr) + log.Fatalf("HTTP API Server failed to start on port %d: %v", port, err) +} diff --git a/extraction/postcode_compressor.go b/cmd/extract_data.go similarity index 66% rename from extraction/postcode_compressor.go rename to cmd/extract_data.go index 2534beaa..7d596ff5 100644 --- a/extraction/postcode_compressor.go +++ b/cmd/extract_data.go @@ -1,4 +1,4 @@ -package extraction +package cmd import ( "archive/tar" @@ -7,6 +7,7 @@ import ( "log" "os" "path/filepath" + "postcode-polygons/internal" "strings" "github.com/dsnet/compress/bzip2" @@ -16,7 +17,7 @@ import ( "github.com/paulmach/orb/geojson" ) -func Extract(tarFile string) { +func ExtractData(tarFile string) { f, err := os.Open(tarFile) if err != nil { @@ -60,13 +61,18 @@ func Extract(tarFile string) { log.Fatalf("Error reading file %s: %v", header.Name, err) } - processed, err := reprocessFile(content) + fc, err := geojson.UnmarshalFeatureCollection(content) if err != nil { - log.Fatalf("Error processing file %s: %v", header.Name, err) + log.Fatalf("Error unmarshalling GeoJSON: %v", err) } - outputFile := "./data/postcodes/" + filename + ".bz2" - newSize, err := compressFile(outputFile, processed) + err = reprocessFeatureCollection(fc) + if err != nil { + log.Fatalf("Error reprocessing feature collection for file %s: %v", header.Name, err) + } + + outputFile := fmt.Sprintf("./data/postcodes/%s.bz2", filename) + newSize, err := internal.CompressFeatureCollection(outputFile, fc) if err != nil { log.Fatalf("Error compressing file %s: %v", outputFile, err) } @@ -83,45 +89,12 @@ func Extract(tarFile string) { } } -func compressFile(filename string, content []byte) (int, error) { - f, err := os.Create(filename) - if err != nil { - return 0, fmt.Errorf("error creating output file: %w", err) - } - defer func() { - if err := f.Close(); err != nil { - log.Printf("Error closing file %s: %v", filename, err) - } - }() - - w, err := bzip2.NewWriter(f, &bzip2.WriterConfig{Level: bzip2.BestCompression}) - if err != nil { - return 0, fmt.Errorf("error creating bzip2 writer: %w", err) - } - - _, err = w.Write(content) - if err != nil { - return 0, fmt.Errorf("error writing bzip2 file: %w", err) - } - - err = w.Close() // Ensure to close the writer to flush the data, so that the output offset is correct - if err != nil { - return 0, fmt.Errorf("error closing bzip2 writer: %w", err) - } - - return int(w.OutputOffset), nil -} - -func reprocessFile(content []byte) ([]byte, error) { - fc, err := geojson.UnmarshalFeatureCollection(content) - if err != nil { - return nil, fmt.Errorf("error unmarshalling GeoJSON: %w", err) - } +func reprocessFeatureCollection(fc *geojson.FeatureCollection) error { for _, feature := range fc.Features { postcode, ok := feature.Properties["postcodes"].(string) if !ok { - return nil, fmt.Errorf("missing or invalid 'postcodes' property in feature %s", feature.ID) + return fmt.Errorf("missing or invalid 'postcodes' property in feature %s", feature.ID) } truncateCoordinates(feature) @@ -130,7 +103,7 @@ func reprocessFile(content []byte) ([]byte, error) { feature.Properties["postcode"] = postcode } - return fc.MarshalJSON() + return nil } func truncateCoordinates(feature *geojson.Feature) { diff --git a/internal/file_operations.go b/internal/file_operations.go new file mode 100644 index 00000000..a8e64562 --- /dev/null +++ b/internal/file_operations.go @@ -0,0 +1,68 @@ +package internal + +import ( + "fmt" + "log" + "os" + + "github.com/dsnet/compress/bzip2" + jsoniter "github.com/json-iterator/go" + "github.com/paulmach/orb/geojson" +) + +var c = jsoniter.Config{EscapeHTML: true, SortMapKeys: true, MarshalFloatWith6Digits: true}.Froze() + +func CompressFeatureCollection(bz2filename string, fc *geojson.FeatureCollection) (int, error) { + f, err := os.Create(bz2filename) + if err != nil { + return 0, fmt.Errorf("error creating output file: %w", err) + } + defer func() { + if err := f.Close(); err != nil { + log.Printf("Error closing file %s: %v", bz2filename, err) + } + }() + + w, err := bzip2.NewWriter(f, &bzip2.WriterConfig{Level: bzip2.BestCompression}) + if err != nil { + return 0, fmt.Errorf("error creating bzip2 writer: %w", err) + } + + geojson.CustomJSONMarshaler = c + geojson.CustomJSONUnmarshaler = c + if err := c.NewEncoder(w).Encode(fc); err != nil { + return 0, fmt.Errorf("error writing bzip2 file: %w", err) + } + + err = w.Close() // Ensure to close the writer to flush the data, so that the output offset is correct + if err != nil { + return 0, fmt.Errorf("error closing bzip2 writer: %w", err) + } + + return int(w.OutputOffset), nil +} + +func DecompressFeatureCollection(bz2filename string) (*geojson.FeatureCollection, error) { + + file, err := os.Open(bz2filename) + if err != nil { + return nil, err + } + defer func() { + if err := file.Close(); err != nil { + log.Printf("Error closing file %s: %v", bz2filename, err) + } + }() + + r, err := bzip2.NewReader(file, &bzip2.ReaderConfig{}) + if err != nil { + return nil, fmt.Errorf("error creating bzip2 reader: %w", err) + } + geojson.CustomJSONMarshaler = c + geojson.CustomJSONUnmarshaler = c + fc := geojson.NewFeatureCollection() + if err := c.NewDecoder(r).Decode(fc); err != nil { + return nil, err + } + return fc, nil +} diff --git a/main.go b/main.go index bb589bf6..ef02fdd8 100644 --- a/main.go +++ b/main.go @@ -1,91 +1,48 @@ package main import ( - "fmt" "log" - "os" - "postcode-polygons/routes" - spatialindex "postcode-polygons/spatial-index" + "postcode-polygons/cmd" - "github.com/Depado/ginprom" - "github.com/aurowora/compress" - "github.com/gin-contrib/cors" - "github.com/gin-contrib/pprof" - "github.com/gin-gonic/gin" "github.com/spf13/cobra" - healthcheck "github.com/tavsec/gin-healthcheck" - "github.com/tavsec/gin-healthcheck/checks" - hc_config "github.com/tavsec/gin-healthcheck/config" - cachecontrol "go.eigsys.de/gin-cachecontrol/v2" ) func main() { var err error - var zipFile string + var polygonTarBz2File string + var codePointZipFile string var port int var debug bool rootCmd := &cobra.Command{ - Use: "http", - Short: "Postcode Polygons API server", - Run: func(cmd *cobra.Command, args []string) { - server(zipFile, port, debug) - }, + Use: "postcode-polygons", + Long: `HTTP server & data extraction`, } - rootCmd.Flags().StringVar(&zipFile, "codepoint", "./data/codepo_gb.zip", "Path to CodePoint Open zip file") - rootCmd.Flags().IntVar(&port, "port", 8080, "Port to run HTTP server on") - rootCmd.Flags().BoolVar(&debug, "debug", false, "Enable debugging (pprof) - WARING: do not enable in production") - - if err = rootCmd.Execute(); err != nil { - log.Fatalf("failed to execute root command: %v", err) - } -} - -func server(zipFile string, port int, debug bool) { - if _, err := os.Stat(zipFile); os.IsNotExist(err) { - log.Fatalf("CodePoint zip file does not exist: %s", zipFile) + apiServerCmd := &cobra.Command{ + Use: "api-server [--codepoint ] [--port ] [--debug]", + Short: "Start HTTP API server", + Run: func(_ *cobra.Command, _ []string) { + cmd.ApiServer(codePointZipFile, port, debug) + }, } - - log.Printf("Loading CodePoint data from: %s", zipFile) - spatialIndex, err := spatialindex.NewCodePointSpatialIndex(zipFile) - if err != nil { - log.Fatalf("failed to create spatial index: %v", err) + apiServerCmd.Flags().StringVar(&codePointZipFile, "codepoint", "./data/codepo_gb.zip", "Path to CodePoint Open zip file") + apiServerCmd.Flags().IntVar(&port, "port", 8080, "Port to run HTTP server on") + apiServerCmd.Flags().BoolVar(&debug, "debug", false, "Enable debugging (pprof) - WARING: do not enable in production") + + extractDataCmd := &cobra.Command{ + Use: "extract-data [--polygon ]", + Short: "Extract NSUL polygons", + Run: func(_ *cobra.Command, _ []string) { + cmd.ExtractData(polygonTarBz2File) + }, } - log.Printf("CodePoint spatial index created with %d entries", spatialIndex.Len()) - - r := gin.New() - - prometheus := ginprom.New( - ginprom.Engine(r), - ginprom.Path("/metrics"), - ginprom.Ignore("/healthz"), - ) + extractDataCmd.Flags().StringVar(&polygonTarBz2File, "polygon", "./data/gb-postcodes-v5.tar.bz2", "Path to NSUL polygons tar.gz file") - r.Use( - gin.Recovery(), - gin.LoggerWithWriter(gin.DefaultWriter, "/healthz", "/metrics"), - prometheus.Instrument(), - compress.Compress(), - cachecontrol.New(cachecontrol.CacheAssetsForeverPreset), - cors.Default(), - ) - - if debug { - log.Println("WARNING: pprof endpoints are enabled and exposed. Do not run with this flag in production.") - pprof.Register(r) - } + rootCmd.AddCommand(apiServerCmd) + rootCmd.AddCommand(extractDataCmd) - err = healthcheck.New(r, hc_config.DefaultConfig(), []checks.Check{}) - if err != nil { - log.Fatalf("failed to initialize healthcheck: %v", err) + if err = rootCmd.Execute(); err != nil { + log.Fatalf("failed to execute root command: %v", err) } - - r.GET("/v1/postcode/codepoints", routes.CodePointSearch(spatialIndex)) - r.GET("/v1/postcode/polygons", routes.PolygonSearch(spatialIndex)) - - addr := fmt.Sprintf(":%d", port) - log.Printf("Starting HTTP API Server on port %d...", port) - err = r.Run(addr) - log.Fatalf("HTTP API Server failed to start on port %d: %v", port, err) } diff --git a/routes/search.go b/routes/search.go index a9997dda..7cd8fb49 100644 --- a/routes/search.go +++ b/routes/search.go @@ -5,15 +5,14 @@ import ( "log" "math" "net/http" + "postcode-polygons/internal" spatialindex "postcode-polygons/spatial-index" "strconv" "strings" "os" - "github.com/dsnet/compress/bzip2" "github.com/gin-gonic/gin" - jsoniter "github.com/json-iterator/go" "github.com/paulmach/orb/geojson" ) @@ -82,7 +81,8 @@ func PolygonSearch(spatialIndex *spatialindex.SpatialIndex) func(c *gin.Context) fc.Features = make([]*geojson.Feature, 0, len(requestedPostcodes)) for district := range districts { - featureCollection, err := loadFromFile(fmt.Sprintf("./data/postcodes/%s.geojson.bz2", district)) + filename := fmt.Sprintf("./data/postcodes/%s.geojson.bz2", district) + featureCollection, err := internal.DecompressFeatureCollection(filename) if err != nil && os.IsNotExist(err) { log.Printf("polygon file for district %s does not exist, skipping", district) continue @@ -104,34 +104,6 @@ func PolygonSearch(spatialIndex *spatialindex.SpatialIndex) func(c *gin.Context) } } -func loadFromFile(bz2filename string) (*geojson.FeatureCollection, error) { - - file, err := os.Open(bz2filename) - if err != nil { - return nil, err - } - defer func() { - if err := file.Close(); err != nil { - log.Printf("Error closing file %s: %v", bz2filename, err) - } - }() - - bz2Reader, err := bzip2.NewReader(file, &bzip2.ReaderConfig{}) - if err != nil { - return nil, fmt.Errorf("error creating bzip2 reader: %w", err) - } - c := jsoniter.Config{EscapeHTML: true, SortMapKeys: false, MarshalFloatWith6Digits: true}.Froze() - geojson.CustomJSONMarshaler = c - geojson.CustomJSONUnmarshaler = c - decoder := c.NewDecoder(bz2Reader) - - fc := geojson.NewFeatureCollection() - if err := decoder.Decode(fc); err != nil { - return nil, err - } - return fc, nil -} - func parseBBox(bboxStr string) ([]uint32, error) { bboxParts := strings.Split(bboxStr, ",") if len(bboxParts) != 4 { From 9802191bfcf5fbc808b0e2cac71ca4505d284fed Mon Sep 17 00:00:00 2001 From: Richard Hull Date: Thu, 26 Jun 2025 00:47:51 +0100 Subject: [PATCH 2/3] Address PR comments --- internal/file_operations.go | 10 ++++++---- main.go | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/internal/file_operations.go b/internal/file_operations.go index a8e64562..5514500d 100644 --- a/internal/file_operations.go +++ b/internal/file_operations.go @@ -12,6 +12,11 @@ import ( var c = jsoniter.Config{EscapeHTML: true, SortMapKeys: true, MarshalFloatWith6Digits: true}.Froze() +func init() { + geojson.CustomJSONMarshaler = c + geojson.CustomJSONUnmarshaler = c +} + func CompressFeatureCollection(bz2filename string, fc *geojson.FeatureCollection) (int, error) { f, err := os.Create(bz2filename) if err != nil { @@ -28,8 +33,6 @@ func CompressFeatureCollection(bz2filename string, fc *geojson.FeatureCollection return 0, fmt.Errorf("error creating bzip2 writer: %w", err) } - geojson.CustomJSONMarshaler = c - geojson.CustomJSONUnmarshaler = c if err := c.NewEncoder(w).Encode(fc); err != nil { return 0, fmt.Errorf("error writing bzip2 file: %w", err) } @@ -58,8 +61,7 @@ func DecompressFeatureCollection(bz2filename string) (*geojson.FeatureCollection if err != nil { return nil, fmt.Errorf("error creating bzip2 reader: %w", err) } - geojson.CustomJSONMarshaler = c - geojson.CustomJSONUnmarshaler = c + fc := geojson.NewFeatureCollection() if err := c.NewDecoder(r).Decode(fc); err != nil { return nil, err diff --git a/main.go b/main.go index ef02fdd8..f6870015 100644 --- a/main.go +++ b/main.go @@ -37,7 +37,7 @@ func main() { cmd.ExtractData(polygonTarBz2File) }, } - extractDataCmd.Flags().StringVar(&polygonTarBz2File, "polygon", "./data/gb-postcodes-v5.tar.bz2", "Path to NSUL polygons tar.gz file") + extractDataCmd.Flags().StringVar(&polygonTarBz2File, "polygon", "./data/gb-postcodes-v5.tar.bz2", "Path to NSUL polygons tar.bz2 file") rootCmd.AddCommand(apiServerCmd) rootCmd.AddCommand(extractDataCmd) From 31d7bc8fb22b96935c356acbbc3b9f999b9aaa65 Mon Sep 17 00:00:00 2001 From: Richard Hull Date: Thu, 26 Jun 2025 00:51:26 +0100 Subject: [PATCH 3/3] Rename variable --- cmd/extract_data.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/extract_data.go b/cmd/extract_data.go index 7d596ff5..f0499804 100644 --- a/cmd/extract_data.go +++ b/cmd/extract_data.go @@ -17,9 +17,9 @@ import ( "github.com/paulmach/orb/geojson" ) -func ExtractData(tarFile string) { +func ExtractData(tarBz2File string) { - f, err := os.Open(tarFile) + f, err := os.Open(tarBz2File) if err != nil { log.Fatalf("Error opening file: %v", err) }