Code reorg

.vscode/launch.json

@@ -1,16 +1,24 @@
 {
-  // Use IntelliSense to learn about possible attributes.
-  // Hover to view descriptions of existing attributes.
-  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-  "version": "0.2.0",
-  "configurations": [
-    {
-      "name": "Launch HTTP server",
-      "type": "go",
-      "request": "launch",
-      "mode": "auto",
-      "program": "main.go",
-      "args": []
-    }
-  ]
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Launch HTTP server",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "main.go",
+            "args": ["api-server"]
+        },
+        {
+            "name": "Launch data extraction",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "main.go",
+            "args": ["extract-data"]
+        }
+    ]
 }

README.md

@@ -6,5 +6,14 @@ First download (or otherwise generate) the **gb-postcodes-v5.tar.bz2** data file
 
 ```console
 $ curl https://postcodes-mapit-static.s3.eu-west-2.amazonaws.com/data/gb-postcodes-v5.tar.bz2 -O data/gb-postcodes-v5.tar.bz2
-$ go run main.go
+$ go run main.go extract-data
+```
+
+This will regenerate the (checked-in) data files under `./data/postcodes`. There is typically no need to run this unless
+the NSUL datafile got updated
+
+## Starting the API server
+
+```console
+$ go run main.go api-server
 ```

cmd/api_server.go

@@ -0,0 +1,68 @@
+package cmd
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"postcode-polygons/routes"
+	spatialindex "postcode-polygons/spatial-index"
+
+	"github.com/Depado/ginprom"
+	"github.com/aurowora/compress"
+	"github.com/gin-contrib/cors"
+	"github.com/gin-contrib/pprof"
+	"github.com/gin-gonic/gin"
+	"github.com/tavsec/gin-healthcheck/checks"
+	cachecontrol "go.eigsys.de/gin-cachecontrol/v2"
+
+	healthcheck "github.com/tavsec/gin-healthcheck"
+	hc_config "github.com/tavsec/gin-healthcheck/config"
+)
+
+func ApiServer(zipFile string, port int, debug bool) {
+	if _, err := os.Stat(zipFile); os.IsNotExist(err) {
+		log.Fatalf("CodePoint zip file does not exist: %s", zipFile)
+	}
+
+	log.Printf("Loading CodePoint data from: %s", zipFile)
+	spatialIndex, err := spatialindex.NewCodePointSpatialIndex(zipFile)
+	if err != nil {
+		log.Fatalf("failed to create spatial index: %v", err)
+	}
+	log.Printf("CodePoint spatial index created with %d entries", spatialIndex.Len())
+
+	r := gin.New()
+
+	prometheus := ginprom.New(
+		ginprom.Engine(r),
+		ginprom.Path("/metrics"),
+		ginprom.Ignore("/healthz"),
+	)
+
+	r.Use(
+		gin.Recovery(),
+		gin.LoggerWithWriter(gin.DefaultWriter, "/healthz", "/metrics"),
+		prometheus.Instrument(),
+		compress.Compress(),
+		cachecontrol.New(cachecontrol.CacheAssetsForeverPreset),
+		cors.Default(),
+	)
+
+	if debug {
+		log.Println("WARNING: pprof endpoints are enabled and exposed. Do not run with this flag in production.")
+		pprof.Register(r)
+	}
+
+	err = healthcheck.New(r, hc_config.DefaultConfig(), []checks.Check{})
+	if err != nil {
+		log.Fatalf("failed to initialize healthcheck: %v", err)
+	}
+
+	r.GET("/v1/postcode/codepoints", routes.CodePointSearch(spatialIndex))
+	r.GET("/v1/postcode/polygons", routes.PolygonSearch(spatialIndex))
+
+	addr := fmt.Sprintf(":%d", port)
+	log.Printf("Starting HTTP API Server on port %d...", port)
+	err = r.Run(addr)
+	log.Fatalf("HTTP API Server failed to start on port %d: %v", port, err)
+}

extraction/postcode_compressor.go → cmd/extract_data.go

@@ -1,4 +1,4 @@
-package extraction
+package cmd
 
 import (
 	"archive/tar"
@@ -7,6 +7,7 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"postcode-polygons/internal"
 	"strings"
 
 	"github.com/dsnet/compress/bzip2"
@@ -16,9 +17,9 @@ import (
 	"github.com/paulmach/orb/geojson"
 )
 
-func Extract(tarFile string) {
+func ExtractData(tarBz2File string) {
 
-	f, err := os.Open(tarFile)
+	f, err := os.Open(tarBz2File)
 	if err != nil {
 		log.Fatalf("Error opening file: %v", err)
 	}
@@ -60,13 +61,18 @@ func Extract(tarFile string) {
 				log.Fatalf("Error reading file %s: %v", header.Name, err)
 			}
 
-			processed, err := reprocessFile(content)
+			fc, err := geojson.UnmarshalFeatureCollection(content)
 			if err != nil {
-				log.Fatalf("Error processing file %s: %v", header.Name, err)
+				log.Fatalf("Error unmarshalling GeoJSON: %v", err)
 			}
 
-			outputFile := "./data/postcodes/" + filename + ".bz2"
-			newSize, err := compressFile(outputFile, processed)
+			err = reprocessFeatureCollection(fc)
+			if err != nil {
+				log.Fatalf("Error reprocessing feature collection for file %s: %v", header.Name, err)
+			}
+
+			outputFile := fmt.Sprintf("./data/postcodes/%s.bz2", filename)
+			newSize, err := internal.CompressFeatureCollection(outputFile, fc)
 			if err != nil {
 				log.Fatalf("Error compressing file %s: %v", outputFile, err)
 			}
@@ -83,45 +89,12 @@ func Extract(tarFile string) {
 	}
 }
 
-func compressFile(filename string, content []byte) (int, error) {
-	f, err := os.Create(filename)
-	if err != nil {
-		return 0, fmt.Errorf("error creating output file: %w", err)
-	}
-	defer func() {
-		if err := f.Close(); err != nil {
-			log.Printf("Error closing file %s: %v", filename, err)
-		}
-	}()
-
-	w, err := bzip2.NewWriter(f, &bzip2.WriterConfig{Level: bzip2.BestCompression})
-	if err != nil {
-		return 0, fmt.Errorf("error creating bzip2 writer: %w", err)
-	}
-
-	_, err = w.Write(content)
-	if err != nil {
-		return 0, fmt.Errorf("error writing bzip2 file: %w", err)
-	}
-
-	err = w.Close() // Ensure to close the writer to flush the data, so that the output offset is correct
-	if err != nil {
-		return 0, fmt.Errorf("error closing bzip2 writer: %w", err)
-	}
-
-	return int(w.OutputOffset), nil
-}
-
-func reprocessFile(content []byte) ([]byte, error) {
-	fc, err := geojson.UnmarshalFeatureCollection(content)
-	if err != nil {
-		return nil, fmt.Errorf("error unmarshalling GeoJSON: %w", err)
-	}
+func reprocessFeatureCollection(fc *geojson.FeatureCollection) error {
 
 	for _, feature := range fc.Features {
 		postcode, ok := feature.Properties["postcodes"].(string)
 		if !ok {
-			return nil, fmt.Errorf("missing or invalid 'postcodes' property in feature %s", feature.ID)
+			return fmt.Errorf("missing or invalid 'postcodes' property in feature %s", feature.ID)
 		}
 
 		truncateCoordinates(feature)
@@ -130,7 +103,7 @@ func reprocessFile(content []byte) ([]byte, error) {
 		feature.Properties["postcode"] = postcode
 	}
 
-	return fc.MarshalJSON()
+	return nil
 }
 
 func truncateCoordinates(feature *geojson.Feature) {

internal/file_operations.go

@@ -0,0 +1,70 @@
+package internal
+
+import (
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/dsnet/compress/bzip2"
+	jsoniter "github.com/json-iterator/go"
+	"github.com/paulmach/orb/geojson"
+)
+
+var c = jsoniter.Config{EscapeHTML: true, SortMapKeys: true, MarshalFloatWith6Digits: true}.Froze()
+
+func init() {
+	geojson.CustomJSONMarshaler = c
+	geojson.CustomJSONUnmarshaler = c
+}
+
+func CompressFeatureCollection(bz2filename string, fc *geojson.FeatureCollection) (int, error) {
+	f, err := os.Create(bz2filename)
+	if err != nil {
+		return 0, fmt.Errorf("error creating output file: %w", err)
+	}
+	defer func() {
+		if err := f.Close(); err != nil {
+			log.Printf("Error closing file %s: %v", bz2filename, err)
+		}
+	}()
+
+	w, err := bzip2.NewWriter(f, &bzip2.WriterConfig{Level: bzip2.BestCompression})
+	if err != nil {
+		return 0, fmt.Errorf("error creating bzip2 writer: %w", err)
+	}
+
+	if err := c.NewEncoder(w).Encode(fc); err != nil {
+		return 0, fmt.Errorf("error writing bzip2 file: %w", err)
+	}
+
+	err = w.Close() // Ensure to close the writer to flush the data, so that the output offset is correct
+	if err != nil {
+		return 0, fmt.Errorf("error closing bzip2 writer: %w", err)
+	}
+
+	return int(w.OutputOffset), nil
+}
+
+func DecompressFeatureCollection(bz2filename string) (*geojson.FeatureCollection, error) {
+
+	file, err := os.Open(bz2filename)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err := file.Close(); err != nil {
+			log.Printf("Error closing file %s: %v", bz2filename, err)
+		}
+	}()
+
+	r, err := bzip2.NewReader(file, &bzip2.ReaderConfig{})
+	if err != nil {
+		return nil, fmt.Errorf("error creating bzip2 reader: %w", err)
+	}
+
+	fc := geojson.NewFeatureCollection()
+	if err := c.NewDecoder(r).Decode(fc); err != nil {
+		return nil, err
+	}
+	return fc, nil
+}