Polygon search

Dockerfile

@@ -15,7 +15,7 @@ COPY . .
 
 ENV GOOS=linux
 
-RUN go build -ldflags="-w -s" -o postcode-polygons .
+RUN go build -tags=jsoniter -ldflags="-w -s" -o postcode-polygons .
 
 FROM alpine:latest AS runtime
 ENV GIN_MODE=release

go.mod

@@ -6,15 +6,17 @@ require (
 	github.com/Depado/ginprom v1.8.1
 	github.com/aurowora/compress v0.0.0-20230724224640-6512772d482f
 	github.com/gin-contrib/cors v1.7.6
+	github.com/gin-contrib/pprof v1.5.3
 	github.com/gin-gonic/gin v1.10.1
+	github.com/json-iterator/go v1.1.12
 	github.com/paulmach/orb v0.11.1
 	github.com/spf13/cobra v1.9.1
 	github.com/tavsec/gin-healthcheck v1.7.8
 	go.eigsys.de/gin-cachecontrol/v2 v2.3.0
 )
 
 require (
-	github.com/andybalholm/brotli v1.1.1 // indirect
+	github.com/andybalholm/brotli v1.2.0 // indirect
 	github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bytedance/sonic v1.13.3 // indirect
@@ -23,7 +25,6 @@ require (
 	github.com/cloudwego/base64x v0.1.5 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/gabriel-vasile/mimetype v1.4.9 // indirect
-	github.com/gin-contrib/pprof v1.5.3 // indirect
 	github.com/gin-contrib/sse v1.1.0 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
@@ -34,7 +35,6 @@ require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/influxdata/influxdb-client-go/v2 v2.14.0 // indirect
 	github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf // indirect
-	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.18.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.10 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
@@ -46,10 +46,10 @@ require (
 	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
 	github.com/prometheus/client_golang v1.22.0 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
-	github.com/prometheus/common v0.64.0 // indirect
+	github.com/prometheus/common v0.65.0 // indirect
 	github.com/prometheus/procfs v0.16.1 // indirect
 	github.com/rabbitmq/amqp091-go v1.10.0 // indirect
-	github.com/redis/go-redis/v9 v9.10.0 // indirect
+	github.com/redis/go-redis/v9 v9.11.0 // indirect
 	github.com/spf13/pflag v1.0.6 // indirect
 	github.com/tidwall/geoindex v1.7.0 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect

go.sum

@@ -12,6 +12,8 @@ github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMz
 github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
 github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
+github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
+github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
 github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ=
 github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk=
 github.com/appleboy/gofight/v2 v2.1.2 h1:VOy3jow4vIK8BRQJoC/I9muxyYlJ2yb9ht2hZoS3rf4=
@@ -214,12 +216,16 @@ github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNw
 github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
 github.com/prometheus/common v0.64.0 h1:pdZeA+g617P7oGv1CzdTzyeShxAGrTBsolKNOLQPGO4=
 github.com/prometheus/common v0.64.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8=
+github.com/prometheus/common v0.65.0 h1:QDwzd+G1twt//Kwj/Ww6E9FQq1iVMmODnILtW1t2VzE=
+github.com/prometheus/common v0.65.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8=
 github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
 github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
 github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
 github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
 github.com/redis/go-redis/v9 v9.10.0 h1:FxwK3eV8p/CQa0Ch276C7u2d0eNC9kCmAYQ7mCXCzVs=
 github.com/redis/go-redis/v9 v9.10.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw=
+github.com/redis/go-redis/v9 v9.11.0 h1:E3S08Gl/nJNn5vkxd2i78wZxWAPNZgUNTp8WIJUAiIs=
+github.com/redis/go-redis/v9 v9.11.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=

main.go

@@ -83,7 +83,8 @@ func server(zipFile string, port int, debug bool) {
 		log.Fatalf("failed to initialize healthcheck: %v", err)
 	}
 
-	r.GET("/v1/postcode/codepoint", routes.CodePointSearch(spatialIndex))
+	r.GET("/v1/postcode/codepoints", routes.CodePointSearch(spatialIndex))
+	r.GET("/v1/postcode/polygons", routes.PolygonSearch(spatialIndex))
 
 	addr := fmt.Sprintf(":%d", port)
 	log.Printf("Starting HTTP API Server on port %d...", port)

routes/search.go

@@ -9,7 +9,12 @@ import (
 	"strconv"
 	"strings"
 
+	"os"
+
+	"github.com/dsnet/compress/bzip2"
 	"github.com/gin-gonic/gin"
+	jsoniter "github.com/json-iterator/go"
+	"github.com/paulmach/orb/geojson"
 )
 
 type SearchResponse struct {
@@ -41,6 +46,92 @@ func CodePointSearch(spatialIndex *spatialindex.SpatialIndex) func(c *gin.Contex
 	}
 }
 
+func PolygonSearch(spatialIndex *spatialindex.SpatialIndex) func(c *gin.Context) {
+	return func(c *gin.Context) {
+		bbox, err := parseBBox(c.Query("bbox"))
+		if err != nil {
+			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+			return
+		}
+
+		requestedPostcodes := make(map[string]struct{}, 100)
+		districts := make(map[string]struct{}, 20)
+
+		err = spatialIndex.SearchIter(bbox, func(min, max [2]uint32, postcode string) bool {
+			district := strings.Split(postcode, " ")[0] // Take the first part of the postcode
+			districts[district] = struct{}{}
+			requestedPostcodes[postcode] = struct{}{}
+			return true
+		})
+		if err != nil {
+			log.Printf("error while fetching postcode data: %v", err)
+			c.JSON(http.StatusInternalServerError, gin.H{"error": "An internal server error occurred"})
+			return
+		}
+
+		isRequestedPostcode := func(feature *geojson.Feature) bool {
+			postcode, ok := feature.Properties["postcode"].(string)
+			if !ok {
+				return false
+			}
+			_, exists := requestedPostcodes[postcode]
+			return exists
+		}
+
+		fc := geojson.NewFeatureCollection()
+		fc.Features = make([]*geojson.Feature, 0, len(requestedPostcodes))
+
+		for district := range districts {
+			featureCollection, err := loadFromFile(fmt.Sprintf("./data/postcodes/%s.geojson.bz2", district))
+			if err != nil && os.IsNotExist(err) {
+				log.Printf("polygon file for district %s does not exist, skipping", district)
+				continue
+			}
+			if err != nil {
+				log.Printf("error loading polygon for district %s: %v", district, err)
+				c.JSON(http.StatusInternalServerError, gin.H{"error": "An internal server error occurred"})
+				return
+			}
+			for _, feature := range featureCollection.Features {
+				if isRequestedPostcode(feature) {
+					fc.Append(feature)
+				}
+			}
+		}
+
+		c.Header("Content-Type", "application/geo+json")
+		c.JSON(http.StatusOK, &fc)
+	}
+}
+
+func loadFromFile(bz2filename string) (*geojson.FeatureCollection, error) {
+
+	file, err := os.Open(bz2filename)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err := file.Close(); err != nil {
+			log.Printf("Error closing file %s: %v", bz2filename, err)
+		}
+	}()
+
+	bz2Reader, err := bzip2.NewReader(file, &bzip2.ReaderConfig{})
+	if err != nil {
+		return nil, fmt.Errorf("error creating bzip2 reader: %w", err)
+	}
+	c := jsoniter.Config{EscapeHTML: true, SortMapKeys: false, MarshalFloatWith6Digits: true}.Froze()
+	geojson.CustomJSONMarshaler = c
+	geojson.CustomJSONUnmarshaler = c
+	decoder := c.NewDecoder(bz2Reader)
+
+	fc := geojson.NewFeatureCollection()
+	if err := decoder.Decode(fc); err != nil {
+		return nil, err
+	}
+	return fc, nil
+}
+
 func parseBBox(bboxStr string) ([]uint32, error) {
 	bboxParts := strings.Split(bboxStr, ",")
 	if len(bboxParts) != 4 {

spatial-index/codepoint.go

@@ -21,15 +21,9 @@ type SpatialIndex struct {
 }
 
 func (idx *SpatialIndex) Search(bounds []uint32) (*[]CodePoint, error) {
-	if len(bounds) != 4 {
-		return nil, fmt.Errorf("search bounds must contain exactly 4 values: min_easting, min_northing, max_easting, max_northing")
-	}
-
-	sw := [2]uint32{bounds[0], bounds[1]} // South-West corner
-	ne := [2]uint32{bounds[2], bounds[3]} // North-East corner
 
 	results := make([]CodePoint, 0, 100)
-	idx.tree.Search(sw, ne, func(min, max [2]uint32, data string) bool {
+	err := idx.SearchIter(bounds, func(min, max [2]uint32, data string) bool {
 		results = append(results, CodePoint{
 			PostCode: data,
 			Easting:  min[0],
@@ -38,9 +32,25 @@ func (idx *SpatialIndex) Search(bounds []uint32) (*[]CodePoint, error) {
 		return true
 	})
 
+	if err != nil {
+		return nil, fmt.Errorf("error during spatial search: %w", err)
+	}
+
 	return &results, nil
 }
 
+func (idx *SpatialIndex) SearchIter(bounds []uint32, iter func(min, max [2]uint32, data string) bool) error {
+	if len(bounds) != 4 {
+		return fmt.Errorf("search bounds must contain exactly 4 values: min_easting, min_northing, max_easting, max_northing")
+	}
+
+	sw := [2]uint32{bounds[0], bounds[1]} // South-West corner
+	ne := [2]uint32{bounds[2], bounds[3]} // North-East corner
+
+	idx.tree.Search(sw, ne, iter)
+	return nil
+}
+
 func NewCodePointSpatialIndex(zipFile string) (*SpatialIndex, error) {
 	idx := SpatialIndex{
 		tree: &rtree.RTreeGN[uint32, string]{},