-
Notifications
You must be signed in to change notification settings - Fork 0
Add spatial index on codepoints #3
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
3 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,6 +2,7 @@ | |
| .vscode | ||
| data/ | ||
| !data/postcodes | ||
| !data/codepo_gb.zip | ||
| LICENSE.md | ||
| README.md | ||
| ATTRIBUTION.md | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Binary file not shown.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,9 +1,84 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "postcode-polygons/extraction" | ||
| "fmt" | ||
| "log" | ||
| "os" | ||
| "postcode-polygons/routes" | ||
| spatialindex "postcode-polygons/spatial-index" | ||
|
|
||
| "github.com/Depado/ginprom" | ||
| "github.com/aurowora/compress" | ||
| "github.com/gin-contrib/cors" | ||
| "github.com/gin-gonic/gin" | ||
| "github.com/spf13/cobra" | ||
| healthcheck "github.com/tavsec/gin-healthcheck" | ||
| "github.com/tavsec/gin-healthcheck/checks" | ||
| hc_config "github.com/tavsec/gin-healthcheck/config" | ||
| cachecontrol "go.eigsys.de/gin-cachecontrol/v2" | ||
| ) | ||
|
|
||
| func main() { | ||
| extraction.Extract("./data/gb-postcodes-v5.tar.bz2") | ||
| var err error | ||
| var zipFile string | ||
| var port int | ||
|
|
||
| rootCmd := &cobra.Command{ | ||
| Use: "http", | ||
| Short: "Postcode Polygons API server", | ||
| Run: func(cmd *cobra.Command, args []string) { | ||
| server(zipFile, port) | ||
| }, | ||
| } | ||
|
|
||
| rootCmd.Flags().StringVar(&zipFile, "codepoint", "./data/codepo_gb.zip", "Path to CodePoint Open zip file") | ||
| rootCmd.Flags().IntVar(&port, "port", 8080, "Port to run HTTP server on") | ||
|
|
||
| if err = rootCmd.Execute(); err != nil { | ||
| log.Fatalf("failed to execute root command: %v", err) | ||
| } | ||
| } | ||
|
|
||
| func server(zipFile string, port int) { | ||
| if _, err := os.Stat(zipFile); os.IsNotExist(err) { | ||
| log.Fatalf("CodePoint zip file does not exist: %s", zipFile) | ||
| } | ||
|
|
||
| log.Printf("Loading CodePoint data from: %s", zipFile) | ||
| spatialIndex, err := spatialindex.NewCodePointSpatialIndex(zipFile) | ||
| if err != nil { | ||
| log.Fatalf("failed to create spatial index: %v", err) | ||
| } | ||
| log.Printf("CodePoint spatial index created with %d entries", spatialIndex.Len()) | ||
|
|
||
| r := gin.New() | ||
|
|
||
| prometheus := ginprom.New( | ||
| ginprom.Engine(r), | ||
| ginprom.Namespace("postcode_polygons"), | ||
| ginprom.Subsystem("api"), | ||
| ginprom.Path("/metrics"), | ||
| ginprom.Ignore("/healthz"), | ||
| ) | ||
|
|
||
| r.Use( | ||
| gin.Recovery(), | ||
| gin.LoggerWithWriter(gin.DefaultWriter, "/healthz", "/metrics"), | ||
| prometheus.Instrument(), | ||
| compress.Compress(), | ||
| cachecontrol.New(cachecontrol.CacheAssetsForeverPreset), | ||
| cors.Default(), | ||
| ) | ||
|
|
||
| err = healthcheck.New(r, hc_config.DefaultConfig(), []checks.Check{}) | ||
| if err != nil { | ||
| log.Fatalf("failed to initialize healthcheck: %v", err) | ||
| } | ||
|
|
||
| r.GET("/v1/postcode/search", routes.Search(spatialIndex)) | ||
|
|
||
| addr := fmt.Sprintf(":%d", port) | ||
| log.Printf("Starting HTTP API Server on port %d...", port) | ||
| err = r.Run(addr) | ||
| log.Fatalf("HTTP API Server failed to start on port %d: %v", port, err) | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| package routes | ||
|
|
||
| var ATTRIBUTION = []string{ | ||
| "CodePoint Open (UK Gov, OS Data Hub), https://osdatahub.os.uk/downloads/open/CodePointOpen", | ||
| "Mark Longair's Blog, https://longair.net/blog/2021/08/23/open-data-gb-postcode-unit-boundaries/", | ||
| "Derived content from: Office for National Statistics licensed under the Open Government Licence v.3.0", | ||
| "Derived content from: Royal Mail data © Royal Mail copyright and database right 2020", | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| package routes | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "log" | ||
| "math" | ||
| "net/http" | ||
| spatialindex "postcode-polygons/spatial-index" | ||
| "strconv" | ||
| "strings" | ||
|
|
||
| "github.com/gin-gonic/gin" | ||
| ) | ||
|
|
||
| type SearchResponse struct { | ||
| Results []spatialindex.CodePoint `json:"results"` | ||
| Attribution []string `json:"attribution"` | ||
| } | ||
|
|
||
| const MAX_BOUNDS = 5000 // Maximum bounds in meters (5 KM) | ||
|
|
||
| func Search(spatialIndex *spatialindex.SpatialIndex) func(c *gin.Context) { | ||
| return func(c *gin.Context) { | ||
| bbox, err := parseBBox(c.Query("bbox")) | ||
| if err != nil { | ||
| c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) | ||
| return | ||
| } | ||
|
|
||
| results, err := spatialIndex.Search(bbox) | ||
| if err != nil { | ||
| log.Printf("error while fetching postcode data: %v", err) | ||
| c.JSON(http.StatusInternalServerError, gin.H{"error": "An internal server error occurred"}) | ||
| return | ||
| } | ||
|
|
||
| c.JSON(http.StatusOK, SearchResponse{ | ||
| Results: *results, | ||
| Attribution: ATTRIBUTION, | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| func parseBBox(bboxStr string) ([]uint32, error) { | ||
| bboxParts := strings.Split(bboxStr, ",") | ||
| if len(bboxParts) != 4 { | ||
| return nil, fmt.Errorf("bbox must have 4 comma-separated values") | ||
| } | ||
|
|
||
| bbox := make([]uint32, 4) | ||
| for i, part := range bboxParts { | ||
| val, err := strconv.ParseUint(strings.TrimSpace(part), 10, 32) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("invalid bbox value '%s': not a valid float", part) | ||
| } | ||
| bbox[i] = uint32(val) | ||
| } | ||
|
|
||
| if bbox[0] > bbox[2] || bbox[1] > bbox[3] { | ||
| return nil, fmt.Errorf("invalid bbox: min values must be less than or equal to max values") | ||
| } | ||
|
|
||
| if math.Abs(float64(bbox[2]-bbox[0])) > MAX_BOUNDS || math.Abs(float64(bbox[3]-bbox[1])) > MAX_BOUNDS { | ||
| return nil, fmt.Errorf("bbox must define a valid area (no more than %d KM in either dimension)", MAX_BOUNDS/1000) | ||
| } | ||
|
rm-hull marked this conversation as resolved.
|
||
|
|
||
| return bbox, nil | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,125 @@ | ||
| package spatialindex | ||
|
|
||
| import ( | ||
| "archive/zip" | ||
| "fmt" | ||
| "log" | ||
| "strconv" | ||
| "strings" | ||
|
|
||
| "github.com/tidwall/rtree" | ||
| ) | ||
|
|
||
| type CodePoint struct { | ||
| PostCode string `json:"post_code"` | ||
| Easting uint32 `json:"easting"` | ||
| Northing uint32 `json:"northing"` | ||
| } | ||
|
|
||
| type SpatialIndex struct { | ||
| tree *rtree.RTreeGN[uint32, string] | ||
| } | ||
|
|
||
| func (idx *SpatialIndex) Search(bounds []uint32) (*[]CodePoint, error) { | ||
| if len(bounds) != 4 { | ||
| return nil, fmt.Errorf("search bounds must contain exactly 4 values: min_easting, min_northing, max_easting, max_northing") | ||
| } | ||
|
|
||
| sw := [2]uint32{bounds[0], bounds[1]} // South-West corner | ||
| ne := [2]uint32{bounds[2], bounds[3]} // North-East corner | ||
|
|
||
| results := make([]CodePoint, 0, 100) | ||
| idx.tree.Search(sw, ne, func(min, max [2]uint32, data string) bool { | ||
| results = append(results, CodePoint{ | ||
| PostCode: data, | ||
| Easting: min[0], | ||
| Northing: min[1], | ||
| }) | ||
| return true | ||
| }) | ||
|
|
||
| return &results, nil | ||
| } | ||
|
|
||
| func NewCodePointSpatialIndex(zipFile string) (*SpatialIndex, error) { | ||
| idx := SpatialIndex{ | ||
| tree: &rtree.RTreeGN[uint32, string]{}, | ||
| } | ||
|
|
||
| err := idx.importCodePoint(zipFile) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to load codepoint index from zip file: %w", err) | ||
| } | ||
|
|
||
| return &idx, nil | ||
| } | ||
|
|
||
| func (idx *SpatialIndex) Len() int { | ||
| return idx.tree.Len() | ||
| } | ||
|
|
||
| func (idx *SpatialIndex) importCodePoint(zipPath string) error { | ||
|
|
||
| r, err := zip.OpenReader(zipPath) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to open zip file: %w", err) | ||
| } | ||
| defer func() { | ||
| if err := r.Close(); err != nil { | ||
| log.Printf("error closing zip file: %v", err) | ||
| } | ||
| }() | ||
|
|
||
| for _, f := range r.File { | ||
| if f.FileInfo().IsDir() || !strings.HasPrefix(f.Name, "Data/CSV/") { | ||
| continue | ||
| } | ||
|
|
||
| if err := idx.processCSV(f); err != nil { | ||
| return fmt.Errorf("failed to process CSV data: %w", err) | ||
| } | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func (idx *SpatialIndex) processCSV(f *zip.File) error { | ||
| r, err := f.Open() | ||
| if err != nil { | ||
| return fmt.Errorf("failed to open embedded file %s in zip: %w", f.Name, err) | ||
| } | ||
| defer func() { | ||
| if err := r.Close(); err != nil { | ||
| log.Printf("error closing embedded zip file: %v", err) | ||
| } | ||
| }() | ||
|
|
||
| for result := range parseCSV(r, false, fromCodePointCSV) { | ||
|
|
||
| if result.Error != nil { | ||
| return fmt.Errorf("error parsing line %d: %w", result.LineNum, result.Error) | ||
| } | ||
|
|
||
| point := [2]uint32{uint32(result.Value.Easting), uint32(result.Value.Northing)} | ||
| idx.tree.Insert(point, point, result.Value.PostCode) | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func fromCodePointCSV(record []string, headers []string) (*CodePoint, error) { | ||
|
|
||
| easting, err := strconv.ParseUint(record[2], 10, 32) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("invalid easting value: %w", err) | ||
| } | ||
| northing, err := strconv.ParseUint(record[3], 10, 32) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("invalid northing value: %w", err) | ||
| } | ||
|
|
||
| return &CodePoint{ | ||
| PostCode: record[0], | ||
| Easting: uint32(easting), | ||
| Northing: uint32(northing), | ||
| }, nil | ||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Check specifically for
http.ErrServerClosedto avoid logging a fatal error during graceful shutdowns. Import"net/http"to use this.