From 3e4b9f33fa262951aa2e0388ad5ddd66f3bd76cd Mon Sep 17 00:00:00 2001 From: Dylan Moreland Date: Sun, 29 Jun 2025 17:05:29 -0400 Subject: [PATCH 1/6] Add a value_location field, a 2-tuple, to signal Yev suggests adding hdop. Let's think about that. --- pkg/migrations/00003_add_location.go | 45 ++++++++++++++++++++++++++++ pkg/migrations/migrations_test.go | 1 + 2 files changed, 46 insertions(+) create mode 100644 pkg/migrations/00003_add_location.go diff --git a/pkg/migrations/00003_add_location.go b/pkg/migrations/00003_add_location.go new file mode 100644 index 00000000..d74ba3a2 --- /dev/null +++ b/pkg/migrations/00003_add_location.go @@ -0,0 +1,45 @@ +package migrations + +import ( + "context" + "database/sql" + "runtime" + + "github.com/pressly/goose/v3" +) + +func init() { + _, filename, _, _ := runtime.Caller(0) + registerFunc := func() { + goose.AddNamedMigrationContext(filename, upAddLocation, downAddLocation) + } + registerFuncs = append(registerFuncs, registerFunc) +} + +func upAddLocation(ctx context.Context, tx *sql.Tx) error { + // This code is executed when the migration is applied. + upStatements := []string{ + "ALTER TABLE signal ADD COLUMN value_location Tuple(longitude Float64, latitude Float64) COMMENT 'Geographic point value, expressed in WGS-84 degrees, with longitude first in order to match most ClickHouse functions.'", + } + for _, upStatement := range upStatements { + _, err := tx.ExecContext(ctx, upStatement) + if err != nil { + return err + } + } + return nil +} + +func downAddLocation(ctx context.Context, tx *sql.Tx) error { + // This code is executed when the migration is rolled back. + downStatements := []string{ + "ALTER TABLE signal DROP COLUMN value_location", + } + for _, downStatement := range downStatements { + _, err := tx.ExecContext(ctx, downStatement) + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/migrations/migrations_test.go b/pkg/migrations/migrations_test.go index f5a96fd8..ebc680de 100644 --- a/pkg/migrations/migrations_test.go +++ b/pkg/migrations/migrations_test.go @@ -44,6 +44,7 @@ func TestSignalMigration(t *testing.T) { {Name: vss.CloudEventIDCol, Type: "String", Comment: "Id of the Cloud Event that this signal was extracted from."}, {Name: vss.ValueNumberCol, Type: "Float64", Comment: "float64 value of the signal collected."}, {Name: vss.ValueStringCol, Type: "String", Comment: "string value of the signal collected."}, + {Name: vss.ValueStringCol, Type: "String", Comment: "string value of the signal collected."}, } // Check if the actual columns match the expected columns From c4eb92b84ac03a04f6616a0d6f72f217d5db3a9b Mon Sep 17 00:00:00 2001 From: Dylan Moreland Date: Wed, 23 Jul 2025 15:12:39 -0400 Subject: [PATCH 2/6] Deconflict the migration and reorder the tuple Latitude is first and HDOP is at the end. In GIS the tradition is longitude first, by analogy with the Cartesian plane. --- go.mod | 1 - go.sum | 4 ---- ...{00003_add_location.go => 00004_add_value_location.go} | 8 ++++---- 3 files changed, 4 insertions(+), 9 deletions(-) rename pkg/migrations/{00003_add_location.go => 00004_add_value_location.go} (64%) diff --git a/go.mod b/go.mod index 2ca5406b..c9887dbb 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,6 @@ require ( github.com/containerd/platforms v0.2.1 // indirect github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/docker v28.2.2+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect diff --git a/go.sum b/go.sum index 088b13b9..b6d2b406 100644 --- a/go.sum +++ b/go.sum @@ -42,10 +42,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= -github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/docker/docker v28.2.2+incompatible h1:CjwRSksz8Yo4+RmQ339Dp/D2tGO5JxwYeqtMOEe0LDw= diff --git a/pkg/migrations/00003_add_location.go b/pkg/migrations/00004_add_value_location.go similarity index 64% rename from pkg/migrations/00003_add_location.go rename to pkg/migrations/00004_add_value_location.go index d74ba3a2..2762c572 100644 --- a/pkg/migrations/00003_add_location.go +++ b/pkg/migrations/00004_add_value_location.go @@ -11,15 +11,15 @@ import ( func init() { _, filename, _, _ := runtime.Caller(0) registerFunc := func() { - goose.AddNamedMigrationContext(filename, upAddLocation, downAddLocation) + goose.AddNamedMigrationContext(filename, upAddValueLocation, downAddValueLocation) } registerFuncs = append(registerFuncs, registerFunc) } -func upAddLocation(ctx context.Context, tx *sql.Tx) error { +func upAddValueLocation(ctx context.Context, tx *sql.Tx) error { // This code is executed when the migration is applied. upStatements := []string{ - "ALTER TABLE signal ADD COLUMN value_location Tuple(longitude Float64, latitude Float64) COMMENT 'Geographic point value, expressed in WGS-84 degrees, with longitude first in order to match most ClickHouse functions.'", + "ALTER TABLE signal ADD COLUMN value_location Tuple(latitude Float64, longitude Float64, hdop Float64) COMMENT 'Geographic point value, expressed in WGS-84 degrees.'", } for _, upStatement := range upStatements { _, err := tx.ExecContext(ctx, upStatement) @@ -30,7 +30,7 @@ func upAddLocation(ctx context.Context, tx *sql.Tx) error { return nil } -func downAddLocation(ctx context.Context, tx *sql.Tx) error { +func downAddValueLocation(ctx context.Context, tx *sql.Tx) error { // This code is executed when the migration is rolled back. downStatements := []string{ "ALTER TABLE signal DROP COLUMN value_location", From 561a092175a2c06208c6dd98c849c3ad2b722dd4 Mon Sep 17 00:00:00 2001 From: Dylan Moreland Date: Wed, 23 Jul 2025 15:15:38 -0400 Subject: [PATCH 3/6] Remove this test line because I don't understand it --- pkg/migrations/migrations_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/migrations/migrations_test.go b/pkg/migrations/migrations_test.go index ebc680de..f5a96fd8 100644 --- a/pkg/migrations/migrations_test.go +++ b/pkg/migrations/migrations_test.go @@ -44,7 +44,6 @@ func TestSignalMigration(t *testing.T) { {Name: vss.CloudEventIDCol, Type: "String", Comment: "Id of the Cloud Event that this signal was extracted from."}, {Name: vss.ValueNumberCol, Type: "Float64", Comment: "float64 value of the signal collected."}, {Name: vss.ValueStringCol, Type: "String", Comment: "string value of the signal collected."}, - {Name: vss.ValueStringCol, Type: "String", Comment: "string value of the signal collected."}, } // Check if the actual columns match the expected columns From b475e9c48554097427a9ce9cae632a40b70a676b Mon Sep 17 00:00:00 2001 From: Dylan Moreland Date: Wed, 23 Jul 2025 15:46:04 -0400 Subject: [PATCH 4/6] Respect value_location in the vss package helpers --- pkg/vss/signal.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pkg/vss/signal.go b/pkg/vss/signal.go index ae8bde06..a0e90dd9 100644 --- a/pkg/vss/signal.go +++ b/pkg/vss/signal.go @@ -25,6 +25,8 @@ const ( ValueNumberCol = "value_number" // ValueStringCol is the name of the value_string column in Clickhouse. ValueStringCol = "value_string" + // ValueLocationCol is the name of the value_location column in Clickhouse. + ValueLocationCol = "value_location" ) // Signal represents a single signal collected from a device. @@ -45,6 +47,9 @@ type Signal struct { // ValueString is the value of the signal collected. ValueString string `ch:"value_string" json:"valueString"` + // ValueLocation is the value of the signal collected. + ValueLocation Location `ch:"value_location" json:"valueLocation"` + // Source is the source of the signal collected. Source string `ch:"source" json:"source"` @@ -55,6 +60,14 @@ type Signal struct { CloudEventID string `ch:"cloud_event_id" json:"cloudEventId"` } +// Location represents a point on the earth in WSG-84 coordinates, +// optionally with a Horizontal Dilution of Position (HDOP) value. +type Location struct { + Latitude float64 `ch:"latitude"` + Longitude float64 `ch:"longitude"` + HDOP float64 `ch:"hdop"` +} + // SetValue dynamically set the appropriate value field based on the type of the value. func (s *Signal) SetValue(val any) { switch typedVal := val.(type) { @@ -62,6 +75,8 @@ func (s *Signal) SetValue(val any) { s.ValueNumber = typedVal case string: s.ValueString = typedVal + case Location: + s.ValueLocation = typedVal default: s.ValueString = fmt.Sprintf("%v", val) } @@ -79,6 +94,7 @@ func SignalToSlice(obj Signal) []any { obj.CloudEventID, obj.ValueNumber, obj.ValueString, + obj.ValueLocation, } } @@ -93,5 +109,6 @@ func SignalColNames() []string { CloudEventIDCol, ValueNumberCol, ValueStringCol, + ValueLocationCol, } } From 9577880cd509ea3c18e6a8fc59ca2dc01084c60c Mon Sep 17 00:00:00 2001 From: Dylan Moreland Date: Wed, 23 Jul 2025 16:02:16 -0400 Subject: [PATCH 5/6] Fix the migration test --- pkg/migrations/00004_add_value_location.go | 2 +- pkg/migrations/migrations_test.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/migrations/00004_add_value_location.go b/pkg/migrations/00004_add_value_location.go index 2762c572..fd1b79cd 100644 --- a/pkg/migrations/00004_add_value_location.go +++ b/pkg/migrations/00004_add_value_location.go @@ -19,7 +19,7 @@ func init() { func upAddValueLocation(ctx context.Context, tx *sql.Tx) error { // This code is executed when the migration is applied. upStatements := []string{ - "ALTER TABLE signal ADD COLUMN value_location Tuple(latitude Float64, longitude Float64, hdop Float64) COMMENT 'Geographic point value, expressed in WGS-84 degrees.'", + "ALTER TABLE signal ADD COLUMN value_location Tuple(latitude Float64, longitude Float64, hdop Float64) COMMENT 'Location value of the signal collected.'", } for _, upStatement := range upStatements { _, err := tx.ExecContext(ctx, upStatement) diff --git a/pkg/migrations/migrations_test.go b/pkg/migrations/migrations_test.go index f5a96fd8..045856db 100644 --- a/pkg/migrations/migrations_test.go +++ b/pkg/migrations/migrations_test.go @@ -44,6 +44,7 @@ func TestSignalMigration(t *testing.T) { {Name: vss.CloudEventIDCol, Type: "String", Comment: "Id of the Cloud Event that this signal was extracted from."}, {Name: vss.ValueNumberCol, Type: "Float64", Comment: "float64 value of the signal collected."}, {Name: vss.ValueStringCol, Type: "String", Comment: "string value of the signal collected."}, + {Name: vss.ValueLocationCol, Type: "Tuple(latitude Float64, longitude Float64, hdop Float64)", Comment: "Location value of the signal collected."}, } // Check if the actual columns match the expected columns From cc79922742cf70d4ee551f40ca2961b3b26be671 Mon Sep 17 00:00:00 2001 From: Dylan Moreland Date: Wed, 23 Jul 2025 16:19:01 -0400 Subject: [PATCH 6/6] Complete pig implementation for Tesla --- pkg/tesla/telemetry/convert_test.go | 2 ++ pkg/tesla/telemetry/outer_convert_funcs_gen.go | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/pkg/tesla/telemetry/convert_test.go b/pkg/tesla/telemetry/convert_test.go index b3256cca..ed899071 100644 --- a/pkg/tesla/telemetry/convert_test.go +++ b/pkg/tesla/telemetry/convert_test.go @@ -1,6 +1,7 @@ package telemetry import ( + "math" "testing" "time" @@ -131,6 +132,7 @@ func TestConvertTyped(t *testing.T) { } expectedSignals := []vss.Signal{ + {TokenID: 7, Timestamp: ts, Name: "currentLocation", ValueLocation: vss.Location{Latitude: 30.267222, Longitude: -97.743056, HDOP: math.NaN()}, Source: teslaConnection}, {TokenID: 7, Timestamp: ts, Name: "currentLocationLatitude", ValueNumber: 30.267222, Source: teslaConnection}, {TokenID: 7, Timestamp: ts, Name: "currentLocationLongitude", ValueNumber: -97.743056, Source: teslaConnection}, {TokenID: 7, Timestamp: ts, Name: "powertrainTractionBatteryCurrentPower", ValueNumber: 5700.000084936619, Source: teslaConnection}, diff --git a/pkg/tesla/telemetry/outer_convert_funcs_gen.go b/pkg/tesla/telemetry/outer_convert_funcs_gen.go index f4f51541..6d155101 100644 --- a/pkg/tesla/telemetry/outer_convert_funcs_gen.go +++ b/pkg/tesla/telemetry/outer_convert_funcs_gen.go @@ -3,6 +3,7 @@ package telemetry import ( "fmt" + "math" "github.com/DIMO-Network/model-garage/pkg/tesla/telemetry/parse" "github.com/DIMO-Network/model-garage/pkg/tesla/telemetry/unit" @@ -53,6 +54,18 @@ func ProcessPayload(payload *protos.Payload, tokenID uint32, source string) ([]v sig.SetValue(res) out = append(out, sig) } + sig := vss.Signal{ + TokenID: tokenID, + Name: "currentLocation", + Timestamp: ts, + Source: source, + } + sig.SetValue(vss.Location{ + Latitude: tvf.Latitude, + Longitude: tvf.Longitude, + HDOP: math.NaN(), + }) + out = append(out, sig) case protos.Field_DetailedChargeState: var tvf protos.DetailedChargeStateValue switch tv := d.GetValue().Value.(type) {