Conversation identity: key conversations by (platform, account_id, user_id)

.github/workflows/replybot-test.yml

@@ -1,14 +1,21 @@
 name: replybot-test
 
+# `devops/migrations/**` is a trigger because `replybot/lib/chatbase` is tested
+# against a real schema (see the step that starts the database below), so a
+# migration can break this suite without touching a file under `replybot/`.
 on:
   pull_request:
     paths:
       - 'replybot/**'
+      - 'testdata/**'
+      - 'devops/migrations/**'
       - '.github/workflows/replybot-test.yml'
   push:
     branches: [main]
     paths:
       - 'replybot/**'
+      - 'testdata/**'
+      - 'devops/migrations/**'
       - '.github/workflows/replybot-test.yml'
 
 jobs:
@@ -22,5 +29,19 @@ jobs:
       - uses: actions/setup-node@v4
         with:
           node-version-file: replybot/.nvmrc
+      # ONE replybot test file needs a real database: `lib/chatbase/chatbase.test.js`,
+      # which came in with the client absorbed from `@vlab-research/chatbase-postgres`.
+      # It tests SQL, so it cannot be mocked; every other replybot suite is hermetic.
+      #
+      # `make -C devops test-db` is the same target developers run locally — it
+      # starts CockroachDB on port 5433 and pipes `devops/migrations/*.sql` into
+      # it — so CI and a laptop agree on the schema by construction rather than by
+      # a second copy of the setup living here.
+      #
+      # Not a `services:` container: cockroach needs a `start-single-node` command,
+      # which GitHub's service blocks cannot supply, and the migrations have to run
+      # before mocha starts either way.
+      - name: Start test database
+        run: make -C ../devops test-db
       - run: npm ci
       - run: npm test

dashboard-server/README.md

@@ -59,7 +59,7 @@ This pattern is used when a "survey" (identified by `survey_name`) can contain m
 | `/users` | Account operations |
 | `/exports` | Async data export (via Kafka) |
 | `/typeform` | Typeform integration |
-| `/credentials` | Credential management |
+| `/credentials` | Credential management. **Messaging entities dual-write the account registry — see "Credentials and the messaging account registry"** |
 | `/facebook` | Facebook integration |
 | `/auth` | Authentication endpoints |
 | `/users/:userId/bails` | User-scoped bail-out system management (list, create, get, update, delete, preview); access controlled via `validateUserAccess` middleware |
@@ -71,6 +71,19 @@ This pattern is used when a "survey" (identified by `survey_name`) can contain m
 | `/message-templates` | Facebook Utility Message templates (CRUD per `(page, name, language)`); see `documentation/utility-messages.md` |
 | `/tickets` | Support tickets — thin UI proxy over Linear (no local storage); see `documentation/tickets.md` |
 
+### Credentials
+
+`POST /api/v1/credentials` is the **single** credential-create path. Both messaging
+connect flows go through it — `dashboard-client` `FacebookPages.js` and
+`WhatsAppEmbedded.js` each POST `entity` themselves — so there is no separate
+"connect" endpoint distinct from a generic one, which is worth knowing before adding
+entity-specific behaviour here.
+
+A messaging credential's `key` **is** the platform account id (`facebook_page` →
+page id, `whatsapp_business` → phone_number_id), and those ids are globally unique
+across messaging entities, so a token can be resolved from an account id alone
+without knowing the platform. See `message-worker/tokenstore.go`.
+
 ### Media
 
 The media feature moved from page-scoped Facebook `attachment_id`s to an

dashboard-server/queries/responses/response.test.js

@@ -86,28 +86,34 @@ describe('Response queries', () => {
     surveyName = survey.survey_name
     afterParam = null
 
-    const MOCK_QUERY = `INSERT INTO responses(parent_surveyid, parent_shortcode, surveyid, shortcode, flowid, userid, question_ref, question_idx, question_text, response, seed, timestamp)
+    // `pageid` is the messaging account, and it became part of the responses primary
+    // key in devops/migrations/28-responses-account-scoped-key.sql -- a conversation
+    // is (platform, account_id, user_id), not a user id on its own. The column is
+    // NOT NULL with no default, so a fixture has to name the account explicitly;
+    // omitting it raises 23502 rather than quietly recording an unattributed
+    // response. These rows previously omitted it and asserted `pageid: null`.
+    const MOCK_QUERY = `INSERT INTO responses(parent_surveyid, parent_shortcode, surveyid, shortcode, flowid, userid, pageid, question_ref, question_idx, question_text, response, seed, timestamp)
       VALUES
         ('${survey.id}', '231', '${survey.id
-      }', '231', 100001, '127', 'ref', 10, 'text', 'last', '6789', '${timestamps[1]
+      }', '231', 100001, '127', 'page1', 'ref', 10, 'text', 'last', '6789', '${timestamps[1]
       }')
        ,('${survey2.id}', '123', '${survey2.id
-      }', '123', 100003, '126', 'ref', 10, 'text', 'last', '6789', '${timestamps[1]
+      }', '123', 100003, '126', 'page1', 'ref', 10, 'text', 'last', '6789', '${timestamps[1]
       }')
        ,('${survey.id}', '231', '${survey.id
-      }', '231', 100004, '127', 'ref', 10, 'text', 'first', '6789', '${timestamps[2]
+      }', '231', 100004, '127', 'page1', 'ref', 10, 'text', 'first', '6789', '${timestamps[2]
       }')
        ,('${survey.id}', '231', '${survey.id
-      }', '231', 100005, '126', 'ref', 10, 'text', 'first', '6789', '${timestamps[3]
+      }', '231', 100005, '126', 'page1', 'ref', 10, 'text', 'first', '6789', '${timestamps[3]
       }')
        ,('${survey2.id}', '123', '${survey2.id
-      }', '123', 100003, '128', 'ref', 10, 'text', 'last', '6789', '${timestamps[1]
+      }', '123', 100003, '128', 'page1', 'ref', 10, 'text', 'last', '6789', '${timestamps[1]
       }')
        ,('${survey.id}', '231', '${survey.id
-      }', '231', 100004, '128', 'ref', 10, 'text', 'first', '6789', '${timestamps[2]
+      }', '231', 100004, '128', 'page1', 'ref', 10, 'text', 'first', '6789', '${timestamps[2]
       }')
        ,('${survey2.id}', '123', '${survey2.id
-      }', '123', 100005, '128', 'ref', 10, 'text', 'do not return me', '6789', '${timestamps[3]
+      }', '123', 100005, '128', 'page1', 'ref', 10, 'text', 'do not return me', '6789', '${timestamps[3]
       }')`;
 
 
@@ -149,7 +155,7 @@ describe('Response queries', () => {
           // `responses.metadata->>'ad_id'` on a NULL metadata column comes
           // back as null rather than erroring.
           ad_id: null,
-          pageid: null,
+          pageid: 'page1',
           translated_response: null,
           token: token.encoded([dbText(timestamps[1]), '127', 'ref']),
         },
@@ -167,7 +173,7 @@ describe('Response queries', () => {
           timestamp: iso(timestamps[2]),
           metadata: null,
           ad_id: null,
-          pageid: null,
+          pageid: 'page1',
           translated_response: null,
           token: token.encoded([dbText(timestamps[2]), '127', 'ref']),
         },
@@ -185,7 +191,7 @@ describe('Response queries', () => {
           timestamp: iso(timestamps[2]),
           metadata: null,
           ad_id: null,
-          pageid: null,
+          pageid: 'page1',
           translated_response: null,
           token: token.encoded([dbText(timestamps[2]), '128', 'ref']),
         },
@@ -203,7 +209,7 @@ describe('Response queries', () => {
           timestamp: iso(timestamps[3]),
           metadata: null,
           ad_id: null,
-          pageid: null,
+          pageid: 'page1',
           translated_response: null,
           token: token.encoded([dbText(timestamps[3]), '126', 'ref']),
         },
@@ -375,22 +381,22 @@ describe('Response queries', () => {
 
       // Row whose metadata JSONB contains an ad_id key.
       await vlabPool.query(
-        `INSERT INTO responses(parent_surveyid, parent_shortcode, surveyid, shortcode, flowid, userid, question_ref, question_idx, question_text, response, seed, timestamp, metadata)
-         VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13)`,
+        `INSERT INTO responses(parent_surveyid, parent_shortcode, surveyid, shortcode, flowid, userid, pageid, question_ref, question_idx, question_text, response, seed, timestamp, metadata)
+         VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14)`,
         [
           adSurvey.id, '999', adSurvey.id, '999',
-          200001, withAdUserid, 'adref', 10, 'text', 'withad', '6789',
+          200001, withAdUserid, 'page1', 'adref', 10, 'text', 'withad', '6789',
           timestamps[1], JSON.stringify({ ad_id: 'ad-12345' }),
         ],
       );
 
       // Row whose metadata JSONB exists but has no ad_id key.
       await vlabPool.query(
-        `INSERT INTO responses(parent_surveyid, parent_shortcode, surveyid, shortcode, flowid, userid, question_ref, question_idx, question_text, response, seed, timestamp, metadata)
-         VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13)`,
+        `INSERT INTO responses(parent_surveyid, parent_shortcode, surveyid, shortcode, flowid, userid, pageid, question_ref, question_idx, question_text, response, seed, timestamp, metadata)
+         VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14)`,
         [
           adSurvey.id, '999', adSurvey.id, '999',
-          200002, withoutAdUserid, 'adref', 10, 'text', 'withoutad', '6789',
+          200002, withoutAdUserid, 'page1', 'adref', 10, 'text', 'withoutad', '6789',
           timestamps[1], JSON.stringify({ utm_source: 'fb' }),
         ],
       );

dean/.gitignore

@@ -25,3 +25,9 @@
 
 # End of https://www.toptal.com/developers/gitignore/api/go
 .env
+
+# Binary output. `go build ./...` here drops a `dean` executable with no
+# extension, which none of the patterns above catch. dinersclub and
+# message-worker already gitignore theirs the same way (/dinersclub,
+# /message-worker); dean was the one that got missed.
+/dean

dean/README.md

@@ -4,19 +4,33 @@ Dean is a service that monitors the chatbase database and sends events to the bo
 
 ## Event Shape & Platform Threading
 
-Dean POSTs `ExternalEvent` JSON to botserver's `/synthetic` endpoint:
+Dean POSTs `ExternalEvent` JSON to hermes' `/synthetic` endpoint:
 
 ```json
-{ "user": "<userid>", "page": "<pageid>", "platform": "whatsapp", "event": { "type": "...", "value": null } }
+{ "user": "<userid>", "account_id": "<account_id>", "platform": "whatsapp", "event": { "type": "...", "value": null } }
 ```
 
+All three of `user`, `account_id` and `platform` are **required** — a synthetic event
+without them cannot be attributed to a conversation, and hermes rejects an incomplete POST
+with 400 once `SYNTHETIC_REQUIRE_CONVERSATION` is enabled. `documentation/event-envelope.md`
+is the contract; dean is one of six posters bound by it.
+
+The request carries `X-Vlab-Poster: dean` so hermes can name dean in a rejection log.
+
+**The field is `account_id`, not `page`.** It was renamed to match the rest of the system
+(`media_handle`, `message_templates`, migration 22); hermes still accepts `page` as a
+deprecated alias for posters that have not migrated. The value is unchanged — it is
+`states.pageid`, which holds the platform account id.
+
 Every query (`Respondings`, `Errored`, `Blocked`, `Payments`, `Timeouts`, `FollowUps`, `Spammers`)
 selects `COALESCE(states.platform, 'messenger')` and threads it into the emitted event.
 `states.platform` is a stored computed column over `state_json->'md'->>'platform'`; legacy rows
-without `md.platform` are NULL and report `messenger`. Botserver passes unknown fields through to
-Kafka untouched, so replybot receives the platform on synthetic events and can route re-entries
-(timeouts, follow-ups, repeat payments) to the correct platform instead of defaulting to Messenger.
-See `documentation/platform-abstraction.md` ("Account ID Routing").
+without `md.platform` are NULL and report `messenger` — exact for every conversation predating
+WhatsApp support, and the reason `platform` is never empty on a dean event. Replybot receives
+the platform on synthetic events and routes re-entries (timeouts, follow-ups, repeat payments)
+to the correct platform instead of defaulting to Messenger.
+See `documentation/event-envelope.md` and `documentation/platform-abstraction.md`
+("Account ID Routing").
 
 `FollowUps` joins states to credentials via `pageid = credentials.key` with
 `entity IN ('facebook_page', 'whatsapp_business')` — `states.pageid` holds the platform account id,

dean/dean.go

@@ -75,11 +75,24 @@ func send(cfg *Config, client *http.Client, e *ExternalEvent) error {
 		return err
 	}
 
-	resp, err := client.Post(cfg.Botserver, "application/json", bytes.NewBuffer(body))
+	req, err := http.NewRequest("POST", cfg.Botserver, bytes.NewBuffer(body))
 	if err != nil {
 		return err
 	}
 
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("X-Vlab-Poster", "dean")
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	// Neither this function nor the botparty.Send it mirrors ever closed the
+	// response body, so every synthetic post leaked its connection to hermes.
+	// Closing it brings dean in line with the new local posters in dinersclub and
+	// message-worker.
+	defer resp.Body.Close()
+
 	code := resp.StatusCode
 	if code != 200 {
 		err := fmt.Errorf("Non 200 response from Botserver: %v", code)

dean/queries.go

@@ -15,14 +15,12 @@ type Event struct {
 }
 
 type ExternalEvent struct {
-	User string `json:"user"`
-	Page string `json:"page"`
-	// Platform is the messaging platform of the conversation
-	// ('messenger' | 'whatsapp'), read as COALESCE(states.platform,
-	// 'messenger') — legacy state rows without md.platform report
-	// 'messenger'. Botserver's /synthetic endpoint passes unknown
-	// fields through, so this rides along to replybot untouched.
-	Platform string `json:"platform,omitempty"`
+	User      string `json:"user"`
+	AccountID string `json:"account_id"`
+	// Messaging platform of the conversation ('messenger' | 'whatsapp'), read as
+	// COALESCE(states.platform, 'messenger') — legacy rows report 'messenger'.
+	// hermes rejects a synthetic post with a missing platform once its gate is on.
+	Platform string `json:"platform"`
 	Event    *Event `json:"event"`
 }
 
@@ -52,7 +50,7 @@ func getRedo(rows pgx.Rows) *ExternalEvent {
 	err := rows.Scan(&userid, &pageid, &platform)
 	handle(err)
 
-	return &ExternalEvent{User: userid, Page: pageid, Platform: platform, Event: &Event{"redo", nil}}
+	return &ExternalEvent{User: userid, AccountID: pageid, Platform: platform, Event: &Event{"redo", nil}}
 }
 
 func getTimeout(rows pgx.Rows) *ExternalEvent {
@@ -64,7 +62,7 @@ func getTimeout(rows pgx.Rows) *ExternalEvent {
 	b, _ := json.Marshal(waitStart)
 	value := json.RawMessage(b)
 
-	return &ExternalEvent{User: userid, Page: pageid, Platform: platform, Event: &Event{"timeout", &value}}
+	return &ExternalEvent{User: userid, AccountID: pageid, Platform: platform, Event: &Event{"timeout", &value}}
 }
 
 func getPayment(rows pgx.Rows) *ExternalEvent {
@@ -81,7 +79,7 @@ func getPayment(rows pgx.Rows) *ExternalEvent {
 	b, _ := json.Marshal(v)
 	value := json.RawMessage(b)
 
-	return &ExternalEvent{User: userid, Page: pageid, Platform: platform, Event: &Event{"repeat_payment", &value}}
+	return &ExternalEvent{User: userid, AccountID: pageid, Platform: platform, Event: &Event{"repeat_payment", &value}}
 }
 
 func getFollowUp(rows pgx.Rows) *ExternalEvent {
@@ -93,7 +91,7 @@ func getFollowUp(rows pgx.Rows) *ExternalEvent {
 	b, _ := json.Marshal(question)
 	value := json.RawMessage(b)
 
-	return &ExternalEvent{User: userid, Page: pageid, Platform: platform, Event: &Event{"follow_up", &value}}
+	return &ExternalEvent{User: userid, AccountID: pageid, Platform: platform, Event: &Event{"follow_up", &value}}
 }
 
 func getBlockUser(rows pgx.Rows) *ExternalEvent {
@@ -102,7 +100,7 @@ func getBlockUser(rows pgx.Rows) *ExternalEvent {
 	handle(err)
 
 	value := json.RawMessage([]byte(`null`))
-	return &ExternalEvent{User: userid, Page: pageid, Platform: platform, Event: &Event{"block_user", &value}}
+	return &ExternalEvent{User: userid, AccountID: pageid, Platform: platform, Event: &Event{"block_user", &value}}
 }
 
 func Respondings(cfg *Config, conn *pgxpool.Pool) <-chan *ExternalEvent {

dean/queries_test.go

@@ -109,10 +109,12 @@ func TestEventsCarryPlatformFromStateWithMessengerDefault(t *testing.T) {
 	assert.Equal(t, "whatsapp", platforms["wa-user"])
 	assert.Equal(t, "messenger", platforms["legacy-user"])
 
-	// The platform field must ride along in the JSON body sent to botserver.
+	// The full triple (user, account_id, platform) must ride along in the JSON body sent to botserver.
 	for _, e := range events {
 		b, err := json.Marshal(e)
 		assert.Nil(t, err)
+		assert.Contains(t, string(b), `"user":`)
+		assert.Contains(t, string(b), `"account_id":`)
 		if e.User == "wa-user" {
 			assert.Contains(t, string(b), `"platform":"whatsapp"`)
 		} else {

devops/Makefile

@@ -81,6 +81,14 @@ test-db:
 	@docker run --name vlab-cockroach -d \
 		-p $(PORT):26257 \
 		cockroachdb/cockroach:$(DB_VERSION) start-single-node --insecure
+	@echo "Waiting for CockroachDB to accept connections..."
+	@for i in $$(seq 1 30); do \
+		if docker run --rm --net=host cockroachdb/cockroach:$(DB_VERSION) \
+			sql --insecure --host localhost --port $(PORT) -e "SELECT 1" >/dev/null 2>&1; then \
+			echo "CockroachDB is ready"; break; \
+		fi; \
+		sleep 1; \
+	done
 	@cat ./migrations/*.sql | docker run -i \
 		--net=host \
 		--rm cockroachdb/cockroach:$(DB_VERSION) \