Instrument recognition v2 + capability-recognition audit & fixes

migrations/033_descriptor_cache.sql

@@ -0,0 +1,28 @@
+-- ============================================================================
+-- Migration 033: v2 descriptor cache (Instrument Recognition & Capability
+-- Protocol — docs/SYSEX_IDENTITY.md)
+-- ============================================================================
+--
+-- The v2 handshake (block 1) carries a per-exemplar `instance_id` and a
+-- monotonic `revision`. Level-1 instruments additionally serve a JSON
+-- capability descriptor (block 0x10 / HTTP). GMB caches the descriptor per
+-- instrument so it can:
+--
+--   * skip re-downloading when `revision` is unchanged (ETag semantics), and
+--   * diff the previous descriptor against the next one to expire user
+--     overrides field-by-field (docs/SYSEX_IDENTITY.md §6).
+--
+-- `instance_id` itself reuses the existing `sysex_device_id` column (the v2
+-- parser writes the instance id there through saveSysExIdentity), so only the
+-- two cache columns are new here.
+--
+-- Additive only, on purpose: SQLite cannot widen a CHECK constraint in place,
+-- and rebuilding the ~50-column `instruments_latency` table is not something we
+-- can validate in an environment without the native `better-sqlite3` binding.
+-- Allowing the new `capabilities_source = 'descriptor'` value (the current
+-- constraint is IN ('manual','sysex','auto')) is therefore deferred to the
+-- slice that actually applies descriptors — see docs/SYSEX_IDENTITY.md §12.
+-- ============================================================================
+
+ALTER TABLE instruments_latency ADD COLUMN descriptor_revision INTEGER;
+ALTER TABLE instruments_latency ADD COLUMN descriptor_json TEXT;

src/midi/devices/DeviceManager.js

@@ -1024,7 +1024,15 @@ class DeviceManager {
       if (universal) return universal;
     }
 
-    // 2) Fall back to the GMB Block 1 custom format (DIY devices)
+    // 2) GMB Handshake v2 (24-byte, proto_ver 0x02) — the current instrument
+    //    recognition protocol. See docs/SYSEX_IDENTITY.md §2.
+    const handshake = this.parseGmbHandshake(bytes);
+    if (handshake) return handshake;
+
+    // 3) Legacy GMB Block 1 v1 (52-byte) — DEPRECATED, retained only for DIY
+    //    devices not yet migrated to the v2 handshake above. No wild devices
+    //    depend on it (v1 firmware was draft); safe to drop once migration
+    //    completes.
     if (bytes.length !== 52) return null;
     if (bytes[0] !== 0xf0) return null;
     if (bytes[1] !== 0x7d) return null;
@@ -1089,6 +1097,76 @@ class DeviceManager {
     };
   }
 
+  /**
+   * Decode a GMB v2 Handshake reply (24 bytes) — the current instrument
+   * recognition protocol (docs/SYSEX_IDENTITY.md §2):
+   *
+   *   F0 7D 00 01 01 <proto_ver> <instance_id[5]> <firmware[3]>
+   *      <descriptor_size[3]> <revision[5]> <flags> F7
+   *
+   * `instance_id` and `revision` are 32-bit values 7-bit-encoded over 5 bytes;
+   * `descriptor_size` is a 21-bit value over 3 bytes (`0` ⇒ level 0, no
+   * descriptor). `flags` bit 0 = HTTP available, bit 1 = push notifications.
+   * Returns null when the payload is not a v2 handshake.
+   *
+   * @param {number[]} bytes
+   * @returns {?Object}
+   */
+  parseGmbHandshake(bytes) {
+    if (!Array.isArray(bytes) || bytes.length !== 24) return null;
+    if (bytes[0] !== 0xf0 || bytes[1] !== 0x7d || bytes[2] !== 0x00) return null;
+    if (bytes[3] !== 0x01 || bytes[4] !== 0x01 || bytes[23] !== 0xf7) return null;
+
+    const protoVer = bytes[5];
+    if (protoVer !== 0x02) return null; // not a v2 handshake frame
+
+    // Full 32-bit little-endian 7-bit decode. NB: the shared
+    // decode7BitTo32Bit() masks the 5th byte to 3 bits (0x07), capping it at
+    // 31 bits — fine for the legacy v1 device id but it would silently drop
+    // bit 31 of a per-exemplar instance_id (halving the id space). The v2
+    // handshake carries the full nibble (0x0f ⇒ bits 28-31).
+    const dec32 = (b) =>
+      ((b[0] & 0x7f) |
+        ((b[1] & 0x7f) << 7) |
+        ((b[2] & 0x7f) << 14) |
+        ((b[3] & 0x7f) << 21) |
+        ((b[4] & 0x0f) << 28)) >>>
+      0;
+
+    const instanceId = dec32(bytes.slice(6, 11));
+    const firmwareMajor = bytes[11];
+    const firmwareMinor = bytes[12];
+    const firmwarePatch = bytes[13];
+    const descriptorSize =
+      (bytes[14] & 0x7f) | ((bytes[15] & 0x7f) << 7) | ((bytes[16] & 0x7f) << 14);
+    const revision = dec32(bytes.slice(17, 22));
+    const flags = bytes[22];
+
+    const instanceHex = `0x${instanceId.toString(16).padStart(8, '0').toUpperCase()}`;
+
+    return {
+      protocol: 'GMB Handshake v2',
+      protocolVersion: protoVer,
+      level: descriptorSize === 0 ? 0 : 1,
+      instanceId: instanceHex,
+      instanceIdDecimal: instanceId,
+      // Alias `deviceId` so the existing saveSysExIdentity() path persists the
+      // per-exemplar instance_id into the `sysex_device_id` column unchanged.
+      deviceId: instanceHex,
+      manufacturerName: 'GeneralMidiBoop',
+      firmwareVersion: `${firmwareMajor}.${firmwareMinor}.${firmwarePatch}`,
+      firmware: { major: firmwareMajor, minor: firmwareMinor, patch: firmwarePatch },
+      descriptorSize,
+      revision,
+      revisionHex: `0x${revision.toString(16).padStart(8, '0').toUpperCase()}`,
+      flags: {
+        httpAvailable: (flags & 0x01) !== 0,
+        pushNotifications: (flags & 0x02) !== 0
+      },
+      rawBytes: bytes.map((b) => b.toString(16).padStart(2, '0').toUpperCase()).join(' ')
+    };
+  }
+
   /**
    * Decode a MIDI Universal Identity Reply
    *   F0 7E <ch> 06 02 <mfr...> <fLSB> <fMSB> <mLSB> <mMSB> <v1 v2 v3 v4> F7

tests/devicemanager-handshake-v2.test.js

@@ -0,0 +1,192 @@
+// tests/devicemanager-handshake-v2.test.js
+// GMB v2 Handshake (block 1, 24 bytes) parsing — docs/SYSEX_IDENTITY.md §2.
+// parseGmbHandshake() decodes instance_id (5×7-bit → 32-bit), firmware (3
+// bytes), descriptor_size (3×7-bit → 21-bit), revision (5×7-bit → 32-bit) and
+// the flags byte, and routes ahead of the deprecated 52-byte v1 frame in
+// parseIdentityReply(). Invoked via prototype.call to avoid constructing the
+// full manager (which needs native easymidi).
+
+import { describe, test, expect } from '@jest/globals';
+import DeviceManager from '../src/midi/devices/DeviceManager.js';
+
+// --- 7-bit little-endian encoders. instance_id/revision use the full 32-bit
+// width: the 5th byte carries bits 28-31 (0x0f), matching parseGmbHandshake.
+function enc32(v) {
+  return [v & 0x7f, (v >>> 7) & 0x7f, (v >>> 14) & 0x7f, (v >>> 21) & 0x7f, (v >>> 28) & 0x0f];
+}
+function enc21(v) {
+  return [v & 0x7f, (v >>> 7) & 0x7f, (v >>> 14) & 0x7f];
+}
+function buildV2({
+  instanceId = 0x0a0b0c0d,
+  fw = [1, 2, 3],
+  descriptorSize = 0,
+  revision = 0,
+  flags = 0,
+  protoVer = 0x02
+} = {}) {
+  return [
+    0xf0,
+    0x7d,
+    0x00,
+    0x01,
+    0x01,
+    protoVer,
+    ...enc32(instanceId),
+    ...fw,
+    ...enc21(descriptorSize),
+    ...enc32(revision),
+    flags,
+    0xf7
+  ];
+}
+
+// Minimal `this` stub: the parser only touches logger + decode7BitTo32Bit.
+function ctx() {
+  return {
+    logger: { debug() {}, info() {}, warn() {} },
+    decode7BitTo32Bit: DeviceManager.prototype.decode7BitTo32Bit,
+    parseGmbHandshake: DeviceManager.prototype.parseGmbHandshake,
+    _parseUniversalIdentityReply: DeviceManager.prototype._parseUniversalIdentityReply,
+    getManufacturerName: DeviceManager.prototype.getManufacturerName
+  };
+}
+const parseHandshake = (bytes) => DeviceManager.prototype.parseGmbHandshake.call(ctx(), bytes);
+const parseIdentity = (bytes) => DeviceManager.prototype.parseIdentityReply.call(ctx(), { bytes });
+
+describe('parseGmbHandshake — GMB v2 handshake (24 bytes)', () => {
+  test('frame is exactly 24 bytes', () => {
+    expect(buildV2()).toHaveLength(24);
+  });
+
+  test('level 0: descriptor_size 0 → recognition only', () => {
+    const r = parseHandshake(buildV2({ instanceId: 0x12345678, fw: [2, 4, 8], descriptorSize: 0 }));
+    expect(r).not.toBeNull();
+    expect(r.protocol).toBe('GMB Handshake v2');
+    expect(r.protocolVersion).toBe(2);
+    expect(r.level).toBe(0);
+    expect(r.instanceIdDecimal).toBe(0x12345678);
+    expect(r.instanceId).toBe('0x12345678');
+    expect(r.deviceId).toBe('0x12345678'); // alias for saveSysExIdentity → sysex_device_id
+    expect(r.firmwareVersion).toBe('2.4.8');
+    expect(r.descriptorSize).toBe(0);
+  });
+
+  test('level 1: non-zero descriptor_size (21-bit) and revision decode', () => {
+    const r = parseHandshake(buildV2({ descriptorSize: 1500, revision: 42 }));
+    expect(r.level).toBe(1);
+    expect(r.descriptorSize).toBe(1500);
+    expect(r.revision).toBe(42);
+  });
+
+  test('descriptor_size uses the full 21-bit width', () => {
+    const r = parseHandshake(buildV2({ descriptorSize: 0x1fffff }));
+    expect(r.descriptorSize).toBe(0x1fffff);
+  });
+
+  test('instance_id and revision survive the full unsigned 32-bit range', () => {
+    const r = parseHandshake(buildV2({ instanceId: 0xffffffff, revision: 0xdeadbeef }));
+    expect(r.instanceIdDecimal).toBe(0xffffffff);
+    expect(r.instanceId).toBe('0xFFFFFFFF');
+    expect(r.revision).toBe(0xdeadbeef);
+    expect(r.revisionHex).toBe('0xDEADBEEF');
+  });
+
+  test('flags: HTTP (bit 0) and push (bit 1) decode independently', () => {
+    expect(parseHandshake(buildV2({ flags: 0 })).flags).toEqual({
+      httpAvailable: false,
+      pushNotifications: false
+    });
+    expect(parseHandshake(buildV2({ flags: 0x01 })).flags).toEqual({
+      httpAvailable: true,
+      pushNotifications: false
+    });
+    expect(parseHandshake(buildV2({ flags: 0x02 })).flags).toEqual({
+      httpAvailable: false,
+      pushNotifications: true
+    });
+    expect(parseHandshake(buildV2({ flags: 0x03 })).flags).toEqual({
+      httpAvailable: true,
+      pushNotifications: true
+    });
+  });
+
+  test('rejects a wrong proto_ver (not 0x02)', () => {
+    expect(parseHandshake(buildV2({ protoVer: 0x01 }))).toBeNull();
+    expect(parseHandshake(buildV2({ protoVer: 0x03 }))).toBeNull();
+  });
+
+  test('rejects a 52-byte v1 frame (wrong length)', () => {
+    const v1 = [0xf0, 0x7d, 0x00, 0x01, 0x01, 0x01, ...new Array(45).fill(0x00), 0xf7];
+    expect(v1).toHaveLength(52);
+    expect(parseHandshake(v1)).toBeNull();
+  });
+
+  test('rejects wrong header / missing terminator / short frames', () => {
+    const good = buildV2();
+    const badHeader = [...good];
+    badHeader[1] = 0x7e; // not GMB manufacturer
+    expect(parseHandshake(badHeader)).toBeNull();
+
+    const noTerminator = [...good];
+    noTerminator[23] = 0x00;
+    expect(parseHandshake(noTerminator)).toBeNull();
+
+    expect(parseHandshake(good.slice(0, 20))).toBeNull();
+    expect(parseHandshake([])).toBeNull();
+    expect(parseHandshake('not-an-array')).toBeNull();
+  });
+});
+
+describe('parseIdentityReply — routes v2 ahead of deprecated v1', () => {
+  test('a v2 handshake is decoded as GMB Handshake v2', () => {
+    const r = parseIdentity(buildV2({ instanceId: 0x00c0ffee, descriptorSize: 800 }));
+    expect(r).not.toBeNull();
+    expect(r.protocol).toBe('GMB Handshake v2');
+    expect(r.instanceIdDecimal).toBe(0x00c0ffee);
+    expect(r.level).toBe(1);
+  });
+
+  test('a legacy 52-byte v1 frame still parses as GMB Block 1 (deprecated path)', () => {
+    // F0 7D 00 01 01 <ver=01> <id[5]> <name[32]> <fw[3]> <features[5]> F7
+    const name = [0x54, 0x65, 0x73, 0x74, ...new Array(28).fill(0x00)]; // "Test" + pad
+    const v1 = [
+      0xf0,
+      0x7d,
+      0x00,
+      0x01,
+      0x01,
+      0x01, // block version 1
+      0x01,
+      0x00,
+      0x00,
+      0x00,
+      0x00, // device id = 1
+      ...name,
+      0x01,
+      0x00,
+      0x00, // firmware 1.0.0
+      0x01,
+      0x00,
+      0x00,
+      0x00,
+      0x00, // features = 0x01
+      0xf7
+    ];
+    expect(v1).toHaveLength(52);
+    const r = parseIdentity(v1);
+    expect(r).not.toBeNull();
+    expect(r.protocol).toBe('GMB Block 1');
+    expect(r.deviceName).toBe('Test');
+  });
+
+  test('a MIDI Universal Identity Reply is unaffected', () => {
+    // F0 7E <ch> 06 02 <mfr=0x41 Roland> <f_lsb f_msb> <m_lsb m_msb> <v1..v4> F7
+    const universal = [
+      0xf0, 0x7e, 0x00, 0x06, 0x02, 0x41, 0x00, 0x01, 0x02, 0x03, 0x01, 0x00, 0x00, 0x00, 0xf7
+    ];
+    const r = parseIdentity(universal);
+    expect(r).not.toBeNull();
+    expect(r.protocol).toBe('Universal Identity Reply');
+  });
+});

wiki/Hardware-Integration.md

@@ -72,4 +72,4 @@ Some hardware (motorised keyboards, automated pianos) needs explicit hand placem
 
 ## SysEx Identity
 
-`device_identity_request` triggers an `F0 7E 7F 06 01 F7` Universal SysEx Identity Request. The response is parsed into manufacturer / family / member / version fields and stored on the device row, enabling per-model defaults. The full protocol catalogue is in [`docs/SYSEX_IDENTITY.md`](https://github.com/glloq/General-Midi-Boop/blob/main/docs/SYSEX_IDENTITY.md) (1 200+ lines).
+`device_identity_request` triggers two SysEx requests: the standard `F0 7E 7F 06 01 F7` Universal Identity Request (parsed into manufacturer / family / member / version) **and** GMBoop's own `F0 7D 00 01 00 F7` v2 handshake for DIY instruments, which returns a per-exemplar `instance_id`. Both are stored on the device row. See the [[Instrument-Developer-Guide]] and the full spec in [`docs/SYSEX_IDENTITY.md`](https://github.com/glloq/General-Midi-Boop/blob/main/docs/SYSEX_IDENTITY.md).