Sketch/Project: Converse

backend/api/requirements.txt

@@ -4,6 +4,7 @@ asyncpg==0.27.0
 boto3==1.24.89
 ftfy==6.1.1
 numpy==1.23.3
+openai==1.3.4
 opencv-python-headless==4.5.4.60
 patchify==0.2.3
 Pillow==10.1.0

backend/api/src/api.py

@@ -4,10 +4,11 @@
 
 import env
 from middleware.error_handler import APIErrorHandler
+from blueprints.converse_action_button.routes import blueprint_converse_action_button
+from blueprints.narrator_camera.routes import blueprint_narrator_camera
 from blueprints.sketch_ping.routes import blueprint_sketch_ping
 from blueprints.sketch_led_state_polling.routes import blueprint_sketch_led_state_polling
 from blueprints.sketch_led_state_action_button.routes import blueprint_sketch_led_state_action_button
-from blueprints.narrator_camera.routes import blueprint_narrator_camera
 
 # INIT
 api_app = Sanic('api')
@@ -28,6 +29,7 @@
 api_app.blueprint(blueprint_sketch_led_state_polling)
 # --- get started pt.3
 api_app.blueprint(blueprint_narrator_camera)
+api_app.blueprint(blueprint_converse_action_button)
 # --- goin wild (TODO)
 # ...
 

backend/api/src/blueprints/converse_action_button/.gitignore

@@ -0,0 +1 @@
+speech.wav

backend/api/src/blueprints/converse_action_button/routes.py

@@ -0,0 +1,39 @@
+from io import BytesIO
+import os
+from sanic import Blueprint, Request, empty, json
+
+from models.gpt import whisper_speech_to_text
+
+
+# BLUEPRINT: aka route prefixing/reference class we attach to the api
+blueprint_converse_action_button = Blueprint("converse_action_button", url_prefix="device/converse_action_button")
+
+
+dir_path = os.path.dirname(os.path.abspath(__file__))
+SHORT_CIRCUIT_REQUESTS = False
+
+# ROUTES
+@blueprint_converse_action_button.route('/say', methods=['POST'])
+async def route_converse_action_button_say(request: Request):
+    speech_path = dir_path + "/speech.wav"
+    # short circuit (reduce api reqs)
+    # if SHORT_CIRCUIT_REQUESTS == True:
+    #     return json({ "success": True, "caption": "Short Circuiting." })
+
+    # REQ
+    try:
+        # --- request body for file buffer
+        speech_bytes = BytesIO(request.body)
+        # --- (optional) file saving for ref & cleanup
+        with open(speech_path, 'wb') as f:
+            print(f'Writing BytesIO to: {speech_path}')
+            speech_bytes.seek(0) # moves cursor to start
+            f.write(speech_bytes.read())
+        # --- image caption
+        speech_bytes.seek(0) # moves cursor to start
+        speech_text = whisper_speech_to_text(speech_bytes.read())
+        # --- respond to device
+        return json({ "success": True, "caption": speech_text })
+    except Exception as err:
+        print(err)
+        return empty(status=500)

backend/api/src/models/elevenlabs.py

@@ -24,18 +24,15 @@ def eleven_labs_text_to_speech(text: str, file_path: str, voice_id="Zlb1dXrM653N
     }
     # Request
     response = requests.post(f"https://api.elevenlabs.io/v1/text-to-speech/{voice_id}?optimize_streaming_latency=0&output_format={output_format}", headers=headers, json=json)
-
     # Write all respose chunks to a file (kinda jank saving to disk in this func but w/e)
     audio_io = BytesIO()
     for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
         if chunk:
             audio_io.write(chunk)
     audio_io.seek(0) # resets file pointer to start for reading later
-
     # Return BytesIO
     return audio_io
 
-
 def mp3_to_wav(mp3_iobytes: BytesIO) -> BytesIO:
     print('mp3_to_wav')
     audio = AudioSegment.from_mp3(mp3_iobytes)

backend/api/src/models/gpt.py

@@ -1,36 +1,44 @@
 import requests
+from openai import OpenAI
 import env
 
+openai_client = OpenAI(api_key=env.env_get_open_ai_api_key())
+
 def gpt_completion_image_caption(image_base64, prompt="What's in this image?", max_tokens=300) -> str:
-  headers = {
-      "Content-Type": "application/json",
-      "Authorization": f"Bearer {env.env_get_open_ai_api_key()}"
-  }
-  json = {
-      "model": "gpt-4-vision-preview",
-      "messages": [
-          {
-            "role": "user",
-            "content": [
-                {
-                    "type": "text",
-                    "text": prompt,
-                },
-                {
-                    "type": "image_url",
-                    "image_url": {
-                        "url": f"data:image/jpeg;base64,{image_base64}"
-                    }
-                }
-            ]
-          }
-      ],
-      "max_tokens": max_tokens
-  }
-  # request
-  response = requests.post("https://api.openai.com/v1/chat/completions", headers=headers, json=json)
-  # print(response.json())
-  # request text
-  response_text = response.json()["choices"][0]["message"]["content"]
-  print(response_text)
-  return response_text
+    print(f"[gpt_completion_image_caption] start")
+    headers = {
+        "Content-Type": "application/json",
+        "Authorization": f"Bearer {env.env_get_open_ai_api_key()}"
+    }
+    json = {
+        "model": "gpt-4-vision-preview",
+        "messages": [
+            {
+                "role": "user",
+                "content": [
+                    { "type": "text", "text": prompt },
+                    { "type": "image_url", "image_url": { "url": f"data:image/jpeg;base64,{image_base64}" } }
+                ]
+            }
+        ],
+        "max_tokens": max_tokens
+    }
+    # request
+    response = requests.post("https://api.openai.com/v1/chat/completions", headers=headers, json=json)
+    # request text
+    response_text = response.json()["choices"][0]["message"]["content"]
+    print(f"[gpt_completion_image_caption] text: {response_text}")
+    return response_text
+
+def whisper_speech_to_text(file_bytes):
+    print(f"[whisper_speech_to_text] start")
+    # transcribe
+    transcript = openai_client.audio.transcriptions.create(
+        model="whisper-1",
+        file=file_bytes,
+        language="en"
+    )
+    transcript_text = transcript.text
+    # return transcription text
+    print(f"[whisper_speech_to_text] text: {transcript_text}")
+    return transcript_text

device/converse_action_button/.gitignore

@@ -0,0 +1,5 @@
+.pio
+.vscode/.browse.c_cpp.db*
+.vscode/c_cpp_properties.json
+.vscode/launch.json
+.vscode/ipch

device/converse_action_button/platformio.ini

@@ -0,0 +1,18 @@
+; PlatformIO Project Configuration File
+;
+;   Build options: build flags, source filter
+;   Upload options: custom upload port, speed and extra flags
+;   Library options: dependencies, extra library storages
+;   Advanced options: extra scripting
+;
+; Please visit documentation for the other options and examples
+; https://docs.platformio.org/page/projectconf.html
+
+[env:esp-s3-wroom-1]
+platform = espressif32
+board = esp32-s3-devkitc-1
+framework = arduino
+lib_deps = ArduinoJson
+monitor_speed = 115200
+board_build.f_flash = 80000000L
+board_build.flash_mode = qio

device/converse_action_button/src/audio/I2SInput.cpp

@@ -0,0 +1,98 @@
+#include <driver/i2s.h>
+#include "soc/i2s_reg.h"
+
+#include "../pins.h" // TODO: this should be configured on instantiation
+#include "I2SInput.h"
+#include "WAVHeader.h"
+
+#define I2S_PORT I2S_NUM_0
+
+// https://www.youtube.com/watch?v=3g7l5bm7fZ8
+i2s_config_t i2s_input_config = {
+    .mode = i2s_mode_t(I2S_MODE_MASTER | I2S_MODE_RX),
+    .sample_rate = 16000,
+    .bits_per_sample = I2S_BITS_PER_SAMPLE_16BIT,
+    .channel_format = I2S_CHANNEL_FMT_RIGHT_LEFT, // I2S_CHANNEL_FMT_RIGHT_LEFT, // I2S_CHANNEL_FMT_ONLY_RIGHT, I2S_CHANNEL_FMT_RIGHT_LEFT
+    .communication_format = I2S_COMM_FORMAT_STAND_I2S,
+    .intr_alloc_flags = ESP_INTR_FLAG_LEVEL1, // Interrupt level
+    .dma_buf_count = 4,                       // Number of DMA buffers. Adjust as needed, but keep within 2-128
+    .dma_buf_len = 1024,                      // Size of each DMA buffer in bytes
+    .use_apll = false,
+    .tx_desc_auto_clear = false,
+    .fixed_mclk = 0};
+
+I2SInput::I2SInput(int bckPin, int lrclkPin, int dataPin) : m_isRecording(false)
+{
+  m_pin_config.bck_io_num = bckPin;
+  m_pin_config.ws_io_num = lrclkPin;
+  m_pin_config.data_out_num = I2S_PIN_NO_CHANGE;
+  m_pin_config.data_in_num = dataPin;
+}
+
+std::vector<uint8_t> I2SInput::record(bool shouldRecord)
+{
+  // if shouldRecord is true, append audio data
+  if (shouldRecord)
+  {
+    if (m_isRecording == false)
+    {
+      // --- install drivers if we aren't already recording install the i2s drivers (we uninstall to avoid conflcit with output, need to verify is an issue tho)
+      Serial.println("[I2SInput::record] i2s install");
+      i2s_driver_install(I2S_PORT, &i2s_input_config, 0, NULL);
+      i2s_set_pin(I2S_PORT, &m_pin_config);
+      // --- timing issue fix for SPH0645 mic breakout (https://youtu.be/3g7l5bm7fZ8?si=ySMNzM_kTW4FgDBl&t=268)
+      // REG_SET_BIT(I2S_TX_TIMING_REG(I2S_PORT), BIT(9));
+      // REG_SET_BIT(I2S_TX_CONF_REG(I2S_PORT), I2S_RX_MSB_SHIFT);
+    }
+    m_isRecording = true;
+    // --- read bytes
+    uint8_t i2s_read_buff[512];
+    size_t bytes_read;
+    esp_err_t result = i2s_read(I2S_PORT, &i2s_read_buff, sizeof(i2s_read_buff), &bytes_read, portMAX_DELAY);
+    if (result != ESP_OK)
+    {
+      Serial.printf("[I2SInput::record] Recording failed with error 0x%x \n", result);
+    }
+    else if (bytes_read == 0)
+    {
+
+      Serial.println("[I2SInput::record] No data read from I2S");
+    }
+    // --- append bytes to internal data vector
+    m_audioData.insert(m_audioData.end(), i2s_read_buff, i2s_read_buff + bytes_read);
+  }
+  // if not signalling recording, but we have audio data, means we're done! let's return it
+  else if (!shouldRecord && !m_audioData.empty())
+  {
+    m_isRecording = false;
+    Serial.println("[I2SInput::record] adding WAV header");
+    // --- create WAV header
+    auto wavHeader = createWavHeader(m_audioData.size() / (i2s_input_config.bits_per_sample / 8), i2s_input_config.sample_rate, 2, i2s_input_config.bits_per_sample);
+    // --- prepend WAV header
+    std::vector<uint8_t> wavData;
+    wavData.reserve(wavHeader.size() + m_audioData.size());
+    wavData.insert(wavData.end(), wavHeader.begin(), wavHeader.end());
+    wavData.insert(wavData.end(), m_audioData.begin(), m_audioData.end());
+    Serial.println("[I2SInput::record] returning audio data");
+    // --- check data
+    Serial.printf("[I2SInput::record] recording (%d)\n", m_audioData.size());
+    for (size_t i = 0; i < 16000; i++)
+    {
+      Serial.print(wavData[i], HEX); // Print each byte in hexadecimal format
+    }
+    Serial.println();
+    // --- return
+    return wavData;
+  }
+  return std::vector<uint8_t>(); // Return an empty vector if not recording
+}
+
+void I2SInput::clear()
+{
+  // --- i think i need to uninstall the buffer?
+  Serial.println("[I2SInput::clear] i2s uninstall");
+  i2s_driver_uninstall(I2S_PORT);
+  // --- clear the vector
+  Serial.println("[I2SInput::clear] clear");
+  m_audioData.clear();
+}

device/converse_action_button/src/audio/I2SInput.h

@@ -0,0 +1,21 @@
+#ifndef __i2s_input_h__
+#define __i2s_input_h__
+
+#include <Arduino.h>
+#include <driver/i2s.h>
+#include <vector>
+
+class I2SInput
+{
+public:
+  I2SInput(int bckPin, int lrclkPin, int dataPin);
+  std::vector<uint8_t> record(bool shouldRecord);
+  void clear();
+
+private:
+  i2s_pin_config_t m_pin_config;
+  bool m_isRecording;
+  std::vector<uint8_t> m_audioData;
+};
+
+#endif // __i2s_input_h__