diff --git a/backend/api/requirements.txt b/backend/api/requirements.txt index a732a98..390a006 100644 --- a/backend/api/requirements.txt +++ b/backend/api/requirements.txt @@ -4,6 +4,7 @@ asyncpg==0.27.0 boto3==1.24.89 ftfy==6.1.1 numpy==1.23.3 +openai==1.3.4 opencv-python-headless==4.5.4.60 patchify==0.2.3 Pillow==10.1.0 diff --git a/backend/api/src/api.py b/backend/api/src/api.py index 2f6a8a4..6cd0f9a 100644 --- a/backend/api/src/api.py +++ b/backend/api/src/api.py @@ -4,10 +4,11 @@ import env from middleware.error_handler import APIErrorHandler +from blueprints.converse_action_button.routes import blueprint_converse_action_button +from blueprints.narrator_camera.routes import blueprint_narrator_camera from blueprints.sketch_ping.routes import blueprint_sketch_ping from blueprints.sketch_led_state_polling.routes import blueprint_sketch_led_state_polling from blueprints.sketch_led_state_action_button.routes import blueprint_sketch_led_state_action_button -from blueprints.narrator_camera.routes import blueprint_narrator_camera # INIT api_app = Sanic('api') @@ -28,6 +29,7 @@ api_app.blueprint(blueprint_sketch_led_state_polling) # --- get started pt.3 api_app.blueprint(blueprint_narrator_camera) +api_app.blueprint(blueprint_converse_action_button) # --- goin wild (TODO) # ... diff --git a/backend/api/src/blueprints/converse_action_button/.gitignore b/backend/api/src/blueprints/converse_action_button/.gitignore new file mode 100644 index 0000000..5ace9b6 --- /dev/null +++ b/backend/api/src/blueprints/converse_action_button/.gitignore @@ -0,0 +1 @@ +speech.wav \ No newline at end of file diff --git a/backend/api/src/blueprints/converse_action_button/routes.py b/backend/api/src/blueprints/converse_action_button/routes.py new file mode 100644 index 0000000..3d50061 --- /dev/null +++ b/backend/api/src/blueprints/converse_action_button/routes.py @@ -0,0 +1,39 @@ +from io import BytesIO +import os +from sanic import Blueprint, Request, empty, json + +from models.gpt import whisper_speech_to_text + + +# BLUEPRINT: aka route prefixing/reference class we attach to the api +blueprint_converse_action_button = Blueprint("converse_action_button", url_prefix="device/converse_action_button") + + +dir_path = os.path.dirname(os.path.abspath(__file__)) +SHORT_CIRCUIT_REQUESTS = False + +# ROUTES +@blueprint_converse_action_button.route('/say', methods=['POST']) +async def route_converse_action_button_say(request: Request): + speech_path = dir_path + "/speech.wav" + # short circuit (reduce api reqs) + # if SHORT_CIRCUIT_REQUESTS == True: + # return json({ "success": True, "caption": "Short Circuiting." }) + + # REQ + try: + # --- request body for file buffer + speech_bytes = BytesIO(request.body) + # --- (optional) file saving for ref & cleanup + with open(speech_path, 'wb') as f: + print(f'Writing BytesIO to: {speech_path}') + speech_bytes.seek(0) # moves cursor to start + f.write(speech_bytes.read()) + # --- image caption + speech_bytes.seek(0) # moves cursor to start + speech_text = whisper_speech_to_text(speech_bytes.read()) + # --- respond to device + return json({ "success": True, "caption": speech_text }) + except Exception as err: + print(err) + return empty(status=500) diff --git a/backend/api/src/models/elevenlabs.py b/backend/api/src/models/elevenlabs.py index f97bcef..0c95cfd 100644 --- a/backend/api/src/models/elevenlabs.py +++ b/backend/api/src/models/elevenlabs.py @@ -24,18 +24,15 @@ def eleven_labs_text_to_speech(text: str, file_path: str, voice_id="Zlb1dXrM653N } # Request response = requests.post(f"https://api.elevenlabs.io/v1/text-to-speech/{voice_id}?optimize_streaming_latency=0&output_format={output_format}", headers=headers, json=json) - # Write all respose chunks to a file (kinda jank saving to disk in this func but w/e) audio_io = BytesIO() for chunk in response.iter_content(chunk_size=CHUNK_SIZE): if chunk: audio_io.write(chunk) audio_io.seek(0) # resets file pointer to start for reading later - # Return BytesIO return audio_io - def mp3_to_wav(mp3_iobytes: BytesIO) -> BytesIO: print('mp3_to_wav') audio = AudioSegment.from_mp3(mp3_iobytes) diff --git a/backend/api/src/models/gpt.py b/backend/api/src/models/gpt.py index d7799cc..f445959 100644 --- a/backend/api/src/models/gpt.py +++ b/backend/api/src/models/gpt.py @@ -1,36 +1,44 @@ import requests +from openai import OpenAI import env +openai_client = OpenAI(api_key=env.env_get_open_ai_api_key()) + def gpt_completion_image_caption(image_base64, prompt="What's in this image?", max_tokens=300) -> str: - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {env.env_get_open_ai_api_key()}" - } - json = { - "model": "gpt-4-vision-preview", - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": prompt, - }, - { - "type": "image_url", - "image_url": { - "url": f"data:image/jpeg;base64,{image_base64}" - } - } - ] - } - ], - "max_tokens": max_tokens - } - # request - response = requests.post("https://api.openai.com/v1/chat/completions", headers=headers, json=json) - # print(response.json()) - # request text - response_text = response.json()["choices"][0]["message"]["content"] - print(response_text) - return response_text \ No newline at end of file + print(f"[gpt_completion_image_caption] start") + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {env.env_get_open_ai_api_key()}" + } + json = { + "model": "gpt-4-vision-preview", + "messages": [ + { + "role": "user", + "content": [ + { "type": "text", "text": prompt }, + { "type": "image_url", "image_url": { "url": f"data:image/jpeg;base64,{image_base64}" } } + ] + } + ], + "max_tokens": max_tokens + } + # request + response = requests.post("https://api.openai.com/v1/chat/completions", headers=headers, json=json) + # request text + response_text = response.json()["choices"][0]["message"]["content"] + print(f"[gpt_completion_image_caption] text: {response_text}") + return response_text + +def whisper_speech_to_text(file_bytes): + print(f"[whisper_speech_to_text] start") + # transcribe + transcript = openai_client.audio.transcriptions.create( + model="whisper-1", + file=file_bytes, + language="en" + ) + transcript_text = transcript.text + # return transcription text + print(f"[whisper_speech_to_text] text: {transcript_text}") + return transcript_text diff --git a/device/converse_action_button/.gitignore b/device/converse_action_button/.gitignore new file mode 100644 index 0000000..89cc49c --- /dev/null +++ b/device/converse_action_button/.gitignore @@ -0,0 +1,5 @@ +.pio +.vscode/.browse.c_cpp.db* +.vscode/c_cpp_properties.json +.vscode/launch.json +.vscode/ipch diff --git a/device/converse_action_button/platformio.ini b/device/converse_action_button/platformio.ini new file mode 100644 index 0000000..0821d5c --- /dev/null +++ b/device/converse_action_button/platformio.ini @@ -0,0 +1,18 @@ +; PlatformIO Project Configuration File +; +; Build options: build flags, source filter +; Upload options: custom upload port, speed and extra flags +; Library options: dependencies, extra library storages +; Advanced options: extra scripting +; +; Please visit documentation for the other options and examples +; https://docs.platformio.org/page/projectconf.html + +[env:esp-s3-wroom-1] +platform = espressif32 +board = esp32-s3-devkitc-1 +framework = arduino +lib_deps = ArduinoJson +monitor_speed = 115200 +board_build.f_flash = 80000000L +board_build.flash_mode = qio diff --git a/device/converse_action_button/src/audio/I2SInput.cpp b/device/converse_action_button/src/audio/I2SInput.cpp new file mode 100644 index 0000000..0bcb64b --- /dev/null +++ b/device/converse_action_button/src/audio/I2SInput.cpp @@ -0,0 +1,98 @@ +#include +#include "soc/i2s_reg.h" + +#include "../pins.h" // TODO: this should be configured on instantiation +#include "I2SInput.h" +#include "WAVHeader.h" + +#define I2S_PORT I2S_NUM_0 + +// https://www.youtube.com/watch?v=3g7l5bm7fZ8 +i2s_config_t i2s_input_config = { + .mode = i2s_mode_t(I2S_MODE_MASTER | I2S_MODE_RX), + .sample_rate = 16000, + .bits_per_sample = I2S_BITS_PER_SAMPLE_16BIT, + .channel_format = I2S_CHANNEL_FMT_RIGHT_LEFT, // I2S_CHANNEL_FMT_RIGHT_LEFT, // I2S_CHANNEL_FMT_ONLY_RIGHT, I2S_CHANNEL_FMT_RIGHT_LEFT + .communication_format = I2S_COMM_FORMAT_STAND_I2S, + .intr_alloc_flags = ESP_INTR_FLAG_LEVEL1, // Interrupt level + .dma_buf_count = 4, // Number of DMA buffers. Adjust as needed, but keep within 2-128 + .dma_buf_len = 1024, // Size of each DMA buffer in bytes + .use_apll = false, + .tx_desc_auto_clear = false, + .fixed_mclk = 0}; + +I2SInput::I2SInput(int bckPin, int lrclkPin, int dataPin) : m_isRecording(false) +{ + m_pin_config.bck_io_num = bckPin; + m_pin_config.ws_io_num = lrclkPin; + m_pin_config.data_out_num = I2S_PIN_NO_CHANGE; + m_pin_config.data_in_num = dataPin; +} + +std::vector I2SInput::record(bool shouldRecord) +{ + // if shouldRecord is true, append audio data + if (shouldRecord) + { + if (m_isRecording == false) + { + // --- install drivers if we aren't already recording install the i2s drivers (we uninstall to avoid conflcit with output, need to verify is an issue tho) + Serial.println("[I2SInput::record] i2s install"); + i2s_driver_install(I2S_PORT, &i2s_input_config, 0, NULL); + i2s_set_pin(I2S_PORT, &m_pin_config); + // --- timing issue fix for SPH0645 mic breakout (https://youtu.be/3g7l5bm7fZ8?si=ySMNzM_kTW4FgDBl&t=268) + // REG_SET_BIT(I2S_TX_TIMING_REG(I2S_PORT), BIT(9)); + // REG_SET_BIT(I2S_TX_CONF_REG(I2S_PORT), I2S_RX_MSB_SHIFT); + } + m_isRecording = true; + // --- read bytes + uint8_t i2s_read_buff[512]; + size_t bytes_read; + esp_err_t result = i2s_read(I2S_PORT, &i2s_read_buff, sizeof(i2s_read_buff), &bytes_read, portMAX_DELAY); + if (result != ESP_OK) + { + Serial.printf("[I2SInput::record] Recording failed with error 0x%x \n", result); + } + else if (bytes_read == 0) + { + + Serial.println("[I2SInput::record] No data read from I2S"); + } + // --- append bytes to internal data vector + m_audioData.insert(m_audioData.end(), i2s_read_buff, i2s_read_buff + bytes_read); + } + // if not signalling recording, but we have audio data, means we're done! let's return it + else if (!shouldRecord && !m_audioData.empty()) + { + m_isRecording = false; + Serial.println("[I2SInput::record] adding WAV header"); + // --- create WAV header + auto wavHeader = createWavHeader(m_audioData.size() / (i2s_input_config.bits_per_sample / 8), i2s_input_config.sample_rate, 2, i2s_input_config.bits_per_sample); + // --- prepend WAV header + std::vector wavData; + wavData.reserve(wavHeader.size() + m_audioData.size()); + wavData.insert(wavData.end(), wavHeader.begin(), wavHeader.end()); + wavData.insert(wavData.end(), m_audioData.begin(), m_audioData.end()); + Serial.println("[I2SInput::record] returning audio data"); + // --- check data + Serial.printf("[I2SInput::record] recording (%d)\n", m_audioData.size()); + for (size_t i = 0; i < 16000; i++) + { + Serial.print(wavData[i], HEX); // Print each byte in hexadecimal format + } + Serial.println(); + // --- return + return wavData; + } + return std::vector(); // Return an empty vector if not recording +} + +void I2SInput::clear() +{ + // --- i think i need to uninstall the buffer? + Serial.println("[I2SInput::clear] i2s uninstall"); + i2s_driver_uninstall(I2S_PORT); + // --- clear the vector + Serial.println("[I2SInput::clear] clear"); + m_audioData.clear(); +} \ No newline at end of file diff --git a/device/converse_action_button/src/audio/I2SInput.h b/device/converse_action_button/src/audio/I2SInput.h new file mode 100644 index 0000000..454994e --- /dev/null +++ b/device/converse_action_button/src/audio/I2SInput.h @@ -0,0 +1,21 @@ +#ifndef __i2s_input_h__ +#define __i2s_input_h__ + +#include +#include +#include + +class I2SInput +{ +public: + I2SInput(int bckPin, int lrclkPin, int dataPin); + std::vector record(bool shouldRecord); + void clear(); + +private: + i2s_pin_config_t m_pin_config; + bool m_isRecording; + std::vector m_audioData; +}; + +#endif // __i2s_input_h__ \ No newline at end of file diff --git a/device/converse_action_button/src/audio/I2SOutput.cpp b/device/converse_action_button/src/audio/I2SOutput.cpp new file mode 100644 index 0000000..1fd0cff --- /dev/null +++ b/device/converse_action_button/src/audio/I2SOutput.cpp @@ -0,0 +1,97 @@ +#include +#include "driver/i2s.h" +#include + +#include "SampleSource.h" +#include "I2SOutput.h" + +// number of frames to try and send at once (a frame is a left and right sample) +#define NUM_FRAMES_TO_SEND 512 + +void taskI2SOutput(void *param) +{ + I2SOutput *output = (I2SOutput *)param; + int availableBytes = 0; + int buffer_position = 0; + Frame_t *frames = (Frame_t *)malloc(sizeof(Frame_t) * NUM_FRAMES_TO_SEND); + + // not allowing this task to return until we hit output->stop() and delete it + while (true) + { + // wait for some data to be requested + i2s_event_t evt; + if (xQueueReceive(output->m_i2sOutputQueue, &evt, portMAX_DELAY) == pdPASS) + { + if (evt.type == I2S_EVENT_TX_DONE) + { + size_t bytesWritten = 0; + // otherwise continue to process + do + { + if (availableBytes == 0) + { + if (!output->m_i2sOutputSampleGenerator->getFrames(frames, NUM_FRAMES_TO_SEND)) + { + // No more frames were filled, end of playback by deleting this task + Serial.println("[taskI2SOutput] stopping"); + output->stop(); + Serial.println("[taskI2SOutput] stopped. YOU SHOULD NOT BE SEEING THIS MESSAGE"); + return; + } + // how many bytes do we now have to send + availableBytes = NUM_FRAMES_TO_SEND * sizeof(uint32_t); + // reset the buffer position back to the start + buffer_position = 0; + } + // do we have something to write? + if (availableBytes > 0) + { + // write data to the i2s peripheral + i2s_write(output->m_i2sOutputPort, buffer_position + (uint8_t *)frames, availableBytes, &bytesWritten, portMAX_DELAY); + availableBytes -= bytesWritten; + buffer_position += bytesWritten; + } + } while (bytesWritten > 0); + } + } + } +} + +void I2SOutput::start(i2s_port_t i2sOutputPort, i2s_pin_config_t &i2sOutputPins, SampleSource *i2sOutputSampleGenerator) +{ + m_i2sOutputSampleGenerator = i2sOutputSampleGenerator; + // i2s config for writing both channels of I2S + i2s_config_t i2sConfig = { + .mode = (i2s_mode_t)(I2S_MODE_MASTER | I2S_MODE_TX), + .sample_rate = m_i2sOutputSampleGenerator->sampleRate(), + .bits_per_sample = I2S_BITS_PER_SAMPLE_16BIT, + .channel_format = I2S_CHANNEL_FMT_RIGHT_LEFT, + .communication_format = (i2s_comm_format_t)(I2S_COMM_FORMAT_I2S), + .intr_alloc_flags = ESP_INTR_FLAG_LEVEL1, + .dma_buf_count = 4, + .dma_buf_len = 1024}; + m_i2sOutputPort = i2sOutputPort; + // install and start i2s driver + i2s_driver_install(m_i2sOutputPort, &i2sConfig, 4, &m_i2sOutputQueue); + // set up the i2s pins + i2s_set_pin(m_i2sOutputPort, &i2sOutputPins); + // clear the DMA buffers + i2s_zero_dma_buffer(m_i2sOutputPort); + // start a task to write samples to the i2s peripheral + TaskHandle_t writerTaskHandle; + Serial.println("[I2SOutput::start] creating task: 'i2s Writer Task'"); + xTaskCreate(taskI2SOutput, "i2s Writer Task", 4096, this, 1, &writerTaskHandle); +} + +// Implented because we want to end the task, unlike the example which plays continuously +void I2SOutput::stop() +{ + // Stop and delete the i2s driver + Serial.println("[I2SOutput::stop] i2s_driver_uninstall'ing"); + i2s_driver_uninstall(m_i2sOutputPort); + // Stop the writer task + Serial.println("[I2SOutput::stop] vTaskDelete'ing"); + vTaskDelete(m_taskI2SOutputHandle); + // code here didn't seemingly run bc we're ending this task? this also means in the above writing task, we never actually return + Serial.println("[I2SOutput::stop] YOU SHOULD NOT BE SEEING THIS MESSAGE"); +} diff --git a/device/converse_action_button/src/audio/I2SOutput.h b/device/converse_action_button/src/audio/I2SOutput.h new file mode 100644 index 0000000..4b55cf7 --- /dev/null +++ b/device/converse_action_button/src/audio/I2SOutput.h @@ -0,0 +1,27 @@ +#ifndef __i2s_output_h__ +#define __i2s_output_h__ + +#include +#include "driver/i2s.h" + +class SampleSource; + +/** + * Base Class for both the ADC and I2S sampler + **/ +class I2SOutput +{ +private: + i2s_port_t m_i2sOutputPort; + TaskHandle_t m_taskI2SOutputHandle; // I2S write task + QueueHandle_t m_i2sOutputQueue; // i2s writer queue + SampleSource *m_i2sOutputSampleGenerator; // src of samples for us to play + +public: + void start(i2s_port_t i2sPort, i2s_pin_config_t &i2sPins, SampleSource *sample_generator); + void stop(); + + friend void taskI2SOutput(void *param); +}; + +#endif \ No newline at end of file diff --git a/device/converse_action_button/src/audio/SampleSource.h b/device/converse_action_button/src/audio/SampleSource.h new file mode 100644 index 0000000..d3830ba --- /dev/null +++ b/device/converse_action_button/src/audio/SampleSource.h @@ -0,0 +1,24 @@ +#ifndef __sample_source_h__ +#define __sample_source_h__ + +#include + +typedef struct +{ + int16_t left; + int16_t right; +} Frame_t; + +/** + * Base class for our sample generators + **/ +class SampleSource +{ +public: + virtual int sampleRate() = 0; + // This should fill the samples buffer with the specified number of frames + // A frame contains a LEFT and a RIGHT sample. Each sample should be signed 16 bits + virtual bool getFrames(Frame_t *frames, int number_frames) = 0; +}; + +#endif \ No newline at end of file diff --git a/device/converse_action_button/src/audio/WAVHeader.cpp b/device/converse_action_button/src/audio/WAVHeader.cpp new file mode 100644 index 0000000..bd4fe29 --- /dev/null +++ b/device/converse_action_button/src/audio/WAVHeader.cpp @@ -0,0 +1,60 @@ +#include +#include + +const int WAV_HEADER_SIZE = 44; + +std::vector createWavHeader(uint32_t numSamples, int sampleRate, int numChannels, int bitsPerSample) +{ + int byteRate = sampleRate * numChannels * bitsPerSample / 8; + int blockAlign = numChannels * bitsPerSample / 8; + + uint32_t dataSize = numSamples * numChannels * bitsPerSample / 8; + uint32_t fileSize = WAV_HEADER_SIZE + dataSize; + + std::vector header(WAV_HEADER_SIZE, 0); + + // RIFF header + header[0] = 'R'; + header[1] = 'I'; + header[2] = 'F'; + header[3] = 'F'; + header[4] = fileSize & 0xff; + header[5] = (fileSize >> 8) & 0xff; + header[6] = (fileSize >> 16) & 0xff; + header[7] = (fileSize >> 24) & 0xff; + header[8] = 'W'; + header[9] = 'A'; + header[10] = 'V'; + header[11] = 'E'; + + // fmt subchunk + header[12] = 'f'; + header[13] = 'm'; + header[14] = 't'; + header[15] = ' '; + header[16] = 16; // Subchunk1Size (16 for PCM) + header[20] = 1; // AudioFormat (PCM = 1) + header[22] = numChannels; + header[24] = sampleRate & 0xff; + header[25] = (sampleRate >> 8) & 0xff; + header[26] = (sampleRate >> 16) & 0xff; + header[27] = (sampleRate >> 24) & 0xff; + header[28] = byteRate & 0xff; + header[29] = (byteRate >> 8) & 0xff; + header[30] = (byteRate >> 16) & 0xff; + header[31] = (byteRate >> 24) & 0xff; + header[32] = blockAlign; + header[34] = bitsPerSample; + + // data subchunk + header[36] = 'd'; + header[37] = 'a'; + header[38] = 't'; + header[39] = 'a'; + header[40] = dataSize & 0xff; + header[41] = (dataSize >> 8) & 0xff; + header[42] = (dataSize >> 16) & 0xff; + header[43] = (dataSize >> 24) & 0xff; + + return header; +} \ No newline at end of file diff --git a/device/converse_action_button/src/audio/WAVHeader.h b/device/converse_action_button/src/audio/WAVHeader.h new file mode 100644 index 0000000..a84d0bf --- /dev/null +++ b/device/converse_action_button/src/audio/WAVHeader.h @@ -0,0 +1,12 @@ +#ifndef __wav_header_h__ +#define __wav_header_h__ + +#include +#include + +const int WAV_HEADER_SIZE = 44; + +// Function to create a WAV file header +std::vector createWavHeader(uint32_t numSamples, int sampleRate, int numChannels, int bitsPerSample); + +#endif \ No newline at end of file diff --git a/device/converse_action_button/src/audio/WAVReader.cpp b/device/converse_action_button/src/audio/WAVReader.cpp new file mode 100644 index 0000000..9dd660c --- /dev/null +++ b/device/converse_action_button/src/audio/WAVReader.cpp @@ -0,0 +1,79 @@ +#include "WAVReader.h" + +#pragma pack(push, 1) +typedef struct +{ + // RIFF Header + char riff_header[4]; // Contains "RIFF" + int wav_size; // Size of the wav portion of the file, which follows the first 8 bytes. File size - 8 + char wave_header[4]; // Contains "WAVE" + + // Format Header + char fmt_header[4]; // Contains "fmt " (includes trailing space) + int fmt_chunk_size; // Should be 16 for PCM + short audio_format; // Should be 1 for PCM. 3 for IEEE Float + short num_channels; + int sample_rate; + int byte_rate; // Number of bytes per second. sample_rate * num_channels * Bytes Per Sample + short sample_alignment; // num_channels * Bytes Per Sample + short bit_depth; // Number of bits per sample + + // Data + char data_header[4]; // Contains "data" + int data_bytes; // Number of bytes in data. Number of samples * num_channels * sample byte size + // uint8_t bytes[]; // Remainder of wave file is bytes +} wav_header_t; +#pragma pack(pop) + +WAVReader::WAVReader(const uint8_t *buffer, size_t bufferSize) + : m_buffer(buffer), m_bufferSize(bufferSize), m_currentPos(0) +{ + // Read the WAV header + wav_header_t wav_header; + memcpy(&wav_header, m_buffer, sizeof(wav_header_t)); + m_currentPos += sizeof(wav_header_t); + // Sanity check the bit depth + if (wav_header.bit_depth != 16) + { + Serial.printf("[WAVReader::WAVReader] ERROR: bit depth %d is not supported\n", wav_header.bit_depth); + } + Serial.printf("[WAVReader::WAVReader] fmt_chunk_size=%d, audio_format=%d, num_channels=%d, sample_rate=%d, sample_alignment=%d, bit_depth=%d, data_bytes=%d\n", + wav_header.fmt_chunk_size, wav_header.audio_format, wav_header.num_channels, wav_header.sample_rate, wav_header.sample_alignment, wav_header.bit_depth, wav_header.data_bytes); + // Making accessible for bytes getting calcs later in getFrames + m_num_channels = wav_header.num_channels; + m_sample_rate = wav_header.sample_rate; +} + +bool WAVReader::getFrames(Frame_t *frames, int number_frames) +{ + bool didFillFrames = false; + // Fill the buffer with data from the memory buffer + for (int i = 0; i < number_frames; i++) + { + // Check if we're about to go beyond the buffer size (ensures that there is enough data left in the buffer for a complete frame) + if (m_currentPos + sizeof(int16_t) * m_num_channels > m_bufferSize) + { + Serial.printf("[WAVReader::getFrames] getFrames reaching beyond bufferSize, didFillFrames = %d\n", didFillFrames); + // We've reached the end of the buffer, stop filling frames + return didFillFrames; + // if you want the audio to be endlessly looped, do the following vvv + // m_currentPos = sizeof(wav_header_t); // Reset to after header + } + // Read the next sample for the left channel + memcpy(&frames[i].left, m_buffer + m_currentPos, sizeof(int16_t)); + m_currentPos += sizeof(int16_t); + // Duplicate or read the right channel sample + if (m_num_channels == 1) + { + frames[i].right = frames[i].left; + } + else + { + memcpy(&frames[i].right, m_buffer + m_currentPos, sizeof(int16_t)); + m_currentPos += sizeof(int16_t); + } + didFillFrames = true; // We have filled at least one frame + } + // No more frames means return (true or false?)! + return didFillFrames; +} diff --git a/device/converse_action_button/src/audio/WAVReader.h b/device/converse_action_button/src/audio/WAVReader.h new file mode 100644 index 0000000..e81a4f6 --- /dev/null +++ b/device/converse_action_button/src/audio/WAVReader.h @@ -0,0 +1,21 @@ +#ifndef __wav_reader_h__ +#define __wav_reader_h__ + +#include "SampleSource.h" + +class WAVReader : public SampleSource +{ +public: + WAVReader(const uint8_t *buffer, size_t bufferSize); + bool getFrames(Frame_t *frames, int number_frames); + int sampleRate() { return m_sample_rate; } + +private: + const uint8_t *m_buffer; + size_t m_bufferSize; + size_t m_currentPos; + int m_num_channels; + int m_sample_rate; +}; + +#endif \ No newline at end of file diff --git a/device/converse_action_button/src/main.cpp b/device/converse_action_button/src/main.cpp new file mode 100644 index 0000000..984ef4c --- /dev/null +++ b/device/converse_action_button/src/main.cpp @@ -0,0 +1,82 @@ +#include +#include // installed via Arduino IDE +#include +#include +#include + +#include "env.h" +#include "pins.h" +#include "audio/I2SInput.h" +// #include "audio/I2SOutput.h" +// #include "audio/WAVReader.h" +#include "network/network.h" + +// APi +String apiHost = std::string(ENV_API_URL).c_str(); +// Audio +// --- input +I2SInput i2sInput(PIN_MIC_BCLK, PIN_MIC_LRCLK, PIN_MIC_DATA); // 2nd arg is I2S_WS_PIN, what is that +// --- output +// i2s_pin_config_t i2sPins = { +// .bck_io_num = PIN_AMP_BCLK, +// .ws_io_num = PIN_AMP_LRC, +// .data_out_num = PIN_AMP_DOUT, +// .data_in_num = -1}; +// I2SOutput *output; +// SampleSource *sampleSource; +// Wifi +NETWORK_H WiFiManager myWiFiManager(ENV_WIFI_SSID, ENV_WIFI_PASSWORD); + +//======================================== + +void setup() +{ + Serial.begin(115200); + Serial.setDebugOutput(true); + Serial.println(); + // --- wifi + myWiFiManager.connect(); + // --- button + pinMode(PIN_BUTTON, INPUT); + // --- leds (indicator of wip) + pinMode(PIN_LED_BUILTIN, OUTPUT); +} + +//======================================== + +void loop() +{ + // RECORDING + // --- check button state, if pressed down we're recording + bool shouldRecord = digitalRead(PIN_BUTTON) == LOW; + // --- this will be blocking if shouldRecord is true + std::vector recordedData = i2sInput.record(shouldRecord); + + // RECORDED, DO SOMETHING + if (!recordedData.empty()) + { + Serial.println("[loop] posting recording: " + String(recordedData.size()) + " bytes"); + + // SERVER: Send for speech-to-text & text-to-speech back + // --- http + HTTPClient http; + String api1Path = "/device/converse_action_button/say"; + http.begin((apiHost + api1Path).c_str()); + http.addHeader("Content-Type", "application/octet-stream"); + // --- post (TODO: graceful err handling) + int httpResponseCode = http.POST(recordedData.data(), recordedData.size()); + // --- clear the buffer now that we're done processing + i2sInput.clear(); + // --- await response + Serial.printf("[loop] httpResponseCode = %d\n", httpResponseCode); + if (httpResponseCode < 0) + { + Serial.printf("[loop] Error code: %d %s\n", httpResponseCode, http.errorToString(httpResponseCode)); + } + // --- parse response + // --- close + http.end(); + + // Do something + } +} diff --git a/device/converse_action_button/src/network/network.cpp b/device/converse_action_button/src/network/network.cpp new file mode 100644 index 0000000..c3aa7cb --- /dev/null +++ b/device/converse_action_button/src/network/network.cpp @@ -0,0 +1,29 @@ +#include "network.h" + +WiFiManager::WiFiManager(const char *ssid, const char *password) +{ + this->ssid = ssid; + this->password = password; +} + +void WiFiManager::connect() +{ + Serial.println("[WiFiManager::connect] Connecting to WiFi..."); + WiFi.begin(ssid, password); + // --- attempt (<15 times) + int attempts = 0; + while (WiFi.status() != WL_CONNECTED && attempts < 15) + { + delay(500); + Serial.print("."); + attempts++; + } + // --- mention failure if hit 15 + if (WiFi.status() != WL_CONNECTED) + { + Serial.println("[WiFiManager::connect] Failed to connect to WiFi. Please check credentials and signal."); + return; // or handle the error differently + } + // --- mention connection! + Serial.println("[WiFiManager::connect] WiFi connected"); +} diff --git a/device/converse_action_button/src/network/network.h b/device/converse_action_button/src/network/network.h new file mode 100644 index 0000000..6f54232 --- /dev/null +++ b/device/converse_action_button/src/network/network.h @@ -0,0 +1,16 @@ +#ifndef NETWORK_H +#define NETWORK_H + +#include + +class WiFiManager +{ +public: + WiFiManager(const char *ssid, const char *password); // Constructor + void connect(); // Method to connect to WiFi +private: + const char *ssid; + const char *password; +}; + +#endif \ No newline at end of file diff --git a/device/converse_action_button/src/pins.h b/device/converse_action_button/src/pins.h new file mode 100644 index 0000000..aeb8ccb --- /dev/null +++ b/device/converse_action_button/src/pins.h @@ -0,0 +1,11 @@ +#define PIN_LED_BUILTIN 39 +// is my board busted? I swear the button works when I'm off by 1 pin lol +#define PIN_BUTTON 41 + +// #define PIN_AMP_DOUT 32 +// #define PIN_AMP_BCLK 33 +// #define PIN_AMP_LRC 14 + +#define PIN_MIC_BCLK 7 +#define PIN_MIC_DATA 6 +#define PIN_MIC_LRCLK 5 \ No newline at end of file