diff --git a/README.md b/README.md index 66a41bd..4fad41c 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,8 @@ OpenAI, and only while you use ChatGPT. go to send. Start talking once `* Recording` shows. Your speech is transcribed by OpenAI's speech-to-text model, then sent as your message to the chat model you picked. If the microphone picks up nothing, the - device says so and doesn't upload the recording. + device says so and doesn't upload the recording. Press **Esc** (T-Deck: swipe left, then Esc) + to stop a recording, a reply being written, or a reply being read aloud. - **Back up**: choose **4) Storage** → **Back up** to copy your settings to the SD card. - **Help**: type `help` to list all commands. @@ -152,6 +153,11 @@ charges its battery over USB-C while SimplrSSH runs. ## Troubleshooting - **WiFi won't connect**: make sure the network is 2.4 GHz and shows up in the scan. +- **SSH "Could not connect"**: the message says why. *Connection refused*: no SSH server on that + port, or the server is blocking the device (for example fail2ban after failed logins). + *No answer* / *no route*: wrong address, or the server isn't reachable from this WiFi (a + different network or a guest network that isolates devices). A host name also shows the + address it resolved to. - **"HOST KEY HAS CHANGED"**: if you know the server was reinstalled, run `hosts forget ` and connect again. - **Keys not found**: the card must be FAT32 and the folder must be named `ssh_keys`. diff --git a/main/board_tab5.cpp b/main/board_tab5.cpp index 6dc0ccb..10f348c 100644 --- a/main/board_tab5.cpp +++ b/main/board_tab5.cpp @@ -24,6 +24,7 @@ const char* const board::NAME = "Tab5"; const char* const board::HINT_TALK = "hold Ctrl"; const char* const board::HINT_SCROLL = "Sym+Up/Down or drag: scroll."; const char* const board::HINT_HISTORY = "Up/Down: history."; +const char* const board::HINT_STOP = "Esc"; const char* const board::SCREEN_DESC = "a 5-inch tablet with a 1280x720 text screen"; // IO expander pins (M5Stack Tab5 pin map) diff --git a/main/board_tdeck.cpp b/main/board_tdeck.cpp index 28bd776..db69a64 100644 --- a/main/board_tdeck.cpp +++ b/main/board_tdeck.cpp @@ -30,6 +30,7 @@ const char* const board::NAME = "T-Deck"; const char* const board::HINT_TALK = "hold the trackball"; const char* const board::HINT_SCROLL = "Trackball up/down or drag: scroll."; const char* const board::HINT_HISTORY = "Left/right: history."; +const char* const board::HINT_STOP = "Esc (swipe left)"; const char* const board::SCREEN_DESC = "a tiny handheld with a 320x240 text screen"; static C3Keyboard* s_keyboard = NULL; diff --git a/main/chat_app.cpp b/main/chat_app.cpp index eba4df9..5da23c5 100644 --- a/main/chat_app.cpp +++ b/main/chat_app.cpp @@ -386,7 +386,8 @@ void SSHTerminal::open_chat() lv_obj_add_flag(terminal_output, LV_OBJ_FLAG_HIDDEN); lv_obj_move_foreground(side_panel); - chat_add_bubble(BUBBLE_INFO, "ChatGPT (" + chat_model + ") - type, or " + board::HINT_TALK + " to talk.\n" + chat_add_bubble(BUBBLE_INFO, "ChatGPT (" + chat_model + ") - type, or " + board::HINT_TALK + " to talk. " + + board::HINT_STOP + " stops a reply.\n" "'clear' new chat | 'settings' | 'exit' menu"); for (const auto& m : chat_history) { chat_add_bubble(m.from_user ? BUBBLE_USER : BUBBLE_AI, m.text); @@ -397,6 +398,7 @@ void SSHTerminal::open_chat() void SSHTerminal::close_chat() { chat_recording = false; + chat_cancel = chat_busy; // Leaving stops a reply or speech in progress chat_generation = chat_generation + 1; // A running worker stops touching the view chat_active = false; input_banner.clear(); @@ -408,6 +410,19 @@ void SSHTerminal::close_chat() update_input_display(); } +// Esc in the chat (display lock held): stops recording, the reply being written, or speech +bool SSHTerminal::chat_stop() +{ + if (!chat_busy) { + return false; + } + chat_cancel = true; + chat_recording = false; + input_banner = "Stopping..."; + update_input_display(); + return true; +} + void SSHTerminal::chat_clear() { if (chat_busy) { @@ -439,7 +454,7 @@ void SSHTerminal::chat_submit(const std::string& raw) wizard.menu = WizardStep::ChatMenu; wizard_goto(WizardStep::ChatMenu); } else if (chat_busy) { - chat_add_bubble(BUBBLE_INFO, "Still answering - one moment."); + chat_add_bubble(BUBBLE_INFO, "Still answering - " + std::string(board::HINT_STOP) + " stops it."); } else if (!wifi_connected) { chat_add_bubble(BUBBLE_INFO, "WiFi is off. Type 'exit', then pick WiFi."); } else { @@ -452,6 +467,7 @@ void SSHTerminal::chat_submit(const std::string& raw) void SSHTerminal::chat_start_job(bool voice) { chat_busy = true; + chat_cancel = false; chat_job_voice = voice; if (xTaskCreate(chat_worker, "chat", WORKER_STACK, this, 4, NULL) != pdPASS) { chat_busy = false; @@ -519,7 +535,7 @@ void SSHTerminal::chat_job() } size_t n = 0; int shown = -1; - while (chat_recording && n < MAX_RECORD_SAMPLES) { + while (chat_recording && !chat_cancel && n < MAX_RECORD_SAMPLES) { n += audio::mic_read(pcm + n, std::min(1600, MAX_RECORD_SAMPLES - n)); int secs = n / audio::MIC_RATE; if (secs != shown) { @@ -529,6 +545,12 @@ void SSHTerminal::chat_job() } chat_recording = false; audio::mic_stop(); + if (chat_cancel) { + heap_caps_free(pcm); + chat_ui_bubble(gen, BUBBLE_INFO, "Stopped."); + vault::wipe(key); + return; + } size_t skip = std::min(n, SKIP_SAMPLES); int16_t* voice = pcm + skip; @@ -551,7 +573,10 @@ void SSHTerminal::chat_job() } } heap_caps_free(pcm); - if (heard.empty()) { + if (heard.empty() || chat_cancel) { + if (chat_cancel) { + chat_ui_bubble(gen, BUBBLE_INFO, "Stopped."); + } vault::wipe(key); return; } @@ -583,6 +608,9 @@ void SSHTerminal::chat_job() std::string reply; int64_t last_draw = 0; esp_err_t err = openai::chat(key, model, context, [&](const std::string& piece) { + if (chat_cancel) { + return false; + } if (reply.size() < 4000) { reply += piece; } @@ -591,8 +619,26 @@ void SSHTerminal::chat_job() last_draw = now; chat_ui_update(gen, label, reply); } + return true; }, error); + if (chat_cancel) { + // Stopped: keep what was written so far (it is part of the conversation), or nothing + chat_ui_update(gen, label, reply.empty() ? std::string("[stopped]") : reply + " [stopped]"); + if (bsp_display_lock(1000)) { + if (!reply.empty()) { + chat_history.push_back({false, reply}); + } else if (!chat_history.empty() && chat_history.back().from_user) { + chat_history.pop_back(); + } + while (chat_history.size() > MAX_HISTORY) { + chat_history.erase(chat_history.begin()); + } + bsp_display_unlock(); + } + vault::wipe(key); + return; + } if (err != ESP_OK || reply.empty()) { chat_ui_update(gen, label, "! " + (error.empty() ? std::string("No reply") : error)); if (bsp_display_lock(1000)) { @@ -618,6 +664,9 @@ void SSHTerminal::chat_job() uint8_t carry = 0; bool has_carry = false; openai::speak(key, tts_model, reply, [&](const uint8_t* data, size_t len) { + if (chat_cancel) { + return false; // Esc: silence now + } // Keep whole 16-bit samples across chunk boundaries if (has_carry && len > 0) { uint8_t pair[2] = {carry, data[0]}; @@ -632,6 +681,7 @@ void SSHTerminal::chat_job() len--; } audio::speaker_write(data, len); + return true; }, error); audio::speaker_stop(); } @@ -650,7 +700,7 @@ void SSHTerminal::on_trackball_hold(bool start, bool long_press) return; } if (chat_busy) { - chat_add_bubble(BUBBLE_INFO, "Still answering - one moment."); + chat_add_bubble(BUBBLE_INFO, "Still answering - " + std::string(board::HINT_STOP) + " stops it."); } else if (!wifi_connected) { chat_add_bubble(BUBBLE_INFO, "WiFi is off. Type 'exit', then pick WiFi."); } else { diff --git a/main/include/board.hpp b/main/include/board.hpp index 2687661..73dd993 100644 --- a/main/include/board.hpp +++ b/main/include/board.hpp @@ -55,6 +55,7 @@ namespace board extern const char* const HINT_TALK; // e.g. "hold the trackball" extern const char* const HINT_SCROLL; extern const char* const HINT_HISTORY; + extern const char* const HINT_STOP; // How to press Esc (stops a ChatGPT reply) extern const char* const SCREEN_DESC; // For the ChatGPT system prompt } diff --git a/main/include/openai_client.hpp b/main/include/openai_client.hpp index d61956e..b7fda67 100644 --- a/main/include/openai_client.hpp +++ b/main/include/openai_client.hpp @@ -32,17 +32,17 @@ namespace openai }; esp_err_t list_models(const std::string& key, Models& out, std::string& error); - // Streams the reply: on_text is called with each new piece of text + // Streams the reply: on_text is called with each new piece of text; returning false stops it esp_err_t chat(const std::string& key, const std::string& model, const std::vector& history, - const std::function& on_text, std::string& error); + const std::function& on_text, std::string& error); // 16-bit mono PCM at `rate` Hz -> text esp_err_t transcribe(const std::string& key, const std::string& model, const int16_t* pcm, size_t samples, uint32_t rate, std::string& text, std::string& error); - // Text -> 24 kHz 16-bit mono PCM, streamed to on_audio + // Text -> 24 kHz 16-bit mono PCM, streamed to on_audio; returning false stops it esp_err_t speak(const std::string& key, const std::string& model, const std::string& text, - const std::function& on_audio, std::string& error); + const std::function& on_audio, std::string& error); } #endif diff --git a/main/include/ssh_terminal.hpp b/main/include/ssh_terminal.hpp index 37fc697..b28719a 100644 --- a/main/include/ssh_terminal.hpp +++ b/main/include/ssh_terminal.hpp @@ -210,6 +210,7 @@ class SSHTerminal bool chat_job_voice = false; volatile bool chat_busy = false; volatile bool chat_recording = false; + volatile bool chat_cancel = false; // Esc: the worker stops recording, answering or speaking volatile uint32_t chat_generation = 0; // Bumped when the chat view closes std::vector chat_models; // From OpenAI, newest first (empty until asked) bool chat_models_for_menu = false; @@ -339,6 +340,7 @@ class SSHTerminal void open_chat(); void close_chat(); void chat_clear(); + bool chat_stop(); void chat_submit(const std::string& text); void chat_start_job(bool voice); void chat_job(); diff --git a/main/openai_client.cpp b/main/openai_client.cpp index b2465f2..aae90b9 100644 --- a/main/openai_client.cpp +++ b/main/openai_client.cpp @@ -111,7 +111,7 @@ static esp_err_t finish(esp_http_client_handle_t client, bool ok) } esp_err_t openai::chat(const std::string& key, const std::string& model, const std::vector& history, - const std::function& on_text, std::string& error) + const std::function& on_text, std::string& error) { cJSON* root = cJSON_CreateObject(); cJSON_AddStringToObject(root, "model", model.c_str()); @@ -169,8 +169,8 @@ esp_err_t openai::chat(const std::string& key, const std::string& model, const s cJSON* event = cJSON_Parse(line.c_str() + 6); cJSON* choice = cJSON_GetArrayItem(cJSON_GetObjectItem(event, "choices"), 0); cJSON* content = cJSON_GetObjectItem(cJSON_GetObjectItem(choice, "delta"), "content"); - if (cJSON_IsString(content) && content->valuestring[0]) { - on_text(content->valuestring); + if (cJSON_IsString(content) && content->valuestring[0] && !on_text(content->valuestring)) { + done = true; // Stopped by the user: the connection is closed below } cJSON_Delete(event); } @@ -228,7 +228,7 @@ esp_err_t openai::transcribe(const std::string& key, const std::string& model, c } esp_err_t openai::speak(const std::string& key, const std::string& model, const std::string& text, - const std::function& on_audio, std::string& error) + const std::function& on_audio, std::string& error) { cJSON* root = cJSON_CreateObject(); cJSON_AddStringToObject(root, "model", model.c_str()); @@ -253,7 +253,9 @@ esp_err_t openai::speak(const std::string& key, const std::string& model, const uint8_t buf[1024]; int n; while ((n = esp_http_client_read(client, (char*)buf, sizeof(buf) & ~1u)) > 0) { - on_audio(buf, n); + if (!on_audio(buf, n)) { + break; // Stopped by the user + } } ESP_LOGI(TAG, "Speech done"); return finish(client, true); diff --git a/main/ssh_terminal.cpp b/main/ssh_terminal.cpp index c28ae54..835b221 100644 --- a/main/ssh_terminal.cpp +++ b/main/ssh_terminal.cpp @@ -30,6 +30,8 @@ #include #include #include +#include +#include #include static const char *TAG = "SSH_TERMINAL"; @@ -898,6 +900,8 @@ void SSHTerminal::handle_key_input(char key) ssh_teardown(); } go_home(); + } else if (chat_active && key == 27) { + chat_stop(); // Esc stops recording, the reply or speech } else if (chat_active && (key == '\n' || key == '\r')) { std::string text = current_input; vault::wipe(current_input); @@ -1338,6 +1342,71 @@ static esp_err_t resolve_host(const char* host, struct in_addr* addr) return ESP_OK; } +// Opens a TCP connection, waiting at most timeout_s. Returns the socket (>= 0) or -errno. +static int tcp_connect(const struct sockaddr_in& sin, int timeout_s) +{ + int fd = socket(AF_INET, SOCK_STREAM, 0); + if (fd < 0) { + return -(errno ? errno : ENOMEM); + } + int flags = fcntl(fd, F_GETFL, 0); + fcntl(fd, F_SETFL, flags | O_NONBLOCK); + int err = 0; + if (::connect(fd, (const struct sockaddr*)&sin, sizeof(sin)) != 0) { + err = errno; + if (err == EINPROGRESS) { + fd_set wfds; + FD_ZERO(&wfds); + FD_SET(fd, &wfds); + struct timeval tv = {timeout_s, 0}; + int rc = select(fd + 1, NULL, &wfds, NULL, &tv); + if (rc == 0) { + err = ETIMEDOUT; + } else if (rc < 0) { + err = errno; + } else { + socklen_t len = sizeof(err); + if (getsockopt(fd, SOL_SOCKET, SO_ERROR, &err, &len) != 0) { + err = errno; + } + } + } + } + if (err) { + close(fd); + return -err; + } + fcntl(fd, F_SETFL, flags); // Back to blocking; libssh2 sets its own mode + struct timeval timeout = {10, 0}; + setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)); + setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(timeout)); + return fd; +} + +// What a failed connection means, for the user +static std::string connect_error_text(int err) +{ + switch (err) { + case ECONNREFUSED: + return "connection refused - is SSH running on that port, or is the device blocked (e.g. fail2ban)?"; + case ETIMEDOUT: + return "no answer in 10 s - check the address, and that the server is reachable from this WiFi"; + case EHOSTUNREACH: + case ENETUNREACH: + return "no route to the server - check the address and the WiFi connection"; + case ECONNRESET: + case ECONNABORTED: + return "the connection was reset"; + case ENFILE: + case EMFILE: + case ENOMEM: + case ENOBUFS: + return "out of network resources - try again, or restart the device"; + default: + return "error " + std::to_string(err) + " (" + strerror(err) + ")"; + } +} + int SSHTerminal::waitsocket(int socket_fd, LIBSSH2_SESSION *session) { struct timeval timeout; @@ -1427,7 +1496,6 @@ esp_err_t SSHTerminal::ssh_begin(PendingSsh&& target) return ESP_FAIL; } - ESP_LOGI(TAG, "Connecting to %s:%d", t.host.c_str(), t.port); append_text(("Connecting to " + t.host + ":" + std::to_string(t.port) + "...\n").c_str()); refresh_display_now(); @@ -1439,23 +1507,27 @@ esp_err_t SSHTerminal::ssh_begin(PendingSsh&& target) wipe_pending_ssh(); return ESP_FAIL; } - - ssh_socket = socket(AF_INET, SOCK_STREAM, 0); - if (ssh_socket < 0) { - append_text("ERROR: Failed to create socket\n"); - wipe_pending_ssh(); - return ESP_FAIL; + char ip[16]; + inet_ntoa_r(sin.sin_addr, ip, sizeof(ip)); + if (t.host != ip) { + append_text((" (" + t.host + " is " + ip + ")\n").c_str()); // Shows a name resolving wrongly } + refresh_display_now(); - struct timeval timeout = {10, 0}; - setsockopt(ssh_socket, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)); - setsockopt(ssh_socket, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(timeout)); - - if (::connect(ssh_socket, (struct sockaddr*)(&sin), sizeof(sin)) != 0) { - append_text("ERROR: Failed to connect socket\n"); - ssh_teardown(); + // One retry for errors a freshly joined network can give (the route or ARP not ready yet) + int fd = tcp_connect(sin, 10); + if (fd == -EHOSTUNREACH || fd == -ENETUNREACH || fd == -ECONNRESET || fd == -ECONNABORTED) { + vTaskDelay(pdMS_TO_TICKS(1000)); + fd = tcp_connect(sin, 10); + } + if (fd < 0) { + ESP_LOGE(TAG, "TCP connect failed: errno %d", -fd); + append_text(("ERROR: Could not connect to " + std::string(ip) + ":" + std::to_string(t.port) + ": " + + connect_error_text(-fd) + "\n").c_str()); + wipe_pending_ssh(); return ESP_FAIL; } + ssh_socket = fd; session = libssh2_session_init(); if (!session) { @@ -2026,6 +2098,12 @@ void SSHTerminal::send_special_key(const char* sequence) return; } + if (strcmp(sequence, "\x1B") == 0 && chat_active && !wizard_active()) { + chat_stop(); + toggle_side_panel(); + return; + } + if (strcmp(sequence, "\x1B") == 0 && wizard_active()) { handle_key_input(27); toggle_side_panel();