diff --git a/README.org b/README.org index e9cc692..58bef57 100644 --- a/README.org +++ b/README.org @@ -18,6 +18,7 @@ Pipe content between your terminal and Emacs buffers - seamlessly bridge your co - *Buffer matching*: Use exact names or regex patterns to find buffers - *Auto-generation*: Creates ~*Piper 1*~, ~*Piper 2*~, etc. when no name specified - *Conflict handling*: Avoids overwriting buffers unless forced +- *Persistent socket*: Auto-boots a TCP eval server inside Emacs for fast, ordered communication (falls back to ~emacsclient~ transparently) The script works with both bash and zsh, handles special characters properly, and includes comprehensive error handling. @@ -88,6 +89,30 @@ Download and put it somewhere in the $PATH Make sure [[https://www.gnu.org/software/emacs/manual/html_node/emacs/Emacs-Server.html][Emacs daemon]] is running and emacsclient works! +** Socket transport + +On first use, ~mxp~ boots a lightweight TCP eval server inside your running Emacs and connects to it directly via bash's ~/dev/tcp~. All subsequent calls reuse this persistent connection, which means: + +- No process spawning per eval (previously each chunk launched a new ~emacsclient~ process) +- Guaranteed ordering for streamed content (no more background job races) +- No "Connection refused" errors under heavy streaming load + +The server starts automatically - no manual ~M-x~ step required. If the socket is unavailable (e.g., bash built without ~/dev/tcp~ support), ~mxp~ falls back to ~emacsclient~ transparently. + +#+attr_html: :width 50% +| Variable | Default | Description | +|---------------+---------+-----------------------------------------------| +| ~MXP_PORT~ | ~17394~ | TCP port for the eval server (localhost only) | +| ~MXP_NO_SOCKET~ | (unset) | Set to ~1~ to force ~emacsclient~-only mode | + +To stop the server manually: + +#+begin_src emacs-lisp +(when (and (boundp 'mxp-server-process) (process-live-p mxp-server-process)) + (delete-process mxp-server-process) + (setq mxp-server-process nil)) +#+end_src + ** Use *** Open files and directories in Emacs diff --git a/changelog.org b/changelog.org index bc3466f..68f28b8 100644 --- a/changelog.org +++ b/changelog.org @@ -3,6 +3,27 @@ #+DATE: 2025 All notable changes to this project will be documented in this file. + +* [0.6.0] - 2026-04-06 + +** Added +- Persistent TCP socket transport between ~mxp~ and Emacs +- Auto-bootstrapping: ~mxp~ starts a lightweight eval server inside Emacs on first use (no manual ~M-x~ required) +- All communication goes over a single persistent connection per invocation via bash's ~/dev/tcp~ +- Transparent fallback to ~emacsclient~ when socket is unavailable +- ~MXP_PORT~ env var for custom server port (default: 17394) +- ~MXP_NO_SOCKET~ env var to force ~emacsclient~-only mode +- Socket transport test suite + +** Changed +- Streaming now sends all chunks sequentially over one socket - guaranteed ordering, no race conditions +- ~open_mode~ uses ~emacsclient -n~ directly (avoids interactive prompts from process filter context) + +** Removed +- Background job concurrency limiter (~jobs -r~ / ~sleep 0.01~ spin-wait) - no longer needed with persistent connection +- Sync/async flush split - all flushes are now uniform +- ~wait~ at end of ~write_mode~ - no background jobs to collect + * [0.5.3] - 2025-10-29 - Correctly handles emojis and other utf-8 chars diff --git a/mxp b/mxp index c7d7bf5..d1f3236 100755 --- a/mxp +++ b/mxp @@ -9,6 +9,8 @@ set -euo pipefail VERSION="0.5.3" CHUNK_SIZE=100 STREAM_TIMEOUT=0.3 +MXP_PORT="${MXP_PORT:-17394}" +MXP_SOCK_FD="" usage() { cat << EOF @@ -191,15 +193,7 @@ self_update() { echo "Backup saved at: ${backup}" } -check_emacsclient() { - if ! command -v emacsclient &> /dev/null; then - die "emacsclient not found. Please install Emacs." - fi - - if ! emacsclient --eval "t" &> /dev/null; then - die "Cannot connect to Emacs server. Start Emacs daemon with: emacs --daemon" - fi -} + # Detect if argument is a file/directory path (vs buffer name) is_path() { @@ -231,14 +225,149 @@ encode_for_elisp() { fi } +# --- Socket transport layer --- +# Boots a TCP eval server inside Emacs (idempotent - safe to call repeatedly) +mxp_server_start() { + local port="$MXP_PORT" + local elisp + read -r -d '' elisp <<'MXP_ELISP' || true +(unless (and (boundp 'mxp-server-process) + mxp-server-process + (process-live-p mxp-server-process)) + (defvar mxp-server-port MXP_PORT_PLACEHOLDER) + (defvar mxp-server-process nil) + (defun mxp--server-filter (proc input) + (let ((buf (concat (or (process-get proc :mxp-buf) "") input))) + (process-put proc :mxp-buf nil) + (while (string-match "\n" buf) + (let* ((line (substring buf 0 (match-beginning 0))) + (rest (substring buf (match-end 0))) + (code (ignore-errors + (decode-coding-string + (base64-decode-string line) 'utf-8))) + (resp + (if (null code) + (concat "ERR " + (base64-encode-string "invalid request" t)) + (condition-case err + (let ((result (eval (read code)))) + (concat "OK " + (base64-encode-string + (encode-coding-string + (format "%S" result) 'utf-8) + t))) + (error + (concat "ERR " + (base64-encode-string + (encode-coding-string + (error-message-string err) 'utf-8) + t))))))) + (process-send-string proc (concat resp "\n")) + (setq buf rest))) + (when (< 0 (length buf)) + (process-put proc :mxp-buf buf)))) + (defun mxp--server-sentinel (proc event) + (process-put proc :mxp-buf nil)) + (setq mxp-server-process + (make-network-process + :name "mxp-server" + :server t + :host "127.0.0.1" + :service mxp-server-port + :family 'ipv4 + :coding 'utf-8 + :filter #'mxp--server-filter + :sentinel #'mxp--server-sentinel + :noquery t))) +mxp-server-port +MXP_ELISP + elisp="${elisp//MXP_PORT_PLACEHOLDER/$port}" + local encoded + encoded=$(printf '%s' "$elisp" | encode_for_elisp) + emacsclient --eval \ + "(eval (read (decode-coding-string (base64-decode-string \"$encoded\") 'utf-8)))" \ + >/dev/null 2>&1 || return 1 + sleep 0.1 + return 0 +} + +# Open a persistent TCP connection to the mxp server +mxp_connect() { + [ -n "$MXP_SOCK_FD" ] && return 0 + { exec 3<>/dev/tcp/127.0.0.1/"$MXP_PORT"; } 2>/dev/null || return 1 + MXP_SOCK_FD=3 + return 0 +} + +# Close the socket +mxp_disconnect() { + if [ -n "$MXP_SOCK_FD" ]; then + exec 3>&- 2>/dev/null || true + MXP_SOCK_FD="" + fi +} + +# Connect to an existing server, or bootstrap one in Emacs +mxp_ensure_connection() { + mxp_connect && return 0 + mxp_server_start || return 1 + mxp_connect +} + +# Evaluate elisp over the socket. Result on stdout, errors on stderr. +mxp_sock_eval() { + local code="$1" + local encoded + encoded=$(printf '%s' "$code" | encode_for_elisp) + echo "$encoded" >&3 + local response + read -r response <&3 || return 1 + local status="${response%% *}" + local data="${response#* }" + local decoded + decoded=$(printf '%s' "$data" | base64 --decode 2>/dev/null) || return 1 + if [ "$status" = "OK" ]; then + echo "$decoded" + return 0 + else + echo "mxp: emacs error: $decoded" >&2 + return 1 + fi +} + +# Evaluate elisp - tries socket first, falls back to emacsclient +emacs_eval() { + local code="$1" + if [ -n "$MXP_SOCK_FD" ]; then + mxp_sock_eval "$code" && return 0 + mxp_disconnect + fi + # Fallback: "$code" preserves embedded quotes as literal characters + emacsclient --eval "$code" 2>/dev/null +} + +# Verify Emacs is reachable and set up the best transport +init_emacs_connection() { + if ! command -v emacsclient &> /dev/null; then + die "emacsclient not found. Please install Emacs." + fi + if [ "${MXP_NO_SOCKET:-}" != "1" ]; then + mxp_ensure_connection && return 0 + fi + # Socket unavailable - verify emacsclient can reach Emacs + if ! emacsclient --eval "t" &> /dev/null; then + die "Cannot connect to Emacs server. Start Emacs daemon with: emacs --daemon" + fi +} + # Find buffer by name or regex pattern (returns buffer name, not object) find_buffer() { local pattern="$1" - emacsclient --eval "(let ((buf (or (get-buffer \"$pattern\") - (car (seq-filter (lambda (b) - (string-match-p \"$pattern\" (buffer-name b))) - (buffer-list)))))) - (when buf (buffer-name buf)))" 2>/dev/null | sed 's/"//g' | grep -v '^nil$' || echo "" + emacs_eval "(let ((buf (or (get-buffer \"$pattern\") + (car (seq-filter (lambda (b) + (string-match-p \"$pattern\" (buffer-name b))) + (buffer-list)))))) + (when buf (buffer-name buf)))" | sed 's/"//g' | grep -v '^nil$' || echo "" } # Generate unique buffer name @@ -249,7 +378,7 @@ generate_buffer_name() { while true; do name="$base $num*" - if ! emacsclient --eval "(buffer-live-p (get-buffer \"$name\"))" 2>/dev/null | grep -q 't'; then + if ! emacs_eval "(buffer-live-p (get-buffer \"$name\"))" | grep -q 't'; then echo "$name" return fi @@ -285,7 +414,7 @@ resolve_buffer_name() { while true; do candidate="${base}<${num}>" - if ! emacsclient --eval "(buffer-live-p (get-buffer \"$candidate\"))" 2>/dev/null | grep -q 't'; then + if ! emacs_eval "(buffer-live-p (get-buffer \"$candidate\"))" | grep -q 't'; then echo "$candidate" return fi @@ -297,8 +426,6 @@ resolve_buffer_name() { read_mode() { local pattern="$1" - check_emacsclient - local buffer buffer=$(find_buffer "$pattern") @@ -306,7 +433,7 @@ read_mode() { die "No buffer matching '$pattern' found" fi - # Get buffer content via temporary file (avoids emacsclient output limits and base64 overhead) + # Get buffer content via temporary file (avoids output limits and base64 overhead) local temp_file temp_file=$(mktemp) @@ -314,7 +441,7 @@ read_mode() { trap "rm -f '$temp_file'" EXIT INT TERM # Write buffer to temp file with explicit UTF-8 encoding to avoid prompts - if ! emacsclient --eval "(let ((coding-system-for-write 'utf-8)) + if ! emacs_eval "(let ((coding-system-for-write 'utf-8)) (with-current-buffer \"$buffer\" (write-region (point-min) (point-max) \"$temp_file\" nil 'quiet)))" > /dev/null 2>&1; then die "Failed to read buffer '$buffer'" @@ -328,8 +455,6 @@ read_mode() { open_mode() { local path="$1" - check_emacsclient - # Expand ~ to home directory if needed if [[ "$path" == "~"* ]]; then path="${path/#\~/$HOME}" @@ -345,14 +470,13 @@ open_mode() { die "Path does not exist: $path" fi - # Open in Emacs (no-wait mode) + # Open in Emacs (emacsclient -n handles frame management and interactive prompts + # correctly, unlike find-file from a process filter context) if [ -d "$path" ]; then - # Directory - open in dired - emacsclient -n "$path" &>/dev/null + emacsclient -n "$path" &>/dev/null || die "Failed to open '$path'" echo "Opened directory in Emacs: $path" else - # File - open with find-file - emacsclient -n "$path" &>/dev/null + emacsclient -n "$path" &>/dev/null || die "Failed to open '$path'" echo "Opened file in Emacs: $path" fi } @@ -364,8 +488,6 @@ write_mode() { local prepend="$3" local force="$4" - check_emacsclient - local buffer_name if [ -z "$buffer_pattern" ]; then buffer_name=$(generate_buffer_name) @@ -389,50 +511,35 @@ write_mode() { # Create/prepare buffer if [ "$append" = "true" ]; then - emacsclient --eval "(with-current-buffer (get-buffer-create \"$buffer_name\") + emacs_eval "(with-current-buffer (get-buffer-create \"$buffer_name\") (goto-char (point-max)) nil)" > /dev/null 2>&1 elif [ "$prepend" = "true" ]; then - emacsclient --eval "(with-current-buffer (get-buffer-create \"$buffer_name\") + emacs_eval "(with-current-buffer (get-buffer-create \"$buffer_name\") (goto-char (point-min)) nil)" > /dev/null 2>&1 else - emacsclient --eval "(with-current-buffer (get-buffer-create \"$buffer_name\") + emacs_eval "(with-current-buffer (get-buffer-create \"$buffer_name\") (erase-buffer) nil)" > /dev/null 2>&1 fi # Display buffer - emacsclient --eval "(display-buffer \"$buffer_name\")" > /dev/null 2>&1 + emacs_eval "(display-buffer \"$buffer_name\")" > /dev/null 2>&1 # Helper function to flush accumulated content to Emacs flush_to_emacs() { if [ -n "$accumulated_content" ] && [ "$line_count" -gt 0 ]; then local encoded encoded=$(printf '%s' "$accumulated_content" | encode_for_elisp) - - # Send synchronously for small chunks, async for large streams - if [ "$total_flushes" -lt 3 ]; then - # First few flushes: synchronous (prevents issues with &>/dev/null) - emacsclient --eval "(with-current-buffer \"$buffer_name\" - (let ((start (goto-char $insert_point)) - (dstr (decode-coding-string (base64-decode-string \"$encoded\") 'utf-8))) - (insert dstr) - (when (boundp 'mxp-buffer-update-hook) - (run-hook-with-args 'mxp-buffer-update-hook \"$buffer_name\" start (point)))))" >/dev/null 2>&1 - else - # Many flushes: use background jobs for performance - while [ $(jobs -r | wc -l | tr -d ' ') -ge 5 ]; do - sleep 0.01 - done - emacsclient --eval "(with-current-buffer \"$buffer_name\" - (let ((start (goto-char $insert_point)) - (dstr (decode-coding-string (base64-decode-string \"$encoded\") 'utf-8))) - (insert dstr) - (when (boundp 'mxp-buffer-update-hook) - (run-hook-with-args 'mxp-buffer-update-hook \"$buffer_name\" start (point)))))" >/dev/null 2>&1 & - fi - + + emacs_eval "(with-current-buffer \"$buffer_name\" + (let ((start (goto-char $insert_point)) + (dstr (decode-coding-string (base64-decode-string \"$encoded\") 'utf-8))) + (insert dstr) + (when (boundp 'mxp-buffer-update-hook) + (run-hook-with-args 'mxp-buffer-update-hook \"$buffer_name\" start (point)))))" >/dev/null 2>&1 + accumulated_content="" line_count=0 total_flushes=$((total_flushes + 1)) @@ -487,15 +594,14 @@ write_mode() { fi done - # Wait for all background jobs to complete - wait - # Run hooks - emacsclient --eval "(with-current-buffer \"$buffer_name\" + emacs_eval "(with-current-buffer \"$buffer_name\" (when (boundp 'mxp-buffer-hook) (run-hook-with-args 'mxp-buffer-hook \"$buffer_name\")) (when (boundp 'mxp-buffer-complete-hook) (run-hook-with-args 'mxp-buffer-complete-hook \"$buffer_name\")))" > /dev/null 2>&1 + + mxp_disconnect } main() { local mode="write" @@ -562,7 +668,11 @@ main() { if [ "$append" = "true" ] && [ "$prepend" = "true" ]; then die "Cannot use both --append and --prepend" fi - + + # Initialize connection to Emacs (socket with emacsclient fallback) + init_emacs_connection + trap 'mxp_disconnect' EXIT + # Execute mode if [ "$mode" = "read" ]; then if [ -z "$buffer_pattern" ]; then diff --git a/test-mxp.sh b/test-mxp.sh index 39cf098..bb6762f 100755 --- a/test-mxp.sh +++ b/test-mxp.sh @@ -594,6 +594,151 @@ else fail "Temp files not cleaned up: before=$before, after=$after" fi +# --- Socket Transport Tests --- + +section "Test 24: Socket Server Bootstrap" +if should_run_test "socket"; then + if [ "$LIST_TESTS" = true ]; then + echo " [socket] Socket server bootstrap" + else + # Stop any existing mxp server to test fresh bootstrap + emacsclient --eval "(when (and (boundp 'mxp-server-process) mxp-server-process (process-live-p mxp-server-process)) (delete-process mxp-server-process) (setq mxp-server-process nil))" &>/dev/null || true + + cleanup_buffer "*test-socket-boot*" + echo "socket boot test" | $SCRIPT "*test-socket-boot*" &>/dev/null + + if buffer_exists "*test-socket-boot*"; then + pass "Socket server auto-bootstraps on first use" + else + fail "Socket server bootstrap failed" + fi + + # Verify server is running in Emacs + server_alive=$(emacsclient --eval "(and (boundp 'mxp-server-process) mxp-server-process (process-live-p mxp-server-process))" 2>/dev/null) + if [[ "$server_alive" != "nil" ]]; then + pass "Server process is alive in Emacs after bootstrap" + else + fail "Server process not found in Emacs" + fi + fi +fi + +section "Test 25: Socket Eval Round-trip" +if should_run_test "socket"; then + if [ "$LIST_TESTS" = true ]; then + echo " [socket] Socket eval round-trip" + else + cleanup_buffer "*test-socket-rt*" + echo "round trip content" | $SCRIPT "*test-socket-rt*" &>/dev/null + output=$($SCRIPT --from "*test-socket-rt*" 2>/dev/null) + if [[ "$output" == *"round trip content"* ]]; then + pass "Write and read via socket transport" + else + fail "Socket round-trip failed: got '$output'" + fi + fi +fi + +section "Test 26: Socket Streaming" +if should_run_test "socket"; then + if [ "$LIST_TESTS" = true ]; then + echo " [socket] Socket streaming (1000 lines)" + else + cleanup_buffer "*test-socket-stream*" + seq 1 1000 | $SCRIPT "*test-socket-stream*" &>/dev/null + output=$($SCRIPT --from "*test-socket-stream*" 2>/dev/null) + line_count=$(echo "$output" | wc -l | tr -d ' ') + + if [ "$line_count" = "1000" ]; then + pass "1000 lines streamed over socket in order" + else + fail "Socket streaming: expected 1000 lines, got $line_count" + fi + + # Verify first and last lines + first=$(echo "$output" | head -1) + last=$(echo "$output" | tail -1) + if [ "$first" = "1" ] && [ "$last" = "1000" ]; then + pass "Stream ordering preserved (first=1, last=1000)" + else + fail "Stream order wrong: first='$first', last='$last'" + fi + fi +fi + +section "Test 27: Emacsclient Fallback" +if should_run_test "socket"; then + if [ "$LIST_TESTS" = true ]; then + echo " [socket] Emacsclient fallback with MXP_NO_SOCKET=1" + else + cleanup_buffer "*test-fallback*" + echo "fallback content" | MXP_NO_SOCKET=1 $SCRIPT "*test-fallback*" &>/dev/null + output=$(MXP_NO_SOCKET=1 $SCRIPT --from "*test-fallback*" 2>/dev/null) + + if [[ "$output" == *"fallback content"* ]]; then + pass "MXP_NO_SOCKET=1 falls back to emacsclient" + else + fail "Fallback failed: got '$output'" + fi + + # Verify streaming also works in fallback + cleanup_buffer "*test-fallback-stream*" + seq 1 50 | MXP_NO_SOCKET=1 $SCRIPT "*test-fallback-stream*" &>/dev/null + output=$(MXP_NO_SOCKET=1 $SCRIPT --from "*test-fallback-stream*" 2>/dev/null) + line_count=$(echo "$output" | wc -l | tr -d ' ') + + if [ "$line_count" = "50" ]; then + pass "Streaming works in fallback mode (50 lines)" + else + fail "Fallback streaming: expected 50 lines, got $line_count" + fi + fi +fi + +section "Test 28: Server Idempotence" +if should_run_test "socket"; then + if [ "$LIST_TESTS" = true ]; then + echo " [socket] Server idempotence" + else + # Run mxp twice - server should already exist, no error + # Buffer names must be regex-distinct (trailing * in Emacs regex + # means "zero or more of preceding char", so *foo-1* matches *foo-*) + cleanup_buffer "*test-idemp-alpha*" + cleanup_buffer "*test-idemp-beta*" + echo "first" | $SCRIPT "*test-idemp-alpha*" &>/dev/null + echo "second" | $SCRIPT "*test-idemp-beta*" &>/dev/null + + out1=$($SCRIPT --from "*test-idemp-alpha*" 2>/dev/null) + out2=$($SCRIPT --from "*test-idemp-beta*" 2>/dev/null) + + if [[ "$out1" == *"first"* ]] && [[ "$out2" == *"second"* ]]; then + pass "Multiple mxp invocations reuse existing server" + else + fail "Server idempotence failed" + fi + fi +fi + +section "Test 29: Custom Port" +if should_run_test "socket"; then + if [ "$LIST_TESTS" = true ]; then + echo " [socket] Custom MXP_PORT" + else + cleanup_buffer "*test-custom-port*" + echo "custom port" | MXP_PORT=17395 $SCRIPT "*test-custom-port*" &>/dev/null + output=$(MXP_PORT=17395 $SCRIPT --from "*test-custom-port*" 2>/dev/null) + + if [[ "$output" == *"custom port"* ]]; then + pass "MXP_PORT override works" + else + fail "Custom port failed: got '$output'" + fi + + # Clean up the extra server + emacsclient --eval "(when (and (boundp 'mxp-server-process) mxp-server-process) (delete-process mxp-server-process) (setq mxp-server-process nil))" &>/dev/null || true + fi +fi + # Summary section "Test Summary" echo ""