some RFB 3.3 compatibility and DES auth

include/auth/auth.h

@@ -16,8 +16,27 @@
 
 #pragma once
 
+#include <stdint.h>
+
 struct nvnc_client;
 
+enum nvnc_auth_creds_type {
+	NVNC_AUTH_CREDS_PLAIN,
+	NVNC_AUTH_CREDS_DES,
+};
+
+struct nvnc_auth_creds {
+	enum nvnc_auth_creds_type type;
+	const char* username;
+	union {
+		const char* password;
+		struct {
+			const uint8_t* challenge;
+			const uint8_t* response;
+		} des;
+	};
+};
+
 int security_handshake_failed(struct nvnc_client* client, const char* username,
 		const char* reason_string);
 int security_handshake_ok(struct nvnc_client* client, const char* username);

include/auth/des-auth.h

@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2026 Andrian Budantsov
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#pragma once
+
+#include <stdint.h>
+#include <stdbool.h>
+
+struct nvnc_client;
+
+int des_auth_send_challenge(struct nvnc_client* client);
+int des_auth_handle_response(struct nvnc_client* client);
+bool des_auth_verify(const uint8_t* challenge, const uint8_t* response,
+		const char* password);

include/common.h

@@ -56,6 +56,7 @@ enum nvnc_client_state {
 	VNC_CLIENT_STATE_WAITING_FOR_VENCRYPT_PLAIN_AUTH,
 #endif
 #ifdef HAVE_CRYPTO
+	VNC_CLIENT_STATE_WAITING_FOR_DES_AUTH_RESPONSE,
 	VNC_CLIENT_STATE_WAITING_FOR_APPLE_DH_RESPONSE,
 	VNC_CLIENT_STATE_WAITING_FOR_RSA_AES_PUBLIC_KEY,
 	VNC_CLIENT_STATE_WAITING_FOR_RSA_AES_CHALLENGE,
@@ -96,6 +97,7 @@ struct nvnc_client {
 	char username[256];
 	struct nvnc* server;
 	enum nvnc_client_state state;
+	uint16_t rfb_minor_version;
 	struct rfb_pixel_format pixfmt;
 	enum rfb_encodings encodings[MAX_ENCODINGS + 1];
 	size_t n_encodings;
@@ -144,6 +146,7 @@ struct nvnc_client {
 	struct aml_idle* close_task;
 
 #ifdef HAVE_CRYPTO
+	uint8_t des_challenge[16];
 	struct crypto_key* apple_dh_secret;
 
 	struct {

include/crypto/des-rfb.h

@@ -0,0 +1,22 @@
+/*
+ * Copyright (c) 2026 Andrian Budantsov
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#pragma once
+
+#include <stdint.h>
+
+void crypto_des_rfb_encrypt(uint8_t* dst, const uint8_t* src,
+		const char* password);

include/neatvnc.h

@@ -57,6 +57,7 @@
 
 struct nvnc;
 struct nvnc_client;
+struct nvnc_auth_creds;
 struct nvnc_desktop_layout;
 struct nvnc_display;
 struct nvnc_fb;
@@ -117,6 +118,7 @@ enum nvnc_log_level {
 enum nvnc_auth_flags {
 	NVNC_AUTH_REQUIRE_AUTH = 1 << 0,
 	NVNC_AUTH_REQUIRE_ENCRYPTION = 1 << 1,
+	NVNC_AUTH_ALLOW_BROKEN_CRYPTO = 1 << 2,
 };
 
 struct nvnc_log_data {
@@ -134,8 +136,7 @@ typedef void (*nvnc_fb_req_fn)(struct nvnc_client*, bool is_incremental,
                                uint16_t height);
 typedef void (*nvnc_client_fn)(struct nvnc_client*);
 typedef void (*nvnc_damage_fn)(struct pixman_region16* damage, void* userdata);
-typedef bool (*nvnc_auth_fn)(const char* username, const char* password,
-                             void* userdata);
+typedef bool (*nvnc_auth_fn)(const struct nvnc_auth_creds*, void* userdata);
 typedef void (*nvnc_cut_text_fn)(struct nvnc_client*, const char* text,
 		uint32_t len);
 typedef void (*nvnc_fb_release_fn)(struct nvnc_fb*, void* context);
@@ -210,6 +211,11 @@ int nvnc_set_tls_creds(struct nvnc* self, const char* privkey_path,
                      const char* cert_path);
 int nvnc_set_rsa_creds(struct nvnc* self, const char* private_key_path);
 
+bool nvnc_auth_creds_verify(const struct nvnc_auth_creds*,
+                            const char* password);
+const char* nvnc_auth_creds_get_username(const struct nvnc_auth_creds*);
+const char* nvnc_auth_creds_get_password(const struct nvnc_auth_creds*);
+
 struct nvnc_fb* nvnc_fb_new(uint16_t width, uint16_t height,
                             uint32_t fourcc_format, uint16_t stride);
 struct nvnc_fb* nvnc_fb_from_buffer(void* buffer, uint16_t width,

meson.build

@@ -166,10 +166,12 @@ if nettle.found() and hogweed.found() and gmp.found()
 	sources += [
 		'src/crypto/random.c',
 		'src/crypto/nettle/cipher.c',
+		'src/crypto/nettle/des-rfb.c',
 		'src/crypto/nettle/hash.c',
 		'src/crypto/nettle/key.c',
 		'src/crypto/nettle/rsa.c',
 		'src/stream/rsa-aes.c',
+		'src/auth/des-auth.c',
 		'src/auth/apple-dh.c',
 		'src/auth/rsa-aes.c',
 	]

src/auth/apple-dh.c

@@ -98,7 +98,13 @@ int apple_dh_handle_response(struct nvnc_client* client)
 
 	update_min_rtt(client);
 
-	if (!server->auth_fn(username, password, server->auth_ud)) {
+	struct nvnc_auth_creds creds = {
+		.type = NVNC_AUTH_CREDS_PLAIN,
+		.username = username,
+		.password = password,
+	};
+
+	if (!server->auth_fn(&creds, server->auth_ud)) {
 		security_handshake_failed(client, username,
 				"Invalid username or password");
 		return -1;

src/auth/common.c

@@ -20,28 +20,27 @@
 #include "common.h"
 #include "neatvnc.h"
 
-int security_handshake_failed(struct nvnc_client* client, const char* username,
-		const char* reason_string)
-{
-	if (username)
-		nvnc_log(NVNC_LOG_INFO, "Security handshake failed for \"%s\": %s",
-				username, reason_string);
-	else
-		nvnc_log(NVNC_LOG_INFO, "Security handshake failed: %s",
-				reason_string);
+#include <string.h>
 
+static int security_send_failure(struct nvnc_client* client,
+		uint32_t result_code, const char* reason_string)
+{
 	char buffer[256];
 
 	uint32_t* result = (uint32_t*)buffer;
+	*result = htonl(result_code);
 
-	struct rfb_error_reason* reason =
-	        (struct rfb_error_reason*)(buffer + sizeof(*result));
-
-	*result = htonl(RFB_SECURITY_HANDSHAKE_FAILED);
-	reason->length = htonl(strlen(reason_string));
-	strcpy(reason->message, reason_string);
+	size_t len;
+	if (reason_string) {
+		struct rfb_error_reason* reason =
+			(struct rfb_error_reason*)(buffer + sizeof(*result));
+		reason->length = htonl(strlen(reason_string));
+		strcpy(reason->message, reason_string);
+		len = sizeof(*result) + sizeof(*reason) + strlen(reason_string);
+	} else {
+		len = sizeof(*result);
+	}
 
-	size_t len = sizeof(*result) + sizeof(*reason) + strlen(reason_string);
 	stream_write(client->net_stream, buffer, len, close_after_write,
 			client->net_stream);
 
@@ -51,6 +50,21 @@ int security_handshake_failed(struct nvnc_client* client, const char* username,
 	return 0;
 }
 
+int security_handshake_failed(struct nvnc_client* client, const char* username,
+		const char* reason_string)
+{
+	if (username)
+		nvnc_log(NVNC_LOG_INFO, "Security handshake failed for \"%s\": %s",
+				username, reason_string);
+	else
+		nvnc_log(NVNC_LOG_INFO, "Security handshake failed: %s",
+				reason_string);
+
+	const char* reason = client->rfb_minor_version >= 8 ? reason_string : NULL;
+	return security_send_failure(client, RFB_SECURITY_HANDSHAKE_FAILED,
+			reason);
+}
+
 int security_handshake_ok(struct nvnc_client* client, const char* username)
 {
 	if (username) {

src/auth/des-auth.c

@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2026 Andrian Budantsov
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "common.h"
+#include "stream/stream.h"
+#include "auth/auth.h"
+#include "auth/des-auth.h"
+#include "crypto.h"
+#include "crypto/des-rfb.h"
+
+#include <string.h>
+
+#define DES_CHALLENGE_SIZE 16
+
+bool des_auth_verify(const uint8_t* challenge, const uint8_t* response,
+		const char* password)
+{
+	uint8_t expected[DES_CHALLENGE_SIZE];
+	crypto_des_rfb_encrypt(expected, challenge, password);
+	return memcmp(expected, response, DES_CHALLENGE_SIZE) == 0;
+}
+
+int des_auth_send_challenge(struct nvnc_client* client)
+{
+	crypto_random(client->des_challenge, DES_CHALLENGE_SIZE);
+	return stream_write(client->net_stream, client->des_challenge,
+			DES_CHALLENGE_SIZE, NULL, NULL);
+}
+
+int des_auth_handle_response(struct nvnc_client* client)
+{
+	struct nvnc* server = client->server;
+
+	if (client->buffer_len - client->buffer_index < DES_CHALLENGE_SIZE)
+		return 0;
+
+	uint8_t* response = client->msg_buffer + client->buffer_index;
+
+	update_min_rtt(client);
+
+	struct nvnc_auth_creds creds = {
+		.type = NVNC_AUTH_CREDS_DES,
+		.username = NULL,
+		.des = {
+			.challenge = client->des_challenge,
+			.response = response,
+		},
+	};
+
+	if (!server->auth_fn(&creds, server->auth_ud)) {
+		security_handshake_failed(client, NULL,
+				"Invalid password");
+		return -1;
+	}
+
+	security_handshake_ok(client, NULL);
+	client->state = VNC_CLIENT_STATE_WAITING_FOR_INIT;
+	return DES_CHALLENGE_SIZE;
+}

src/auth/rsa-aes.c

@@ -293,7 +293,13 @@ static int on_rsa_aes_credentials(struct nvnc_client* client)
 
 	update_min_rtt(client);
 
-	if (!server->auth_fn(username, password, server->auth_ud)) {
+	struct nvnc_auth_creds creds = {
+		.type = NVNC_AUTH_CREDS_PLAIN,
+		.username = username,
+		.password = password,
+	};
+
+	if (!server->auth_fn(&creds, server->auth_ud)) {
 		security_handshake_failed(client, username,
 				"Invalid username or password");
 		return -1;

src/auth/vencrypt.c

@@ -126,7 +126,13 @@ static int on_vencrypt_plain_auth_message(struct nvnc_client* client)
 
 	update_min_rtt(client);
 
-	if (!server->auth_fn(username, password, server->auth_ud)) {
+	struct nvnc_auth_creds creds = {
+		.type = NVNC_AUTH_CREDS_PLAIN,
+		.username = username,
+		.password = password,
+	};
+
+	if (!server->auth_fn(&creds, server->auth_ud)) {
 		security_handshake_failed(client, username,
 				"Invalid username or password");
 		return -1;

src/crypto/nettle/des-rfb.c

@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2026 Andrian Budantsov
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "crypto/des-rfb.h"
+
+#include <string.h>
+#include <nettle/des.h>
+
+static void des_rfb_reverse_bits(uint8_t* dst, const char* src)
+{
+	for (int i = 0; i < 8; i++) {
+		uint8_t b = (uint8_t)src[i];
+		b = ((b & 0xf0) >> 4) | ((b & 0x0f) << 4);
+		b = ((b & 0xcc) >> 2) | ((b & 0x33) << 2);
+		b = ((b & 0xaa) >> 1) | ((b & 0x55) << 1);
+		dst[i] = b;
+	}
+}
+
+void crypto_des_rfb_encrypt(uint8_t* dst, const uint8_t* src,
+		const char* password)
+{
+	char key[8] = {};
+	size_t len = strlen(password);
+	if (len > 8)
+		len = 8;
+	memcpy(key, password, len);
+
+	uint8_t vnc_key[8];
+	des_rfb_reverse_bits(vnc_key, key);
+
+	struct des_ctx ctx;
+	des_set_key(&ctx, vnc_key);
+	des_encrypt(&ctx, 8, dst, src);
+	des_encrypt(&ctx, 8, dst + 8, src + 8);
+}