From 02ac5ee1209c26a2be15f35b2e67614b74004cff Mon Sep 17 00:00:00 2001 From: Wayne Gao Date: Thu, 11 Jun 2026 14:19:58 +0800 Subject: [PATCH 01/24] feat(kv): add shared KV backend interfaces under src/plugins/kv Signed-off-by: Wayne Gao --- meson.build | 5 +- src/plugins/kv/kv_engine.cpp | 82 +++++++++++++++++++++ src/plugins/kv/kv_engine.h | 127 ++++++++++++++++++++++++++++++++ src/plugins/kv/kv_engine_impl.h | 100 +++++++++++++++++++++++++ src/plugins/kv/kv_store.h | 60 +++++++++++++++ 5 files changed, 373 insertions(+), 1 deletion(-) create mode 100644 src/plugins/kv/kv_engine.cpp create mode 100644 src/plugins/kv/kv_engine.h create mode 100644 src/plugins/kv/kv_engine_impl.h create mode 100644 src/plugins/kv/kv_store.h diff --git a/meson.build b/meson.build index 6e1f7ad7ca..4cb898abc2 100644 --- a/meson.build +++ b/meson.build @@ -508,7 +508,7 @@ if get_option('buildtype') == 'debug' run_command('truncate', '-s 0', plugfile, check: true) endif -nixl_inc_dirs = include_directories('src/api/cpp', 'src/api/cpp/backend', 'src/infra', 'src/core', 'src/core/telemetry') +nixl_inc_dirs = include_directories('src/api/cpp', 'src/api/cpp/backend', 'src/infra', 'src/core', 'src/core/telemetry', 'src/plugins/kv') nixl_gpu_inc_dirs = include_directories('src/api/gpu/ucx') plugins_inc_dirs = include_directories('src/plugins') utils_inc_dirs = include_directories('src/utils') @@ -538,6 +538,9 @@ if get_option('install_headers') install_headers('src/core/transfer_request.h', install_dir: prefix_inc) install_headers('src/core/agent_data.h', install_dir: prefix_inc) install_headers('src/infra/mem_section.h', install_dir: prefix_inc) + install_headers('src/plugins/kv/kv_store.h', install_dir: prefix_inc + '/plugins/kv') + install_headers('src/plugins/kv/kv_engine_impl.h', install_dir: prefix_inc + '/plugins/kv') + install_headers('src/plugins/kv/kv_engine.h', install_dir: prefix_inc + '/plugins/kv') install_headers('src/core/telemetry/telemetry_event.h', install_dir: prefix_inc) if ucx_gpu_device_api_available diff --git a/src/plugins/kv/kv_engine.cpp b/src/plugins/kv/kv_engine.cpp new file mode 100644 index 0000000000..b086a8c2bb --- /dev/null +++ b/src/plugins/kv/kv_engine.cpp @@ -0,0 +1,82 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @file kv_engine.cpp + * @brief nixlKVEngine delegation layer; forwards nixlBackendEngine calls to impl_. + */ + +#include "kv_engine.h" + +nixlKVEngine::nixlKVEngine(const nixlBackendInitParams *init_params, + std::unique_ptr impl) + : nixlBackendEngine(init_params), + impl_(std::move(impl)) {} + +nixlKVEngine::~nixlKVEngine() = default; + +nixl_mem_list_t +nixlKVEngine::getSupportedMems() const { + return impl_->getSupportedMems(); +} + +nixl_status_t +nixlKVEngine::registerMem(const nixlBlobDesc &mem, + const nixl_mem_t &nixl_mem, + nixlBackendMD *&out) { + return impl_->registerMem(mem, nixl_mem, out); +} + +nixl_status_t +nixlKVEngine::deregisterMem(nixlBackendMD *meta) { + return impl_->deregisterMem(meta); +} + +nixl_status_t +nixlKVEngine::queryMem(const nixl_reg_dlist_t &descs, std::vector &resp) const { + return impl_->queryMem(descs, resp); +} + +nixl_status_t +nixlKVEngine::prepXfer(const nixl_xfer_op_t &operation, + const nixl_meta_dlist_t &local, + const nixl_meta_dlist_t &remote, + const std::string &remote_agent, + nixlBackendReqH *&handle, + const nixl_opt_b_args_t *opt_args) const { + return impl_->prepXfer(operation, local, remote, remote_agent, localAgent, handle, opt_args); +} + +nixl_status_t +nixlKVEngine::postXfer(const nixl_xfer_op_t &operation, + const nixl_meta_dlist_t &local, + const nixl_meta_dlist_t &remote, + const std::string &remote_agent, + nixlBackendReqH *&handle, + const nixl_opt_b_args_t *opt_args) const { + return impl_->postXfer(operation, local, remote, remote_agent, handle, opt_args); +} + +nixl_status_t +nixlKVEngine::checkXfer(nixlBackendReqH *handle) const { + return impl_->checkXfer(handle); +} + +nixl_status_t +nixlKVEngine::releaseReqH(nixlBackendReqH *handle) const { + return impl_->releaseReqH(handle); +} diff --git a/src/plugins/kv/kv_engine.h b/src/plugins/kv/kv_engine.h new file mode 100644 index 0000000000..9765f9bc2e --- /dev/null +++ b/src/plugins/kv/kv_engine.h @@ -0,0 +1,127 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @file kv_engine.h + * @brief Shared thin wrapper base class for KV-style NIXL backend engines. + * + * nixlKVEngine implements nixlBackendEngine and forwards all data-plane calls + * to a nixlKVEngineImpl instance supplied by each plugin. + * + * Plugin authors subclass nixlKVEngine (or use it directly with a factory) and + * register the resulting engine with nixlBackendPluginCreator. + */ + +#ifndef NIXL_KV_ENGINE_H +#define NIXL_KV_ENGINE_H + +#include "kv_engine_impl.h" +#include + +/** + * @class nixlKVEngine + * @brief Thin nixlBackendEngine wrapper that delegates to nixlKVEngineImpl. + * + * Common KV backend capabilities (local-only, no notifications) are declared here + * so individual plugins only need to supply a concrete nixlKVEngineImpl. + */ +class nixlKVEngine : public nixlBackendEngine { +public: + /** + * @brief Construct a KV engine with a plugin-specific implementation. + * @param init_params NIXL initialization parameters. + * @param impl Concrete nixlKVEngineImpl (ownership transferred). + */ + nixlKVEngine(const nixlBackendInitParams *init_params, std::unique_ptr impl); + + ~nixlKVEngine() override; + + bool + supportsRemote() const override { + return false; + } + + bool + supportsLocal() const override { + return true; + } + + bool + supportsNotif() const override { + return false; + } + + nixl_mem_list_t + getSupportedMems() const override; + + nixl_status_t + registerMem(const nixlBlobDesc &mem, const nixl_mem_t &nixl_mem, nixlBackendMD *&out) override; + + nixl_status_t + deregisterMem(nixlBackendMD *meta) override; + + nixl_status_t + queryMem(const nixl_reg_dlist_t &descs, std::vector &resp) const override; + + nixl_status_t + connect(const std::string &remote_agent) override { + return NIXL_SUCCESS; + } + + nixl_status_t + disconnect(const std::string &remote_agent) override { + return NIXL_SUCCESS; + } + + nixl_status_t + unloadMD(nixlBackendMD *input) override { + return NIXL_SUCCESS; + } + + nixl_status_t + loadLocalMD(nixlBackendMD *input, nixlBackendMD *&output) override { + output = input; + return NIXL_SUCCESS; + } + + nixl_status_t + prepXfer(const nixl_xfer_op_t &operation, + const nixl_meta_dlist_t &local, + const nixl_meta_dlist_t &remote, + const std::string &remote_agent, + nixlBackendReqH *&handle, + const nixl_opt_b_args_t *opt_args = nullptr) const override; + + nixl_status_t + postXfer(const nixl_xfer_op_t &operation, + const nixl_meta_dlist_t &local, + const nixl_meta_dlist_t &remote, + const std::string &remote_agent, + nixlBackendReqH *&handle, + const nixl_opt_b_args_t *opt_args = nullptr) const override; + + nixl_status_t + checkXfer(nixlBackendReqH *handle) const override; + + nixl_status_t + releaseReqH(nixlBackendReqH *handle) const override; + +private: + std::unique_ptr impl_; +}; + +#endif // NIXL_KV_ENGINE_H diff --git a/src/plugins/kv/kv_engine_impl.h b/src/plugins/kv/kv_engine_impl.h new file mode 100644 index 0000000000..0891ed8fd1 --- /dev/null +++ b/src/plugins/kv/kv_engine_impl.h @@ -0,0 +1,100 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @file kv_engine_impl.h + * @brief Abstract implementation interface for KV-style NIXL backend engines. + * + * Architecture: + * + * nixlBackendEngine NIXL agent-facing protocol (registerMem, prepXfer, ...) + * ^ + * | nixlKVEngine Thin wrapper; delegates lifecycle calls to impl_ + * | + * nixlKVEngineImpl Vendor/backend-specific logic (this header) + * ^ + * | + * nixlInMemKVEngineImpl Example: in-process map via iKVStore + * nixlRedisKVEngineImpl Future: Redis via hiredis implementing iKVStore + * + * Storage is further factored through iKVStore (kv_store.h) so impl classes focus + * on NIXL descriptor/key mapping while iKVStore handles put/get/exists. + */ + +#ifndef NIXL_KV_ENGINE_IMPL_H +#define NIXL_KV_ENGINE_IMPL_H + +#include "backend/backend_engine.h" +#include +#include + +/** + * @class nixlKVEngineImpl + * @brief Abstract implementation interface for KV-style backend engines. + * + * Each KV plugin (INMEMKV example, REDIS src plugin, etc.) provides a concrete + * subclass that implements register/deregister, query, and synchronous transfer + * operations against its chosen iKVStore backend. + */ +class nixlKVEngineImpl { +public: + virtual ~nixlKVEngineImpl() = default; + + /** @return Memory segment types supported by this KV backend (typically DRAM_SEG). */ + virtual nixl_mem_list_t + getSupportedMems() const = 0; + + virtual nixl_status_t + registerMem(const nixlBlobDesc &mem, const nixl_mem_t &nixl_mem, nixlBackendMD *&out) = 0; + + virtual nixl_status_t + deregisterMem(nixlBackendMD *meta) = 0; + + virtual nixl_status_t + queryMem(const nixl_reg_dlist_t &descs, std::vector &resp) const = 0; + + /** + * @brief Prepare a transfer request. + * + * @param local_agent Agent name from nixlKVEngine::localAgent (passed explicitly + * so impl does not depend on nixlBackendEngine protected members). + */ + virtual nixl_status_t + prepXfer(const nixl_xfer_op_t &operation, + const nixl_meta_dlist_t &local, + const nixl_meta_dlist_t &remote, + const std::string &remote_agent, + const std::string &local_agent, + nixlBackendReqH *&handle, + const nixl_opt_b_args_t *opt_args) const = 0; + + virtual nixl_status_t + postXfer(const nixl_xfer_op_t &operation, + const nixl_meta_dlist_t &local, + const nixl_meta_dlist_t &remote, + const std::string &remote_agent, + nixlBackendReqH *&handle, + const nixl_opt_b_args_t *opt_args) const = 0; + + virtual nixl_status_t + checkXfer(nixlBackendReqH *handle) const = 0; + + virtual nixl_status_t + releaseReqH(nixlBackendReqH *handle) const = 0; +}; + +#endif // NIXL_KV_ENGINE_IMPL_H diff --git a/src/plugins/kv/kv_store.h b/src/plugins/kv/kv_store.h new file mode 100644 index 0000000000..003f83194f --- /dev/null +++ b/src/plugins/kv/kv_store.h @@ -0,0 +1,60 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @file kv_store.h + * @brief Shared minimal key-value storage interface for NIXL KV backends. + * + * Implementations (e.g. INMEMKV example plugin, REDIS src plugin) provide + * backend-specific storage while sharing this synchronous put/get/exists API. + */ + +#ifndef NIXL_KV_STORE_H +#define NIXL_KV_STORE_H + +#include "nixl_types.h" +#include +#include +#include + +/** + * @brief Minimal key-value storage interface for KV-style NIXL backends. + * + * This interface intentionally keeps only the smallest synchronous API + * needed by current KV backend flows (INMEMKV, REDIS, etc.). + */ +class iKVStore { +public: + virtual ~iKVStore() = default; + + virtual nixl_status_t + put(std::string_view key, const uint8_t *data, size_t len) = 0; + + /** + * @brief Reads value for key into buffer. + * + * bytes_read is set to min(stored_len, len) on success. + * Returns NIXL_ERR_BACKEND when key does not exist. + */ + virtual nixl_status_t + get(std::string_view key, uint8_t *buffer, size_t len, size_t &bytes_read) const = 0; + + virtual bool + exists(std::string_view key) const = 0; +}; + +#endif // NIXL_KV_STORE_H From a190d3c75f5460d7649edce24d2bdf3862a80ead Mon Sep 17 00:00:00 2001 From: Wayne Gao Date: Sat, 13 Jun 2026 01:15:36 +0800 Subject: [PATCH 02/24] fix(kv): code style fixes Signed-off-by: Wayne Gao --- src/plugins/kv/kv_engine.cpp | 5 ++++- src/plugins/kv/kv_engine.h | 16 +++++++++++++--- src/plugins/kv/kv_engine_impl.h | 6 +++--- src/plugins/kv/kv_store.h | 27 ++++++++++++++++++++++----- 4 files changed, 42 insertions(+), 12 deletions(-) diff --git a/src/plugins/kv/kv_engine.cpp b/src/plugins/kv/kv_engine.cpp index b086a8c2bb..aa6bd22c9c 100644 --- a/src/plugins/kv/kv_engine.cpp +++ b/src/plugins/kv/kv_engine.cpp @@ -21,11 +21,14 @@ */ #include "kv_engine.h" +#include nixlKVEngine::nixlKVEngine(const nixlBackendInitParams *init_params, std::unique_ptr impl) : nixlBackendEngine(init_params), - impl_(std::move(impl)) {} + impl_(std::move(impl)) { + assert(impl_ && "nixlKVEngine requires a non-null nixlKVEngineImpl"); +} nixlKVEngine::~nixlKVEngine() = default; diff --git a/src/plugins/kv/kv_engine.h b/src/plugins/kv/kv_engine.h index 9765f9bc2e..6de1280ff4 100644 --- a/src/plugins/kv/kv_engine.h +++ b/src/plugins/kv/kv_engine.h @@ -26,8 +26,8 @@ * register the resulting engine with nixlBackendPluginCreator. */ -#ifndef NIXL_KV_ENGINE_H -#define NIXL_KV_ENGINE_H +#ifndef NIXL_SRC_PLUGINS_KV_KV_ENGINE_H +#define NIXL_SRC_PLUGINS_KV_KV_ENGINE_H #include "kv_engine_impl.h" #include @@ -92,8 +92,18 @@ class nixlKVEngine : public nixlBackendEngine { return NIXL_SUCCESS; } + /** + * @brief Passes local metadata through unchanged. + * + * Local-only KV backends do not transform metadata. input must be the + * nixlBackendMD* returned by registerMem(). + */ nixl_status_t loadLocalMD(nixlBackendMD *input, nixlBackendMD *&output) override { + if (input == nullptr) { + output = nullptr; + return NIXL_ERR_INVALID_PARAM; + } output = input; return NIXL_SUCCESS; } @@ -124,4 +134,4 @@ class nixlKVEngine : public nixlBackendEngine { std::unique_ptr impl_; }; -#endif // NIXL_KV_ENGINE_H +#endif // NIXL_SRC_PLUGINS_KV_KV_ENGINE_H diff --git a/src/plugins/kv/kv_engine_impl.h b/src/plugins/kv/kv_engine_impl.h index 0891ed8fd1..4dfd4c30a6 100644 --- a/src/plugins/kv/kv_engine_impl.h +++ b/src/plugins/kv/kv_engine_impl.h @@ -35,8 +35,8 @@ * on NIXL descriptor/key mapping while iKVStore handles put/get/exists. */ -#ifndef NIXL_KV_ENGINE_IMPL_H -#define NIXL_KV_ENGINE_IMPL_H +#ifndef NIXL_SRC_PLUGINS_KV_KV_ENGINE_IMPL_H +#define NIXL_SRC_PLUGINS_KV_KV_ENGINE_IMPL_H #include "backend/backend_engine.h" #include @@ -97,4 +97,4 @@ class nixlKVEngineImpl { releaseReqH(nixlBackendReqH *handle) const = 0; }; -#endif // NIXL_KV_ENGINE_IMPL_H +#endif // NIXL_SRC_PLUGINS_KV_KV_ENGINE_IMPL_H diff --git a/src/plugins/kv/kv_store.h b/src/plugins/kv/kv_store.h index 003f83194f..884579951e 100644 --- a/src/plugins/kv/kv_store.h +++ b/src/plugins/kv/kv_store.h @@ -23,8 +23,8 @@ * backend-specific storage while sharing this synchronous put/get/exists API. */ -#ifndef NIXL_KV_STORE_H -#define NIXL_KV_STORE_H +#ifndef NIXL_SRC_PLUGINS_KV_KV_STORE_H +#define NIXL_SRC_PLUGINS_KV_KV_STORE_H #include "nixl_types.h" #include @@ -41,20 +41,37 @@ class iKVStore { public: virtual ~iKVStore() = default; + /** + * @brief Stores value bytes for key. + * + * @param key Storage key. + * @param data Pointer to value bytes to store. + * @param len Number of bytes to copy from data. + * @return NIXL_SUCCESS on success, or a backend error code on failure. + */ virtual nixl_status_t put(std::string_view key, const uint8_t *data, size_t len) = 0; /** * @brief Reads value for key into buffer. * - * bytes_read is set to min(stored_len, len) on success. - * Returns NIXL_ERR_BACKEND when key does not exist. + * @param key Storage key. + * @param buffer Output buffer for stored value bytes. + * @param len Maximum number of bytes to read into buffer. + * @param bytes_read Set to min(stored_len, len) on success. + * @return NIXL_SUCCESS on success, or NIXL_ERR_BACKEND when key does not exist. */ virtual nixl_status_t get(std::string_view key, uint8_t *buffer, size_t len, size_t &bytes_read) const = 0; + /** + * @brief Checks whether key is present in the store. + * + * @param key Storage key. + * @return true if key exists, false otherwise. + */ virtual bool exists(std::string_view key) const = 0; }; -#endif // NIXL_KV_STORE_H +#endif // NIXL_SRC_PLUGINS_KV_KV_STORE_H From a2eb4f768a54f0269192998ccb143987d122b22b Mon Sep 17 00:00:00 2001 From: Wayne Gao Date: Sat, 13 Jun 2026 09:51:29 +0800 Subject: [PATCH 03/24] fix(kv): document NIXL_ERR_NOT_FOUND for missing key in get() Signed-off-by: Wayne Gao --- src/plugins/kv/kv_store.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/kv/kv_store.h b/src/plugins/kv/kv_store.h index 884579951e..cfb87bc43b 100644 --- a/src/plugins/kv/kv_store.h +++ b/src/plugins/kv/kv_store.h @@ -59,7 +59,7 @@ class iKVStore { * @param buffer Output buffer for stored value bytes. * @param len Maximum number of bytes to read into buffer. * @param bytes_read Set to min(stored_len, len) on success. - * @return NIXL_SUCCESS on success, or NIXL_ERR_BACKEND when key does not exist. + * @return NIXL_SUCCESS on success, or NIXL_ERR_NOT_FOUND when key does not exist. */ virtual nixl_status_t get(std::string_view key, uint8_t *buffer, size_t len, size_t &bytes_read) const = 0; From 75d1190e9314646b8a1afa5b512971a993f3426a Mon Sep 17 00:00:00 2001 From: Wayne Gao Date: Sat, 13 Jun 2026 09:56:50 +0800 Subject: [PATCH 04/24] fix(kv): add local_agent to postXfer for prep/post symmetry Signed-off-by: Wayne Gao --- src/plugins/kv/kv_engine.cpp | 2 +- src/plugins/kv/kv_engine_impl.h | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/plugins/kv/kv_engine.cpp b/src/plugins/kv/kv_engine.cpp index aa6bd22c9c..12191e8da8 100644 --- a/src/plugins/kv/kv_engine.cpp +++ b/src/plugins/kv/kv_engine.cpp @@ -71,7 +71,7 @@ nixlKVEngine::postXfer(const nixl_xfer_op_t &operation, const std::string &remote_agent, nixlBackendReqH *&handle, const nixl_opt_b_args_t *opt_args) const { - return impl_->postXfer(operation, local, remote, remote_agent, handle, opt_args); + return impl_->postXfer(operation, local, remote, remote_agent, localAgent, handle, opt_args); } nixl_status_t diff --git a/src/plugins/kv/kv_engine_impl.h b/src/plugins/kv/kv_engine_impl.h index 4dfd4c30a6..48f33d81d7 100644 --- a/src/plugins/kv/kv_engine_impl.h +++ b/src/plugins/kv/kv_engine_impl.h @@ -82,11 +82,18 @@ class nixlKVEngineImpl { nixlBackendReqH *&handle, const nixl_opt_b_args_t *opt_args) const = 0; + /** + * @brief Execute a prepared transfer request. + * + * @param local_agent Agent name from nixlKVEngine::localAgent (passed explicitly + * so impl does not depend on nixlBackendEngine protected members). + */ virtual nixl_status_t postXfer(const nixl_xfer_op_t &operation, const nixl_meta_dlist_t &local, const nixl_meta_dlist_t &remote, const std::string &remote_agent, + const std::string &local_agent, nixlBackendReqH *&handle, const nixl_opt_b_args_t *opt_args) const = 0; From a0064ba6453b164d2d85ba33d714f52f6ec31260 Mon Sep 17 00:00:00 2001 From: Wayne Gao Date: Sat, 13 Jun 2026 10:08:00 +0800 Subject: [PATCH 05/24] docs(kv): add Doxygen for nixlKVEngineImpl public APIs Signed-off-by: Wayne Gao --- src/plugins/kv/kv_engine_impl.h | 59 ++++++++++++++++++++++++++++++--- 1 file changed, 54 insertions(+), 5 deletions(-) diff --git a/src/plugins/kv/kv_engine_impl.h b/src/plugins/kv/kv_engine_impl.h index 48f33d81d7..93ec9d3281 100644 --- a/src/plugins/kv/kv_engine_impl.h +++ b/src/plugins/kv/kv_engine_impl.h @@ -54,24 +54,54 @@ class nixlKVEngineImpl { public: virtual ~nixlKVEngineImpl() = default; - /** @return Memory segment types supported by this KV backend (typically DRAM_SEG). */ + /** @brief Memory segment types supported by this KV backend. @return Segment list (typically DRAM_SEG). */ virtual nixl_mem_list_t getSupportedMems() const = 0; + /** + * @brief Registers a local memory descriptor as a KV key. + * + * @param mem Descriptor blob including key metadata (metaInfo or devId). + * @param nixl_mem Memory segment type; must be supported by getSupportedMems(). + * @param out On success, receives a newly allocated nixlBackendMD* owned by the caller + * until deregisterMem(). + * @return NIXL_SUCCESS on success, or an error status on invalid input or backend failure. + */ virtual nixl_status_t registerMem(const nixlBlobDesc &mem, const nixl_mem_t &nixl_mem, nixlBackendMD *&out) = 0; + /** + * @brief Deregisters backend metadata created by registerMem(). + * + * @param meta Metadata pointer returned by registerMem(); must be non-null. + * @return NIXL_SUCCESS on success, or an error status if meta is invalid. + */ virtual nixl_status_t deregisterMem(nixlBackendMD *meta) = 0; + /** + * @brief Queries whether registered keys exist in the backing store. + * + * @param descs Registered descriptors to query. + * @param resp Output vector; each entry is set when the key exists, or std::nullopt otherwise. + * @return NIXL_SUCCESS on success, or an error status on backend failure. + */ virtual nixl_status_t queryMem(const nixl_reg_dlist_t &descs, std::vector &resp) const = 0; /** - * @brief Prepare a transfer request. + * @brief Prepares a transfer request and allocates a backend request handle. * - * @param local_agent Agent name from nixlKVEngine::localAgent (passed explicitly + * @param operation Transfer operation (NIXL_WRITE or NIXL_READ). + * @param local Local descriptor metadata list. + * @param remote Remote descriptor metadata list. + * @param remote_agent Remote agent name (unused for local-only KV backends). + * @param local_agent Local agent name from nixlKVEngine::localAgent (passed explicitly * so impl does not depend on nixlBackendEngine protected members). + * @param handle On success, receives a newly allocated nixlBackendReqH* owned by the caller + * until releaseReqH(). + * @param opt_args Optional backend arguments; may be nullptr. + * @return NIXL_SUCCESS on success, or an error status on invalid parameters. */ virtual nixl_status_t prepXfer(const nixl_xfer_op_t &operation, @@ -83,10 +113,17 @@ class nixlKVEngineImpl { const nixl_opt_b_args_t *opt_args) const = 0; /** - * @brief Execute a prepared transfer request. + * @brief Executes a prepared transfer request. * - * @param local_agent Agent name from nixlKVEngine::localAgent (passed explicitly + * @param operation Transfer operation (NIXL_WRITE or NIXL_READ). + * @param local Local descriptor metadata list. + * @param remote Remote descriptor metadata list. + * @param remote_agent Remote agent name (unused for local-only KV backends). + * @param local_agent Local agent name from nixlKVEngine::localAgent (passed explicitly * so impl does not depend on nixlBackendEngine protected members). + * @param handle Request handle allocated by prepXfer(); must be non-null. + * @param opt_args Optional backend arguments; may be nullptr. + * @return NIXL_SUCCESS on success, or an error status on transfer or backend failure. */ virtual nixl_status_t postXfer(const nixl_xfer_op_t &operation, @@ -97,9 +134,21 @@ class nixlKVEngineImpl { nixlBackendReqH *&handle, const nixl_opt_b_args_t *opt_args) const = 0; + /** + * @brief Checks transfer completion status for a request handle. + * + * @param handle Request handle from prepXfer(); must be non-null. + * @return NIXL_SUCCESS when the transfer is complete, or an in-progress/error status. + */ virtual nixl_status_t checkXfer(nixlBackendReqH *handle) const = 0; + /** + * @brief Releases a request handle allocated by prepXfer(). + * + * @param handle Request handle to release; must be non-null. + * @return NIXL_SUCCESS on success. + */ virtual nixl_status_t releaseReqH(nixlBackendReqH *handle) const = 0; }; From 45c71dd56fabbc1344659c0eada4efe634370766 Mon Sep 17 00:00:00 2001 From: Wayne Gao Date: Sat, 13 Jun 2026 10:23:38 +0800 Subject: [PATCH 06/24] fix(kv): wrap getSupportedMems Doxygen comment for clang-format Signed-off-by: Wayne Gao --- src/plugins/kv/kv_engine_impl.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/plugins/kv/kv_engine_impl.h b/src/plugins/kv/kv_engine_impl.h index 93ec9d3281..d4f26e80cd 100644 --- a/src/plugins/kv/kv_engine_impl.h +++ b/src/plugins/kv/kv_engine_impl.h @@ -54,7 +54,8 @@ class nixlKVEngineImpl { public: virtual ~nixlKVEngineImpl() = default; - /** @brief Memory segment types supported by this KV backend. @return Segment list (typically DRAM_SEG). */ + /** @brief Memory segment types supported by this KV backend. @return Segment list (typically + * DRAM_SEG). */ virtual nixl_mem_list_t getSupportedMems() const = 0; From 327e9fd6d3c6276d729adbdaa5e4ed2b1119c4a8 Mon Sep 17 00:00:00 2001 From: Wayne Gao Date: Mon, 15 Jun 2026 17:16:32 +0800 Subject: [PATCH 07/24] feat(kv): add REDIS plugin with async SET/GET and nixlbench support Introduce the REDIS KV backend under src/plugins/kv/redis/, built on the shared nixlKVEngine/nixlKVEngineImpl layer. Transfer uses hiredis-async (SET/GET via std::promise for postXfer/checkXfer); queryMem uses a separate synchronous hiredis connection for EXISTS. - Add iRedisClient, hiredisAsyncClient, and nixlRedisKVEngineImpl - Register static/shared REDIS plugin; wire meson and plugin manager - Extend nixlbench with REDIS backend, putRedis seeding, and --redis-only - Update Dockerfile/build.sh for libhiredis/libevent; add plugin README Signed-off-by: Wayne Gao --- benchmark/nixlbench/contrib/Dockerfile | 11 +- benchmark/nixlbench/contrib/build.sh | 4 + benchmark/nixlbench/src/utils/meson.build | 3 +- benchmark/nixlbench/src/utils/utils.cpp | 83 +++- benchmark/nixlbench/src/utils/utils.h | 3 + .../nixlbench/src/worker/nixl/nixl_worker.cpp | 40 ++ meson.build | 2 +- src/core/meson.build | 4 + src/core/nixl_plugin_manager.cpp | 4 + src/plugins/kv/kv_engine_impl.h | 14 +- src/plugins/kv/kv_store.h | 8 +- src/plugins/kv/meson.build | 21 + src/plugins/kv/redis/README.md | 176 +++++++ src/plugins/kv/redis/client.cpp | 458 ++++++++++++++++++ src/plugins/kv/redis/client.h | 122 +++++ src/plugins/kv/redis/engine_impl.cpp | 303 ++++++++++++ src/plugins/kv/redis/engine_impl.h | 90 ++++ src/plugins/kv/redis/meson.build | 122 +++++ src/plugins/kv/redis/redis_engine.cpp | 37 ++ src/plugins/kv/redis/redis_engine.h | 114 +++++ src/plugins/kv/redis/redis_executor.h | 123 +++++ src/plugins/kv/redis/redis_plugin.cpp | 45 ++ src/plugins/meson.build | 4 + 23 files changed, 1775 insertions(+), 16 deletions(-) create mode 100644 src/plugins/kv/meson.build create mode 100644 src/plugins/kv/redis/README.md create mode 100644 src/plugins/kv/redis/client.cpp create mode 100644 src/plugins/kv/redis/client.h create mode 100644 src/plugins/kv/redis/engine_impl.cpp create mode 100644 src/plugins/kv/redis/engine_impl.h create mode 100644 src/plugins/kv/redis/meson.build create mode 100644 src/plugins/kv/redis/redis_engine.cpp create mode 100644 src/plugins/kv/redis/redis_engine.h create mode 100644 src/plugins/kv/redis/redis_executor.h create mode 100644 src/plugins/kv/redis/redis_plugin.cpp diff --git a/benchmark/nixlbench/contrib/Dockerfile b/benchmark/nixlbench/contrib/Dockerfile index d826b7f04d..4823b1fa04 100644 --- a/benchmark/nixlbench/contrib/Dockerfile +++ b/benchmark/nixlbench/contrib/Dockerfile @@ -47,7 +47,10 @@ RUN apt-get update -y && \ libgtest-dev \ hwloc \ libhwloc-dev \ - build-essential + build-essential \ + libhiredis-dev \ + libevent-dev \ + redis-tools # Add DOCA repo + upgrade always so the RDMA reinstall below resolves MLNX/DOCA versions. # Skip only the DOCA SDK install when the base image already ships it. @@ -123,6 +126,8 @@ ARG LIBFABRIC_VERSION="v1.21.0" ARG NPROC ARG ABSL_TAG="lts_2025_08_14" ARG GRPC_TAG="v1.73.0" +ARG NIXL_ENABLE_PLUGINS="REDIS" +ARG NIXL_STATIC_PLUGINS="REDIS" WORKDIR /workspace @@ -258,7 +263,9 @@ WORKDIR /workspace/nixl RUN rm -rf build && \ mkdir build && \ - meson setup build --prefix=/usr/local/nixl --buildtype=$BUILD_TYPE && \ + meson setup build --prefix=/usr/local/nixl --buildtype=$BUILD_TYPE \ + -Denable_plugins=${NIXL_ENABLE_PLUGINS} \ + -Dstatic_plugins=${NIXL_STATIC_PLUGINS} && \ cd build && \ ninja -j${NPROC:-$(nproc)} && \ ninja install diff --git a/benchmark/nixlbench/contrib/build.sh b/benchmark/nixlbench/contrib/build.sh index dbd2eea339..cec9d4b682 100755 --- a/benchmark/nixlbench/contrib/build.sh +++ b/benchmark/nixlbench/contrib/build.sh @@ -129,6 +129,9 @@ get_options() { missing_requirement $1 fi ;; + --redis-only) + BUILD_ARGS+=" --build-arg NIXL_ENABLE_PLUGINS=REDIS --build-arg NIXL_STATIC_PLUGINS=REDIS" + ;; --) shift break @@ -185,6 +188,7 @@ show_help() { echo " [--python-versions python versions to build for, comma separated]" echo " [--tag tag for image]" echo " [--arch [x86_64|aarch64] to select target architecture]" + echo " [--redis-only build NIXL with REDIS plugin only (static)]" exit 0 } diff --git a/benchmark/nixlbench/src/utils/meson.build b/benchmark/nixlbench/src/utils/meson.build index 3bb017ba0b..c8157e6a8b 100644 --- a/benchmark/nixlbench/src/utils/meson.build +++ b/benchmark/nixlbench/src/utils/meson.build @@ -25,7 +25,8 @@ utils_deps = [ openmp_dep, gflags_dep, tomlplusplus_dep, - dependency('dl', required: true) + dependency('dl', required: true), + dependency('hiredis', required: true), ] if cuda_available diff --git a/benchmark/nixlbench/src/utils/utils.cpp b/benchmark/nixlbench/src/utils/utils.cpp index 32942ea0b8..092454573d 100644 --- a/benchmark/nixlbench/src/utils/utils.cpp +++ b/benchmark/nixlbench/src/utils/utils.cpp @@ -35,6 +35,7 @@ #include #include #include +#include #include "runtime/etcd/etcd_rt.h" #include "utils/neuron.h" @@ -63,7 +64,7 @@ NB_ARG_STRING(worker_type, XFERBENCH_WORKER_NIXL, "Type of worker [nixl, nvshmem NB_ARG_STRING(backend, XFERBENCH_BACKEND_UCX, "Name of NIXL backend [UCX, GDS, GDS_MT, POSIX, GPUNETIO, Mooncake, HF3FS, OBJ, " - "GUSLI, AZURE_BLOB] (only used with nixl worker)"); + "REDIS, GUSLI, AZURE_BLOB] (only used with nixl worker)"); NB_ARG_STRING(initiator_seg_type, XFERBENCH_SEG_TYPE_DRAM, "Type of memory segment for initiator [DRAM, VRAM]. Note: Storage backends always " @@ -679,7 +680,7 @@ xferBenchConfig::printConfig() { } printOption("Worker type (--worker_type=[nixl,nvshmem])", worker_type); if (worker_type == XFERBENCH_WORKER_NIXL) { - printOption("Backend (--backend=[UCX,GDS,GDS_MT,POSIX,Mooncake,HF3FS,OBJ,AZURE_BLOB])", + printOption("Backend (--backend=[UCX,GDS,GDS_MT,POSIX,Mooncake,HF3FS,OBJ,REDIS,AZURE_BLOB])", backend); printOption("Enable pt (--enable_pt=[0,1])", std::to_string(enable_pt)); printOption("Progress threads (--progress_threads=N)", std::to_string(progress_threads)); @@ -823,6 +824,7 @@ xferBenchConfig::isStorageBackend() { XFERBENCH_BACKEND_HF3FS == xferBenchConfig::backend || XFERBENCH_BACKEND_POSIX == xferBenchConfig::backend || XFERBENCH_BACKEND_OBJ == xferBenchConfig::backend || + XFERBENCH_BACKEND_REDIS == xferBenchConfig::backend || XFERBENCH_BACKEND_GUSLI == xferBenchConfig::backend || XFERBENCH_BACKEND_AZURE_BLOB == xferBenchConfig::backend || XFERBENCH_BACKEND_INFINIA == xferBenchConfig::backend); @@ -1303,6 +1305,83 @@ xferBenchUtils::buildAwsCredentials() { return env_setup; } +bool +xferBenchUtils::putRedis(size_t buffer_size, const std::string &key) { + if (buffer_size == 0) { + std::cerr << "Invalid Redis seed size: 0" << std::endl; + return false; + } + + const char *host = getenv("REDIS_HOST"); + if (!host) { + host = "127.0.0.1"; + } + int port = 6379; + const char *port_env = getenv("REDIS_PORT"); + if (port_env) { + port = std::atoi(port_env); + } + + void *buf = nullptr; + const size_t align = xferBenchConfig::page_size > 0 ? xferBenchConfig::page_size : 4096; + if (posix_memalign(&buf, align, buffer_size) != 0) { + std::cerr << "Failed to allocate buffer for Redis seed data" << std::endl; + return false; + } + memset(buf, XFERBENCH_TARGET_BUFFER_ELEMENT, buffer_size); + + redisContext *ctx = redisConnect(host, port); + if (!ctx || ctx->err) { + std::cerr << "Redis connect failed for seed SET: " + << (ctx ? ctx->errstr : "null context") << std::endl; + if (ctx) { + redisFree(ctx); + } + free(buf); + return false; + } + + const char *password = getenv("REDIS_PASSWORD"); + if (password && password[0] != '\0') { + redisReply *auth_reply = + (redisReply *)redisCommand(ctx, "AUTH %s", password); + if (!auth_reply || auth_reply->type == REDIS_REPLY_ERROR) { + std::cerr << "Redis AUTH failed during seed SET" << std::endl; + if (auth_reply) { + freeReplyObject(auth_reply); + } + redisFree(ctx); + free(buf); + return false; + } + freeReplyObject(auth_reply); + } + + redisReply *reply = (redisReply *)redisCommand(ctx, + "SET %b %b", + key.data(), + (size_t)key.size(), + buf, + (size_t)buffer_size); + free(buf); + + bool ok = false; + if (reply && (reply->type == REDIS_REPLY_STATUS || reply->type == REDIS_REPLY_STRING)) { + ok = true; + std::cout << "Seeded Redis key for READ: " << key << std::endl; + } else if (reply && reply->type == REDIS_REPLY_ERROR) { + std::cerr << "Redis SET failed for key " << key << ": " << reply->str << std::endl; + } else { + std::cerr << "Redis SET failed for key " << key << std::endl; + } + + if (reply) { + freeReplyObject(reply); + } + redisFree(ctx); + return ok; +} + bool xferBenchUtils::putObj(size_t buffer_size, const std::string &name) { if (xferBenchConfig::backend == XFERBENCH_BACKEND_INFINIA) { diff --git a/benchmark/nixlbench/src/utils/utils.h b/benchmark/nixlbench/src/utils/utils.h index a0ed72ece8..4dca773b35 100644 --- a/benchmark/nixlbench/src/utils/utils.h +++ b/benchmark/nixlbench/src/utils/utils.h @@ -94,6 +94,7 @@ #define XFERBENCH_BACKEND_MOONCAKE "Mooncake" #define XFERBENCH_BACKEND_HF3FS "HF3FS" #define XFERBENCH_BACKEND_OBJ "OBJ" +#define XFERBENCH_BACKEND_REDIS "REDIS" #define XFERBENCH_BACKEND_GUSLI "GUSLI" #define XFERBENCH_BACKEND_UCCL "UCCL" #define XFERBENCH_BACKEND_AZURE_BLOB "AZURE_BLOB" @@ -376,6 +377,8 @@ class xferBenchUtils { static bool putObj(size_t buffer_size, const std::string &name); static bool + putRedis(size_t buffer_size, const std::string &key); + static bool getObj(const std::string &name); static bool rmObj(const std::string &name); diff --git a/benchmark/nixlbench/src/worker/nixl/nixl_worker.cpp b/benchmark/nixlbench/src/worker/nixl/nixl_worker.cpp index 8b88a320db..7860f19053 100644 --- a/benchmark/nixlbench/src/worker/nixl/nixl_worker.cpp +++ b/benchmark/nixlbench/src/worker/nixl/nixl_worker.cpp @@ -263,6 +263,10 @@ xferBenchNixlWorker::xferBenchNixlWorker(const std::vector &devices } else { std::cout << "OBJ backend with standard S3 enabled" << std::endl; } + } else if (0 == xferBenchConfig::backend.compare(XFERBENCH_BACKEND_REDIS)) { + // REDIS backend: host/port/password via REDIS_HOST, REDIS_PORT, REDIS_PASSWORD + std::cout << "REDIS backend configured (using defaults or environment variables)" + << std::endl; } else if (0 == xferBenchConfig::backend.compare(XFERBENCH_BACKEND_GUSLI)) { // GUSLI backend requires direct I/O - enable it automatically if (!xferBenchConfig::storage_enable_direct) { @@ -1021,6 +1025,33 @@ xferBenchNixlWorker::allocateMemory(int num_threads) { CHECK_NIXL_ERROR(agent->registerMem(desc_list, &opt_args), "registerMem failed"); remote_regs_.emplace_back(*agent, backend_engine, BLK_SEG, std::move(iov_list)); } + } else if (xferBenchConfig::backend == XFERBENCH_BACKEND_REDIS) { + struct timeval tv; + gettimeofday(&tv, nullptr); + uint64_t timestamp = tv.tv_sec * 1000000ULL + tv.tv_usec; + + for (int list_idx = 0; list_idx < num_threads; list_idx++) { + std::vector iov_list; + for (i = 0; i < num_devices; i++) { + std::string unique_name = "nixlbench_redis" + std::to_string(list_idx) + "_" + + std::to_string(i) + "_" + std::to_string(timestamp); + + if (xferBenchConfig::op_type == XFERBENCH_OP_READ) { + const size_t seed_size = xferBenchConfig::max_block_size; + if (!xferBenchUtils::putRedis(seed_size, unique_name)) { + std::cerr << "Failed to seed Redis key: " << unique_name << std::endl; + continue; + } + } + + xferBenchIOV redis_desc(0, buffer_size, i, unique_name); + std::cout << "Creating Redis key: " << unique_name << std::endl; + iov_list.push_back(redis_desc); + } + nixl_reg_dlist_t desc_list = iovListToNixlRegDlist(iov_list, DRAM_SEG); + CHECK_NIXL_ERROR(agent->registerMem(desc_list, &opt_args), "registerMem failed"); + remote_regs_.emplace_back(*agent, backend_engine, DRAM_SEG, std::move(iov_list)); + } } else if (xferBenchConfig::isStorageBackend()) { int num_buffers = num_threads * num_devices; int num_files = xferBenchConfig::num_files; @@ -1227,6 +1258,11 @@ xferBenchNixlWorker::exchangeIOV(const std::vector> &l if (basic_desc) { remote_iov_list.push_back(basic_desc.value()); } + } else if (XFERBENCH_BACKEND_REDIS == xferBenchConfig::backend) { + xferBenchIOV redis_remote(iov); + redis_remote.addr = 0; + redis_remote.metaInfo = iov.metaInfo; + remote_iov_list.push_back(redis_remote); } else if (XFERBENCH_BACKEND_GUSLI == xferBenchConfig::backend) { xferBenchIOV iov_remote(iov); iov_remote.addr = gusli_devices[devidx].dev_offset + file_offset; @@ -1316,6 +1352,8 @@ prepareTransferDescriptors(nixl_xfer_dlist_t &local_desc, // Set remote descriptor type based on backend if (xferBenchConfig::isObjStorageBackend()) { remote_desc = nixl_xfer_dlist_t(OBJ_SEG); + } else if (XFERBENCH_BACKEND_REDIS == xferBenchConfig::backend) { + remote_desc = nixl_xfer_dlist_t(DRAM_SEG); } else if (XFERBENCH_BACKEND_GUSLI == xferBenchConfig::backend) { remote_desc = nixl_xfer_dlist_t(BLK_SEG); } else if (xferBenchConfig::isStorageBackend()) { @@ -1330,6 +1368,8 @@ static nixl_mem_t getRemoteSegType() { if (xferBenchConfig::isObjStorageBackend()) { return OBJ_SEG; + } else if (XFERBENCH_BACKEND_REDIS == xferBenchConfig::backend) { + return DRAM_SEG; } else if (XFERBENCH_BACKEND_GUSLI == xferBenchConfig::backend) { return BLK_SEG; } else if (xferBenchConfig::isStorageBackend()) { diff --git a/meson.build b/meson.build index 4cb898abc2..7d52019ac0 100644 --- a/meson.build +++ b/meson.build @@ -29,7 +29,7 @@ if enable_plugins_opt != '' and disable_plugins_opt != '' error('Cannot specify both enable_plugins and disable_plugins options') endif -all_plugins = ['UCX', 'LIBFABRIC', 'POSIX', 'OBJ', 'GDS', 'GDS_MT', 'MOONCAKE', 'HF3FS', 'GUSLI', 'GPUNETIO', 'UCCL', 'AZURE_BLOB', 'INFINIA', 'TELEMETRY_DOCA'] +all_plugins = ['UCX', 'LIBFABRIC', 'POSIX', 'OBJ', 'REDIS', 'GDS', 'GDS_MT', 'MOONCAKE', 'HF3FS', 'GUSLI', 'GPUNETIO', 'UCCL', 'AZURE_BLOB', 'INFINIA', 'TELEMETRY_DOCA'] enabled_plugins = {} diff --git a/src/core/meson.build b/src/core/meson.build index ee542bf94d..ec305afdc4 100644 --- a/src/core/meson.build +++ b/src/core/meson.build @@ -42,6 +42,10 @@ if 'OBJ' in static_plugins nixl_lib_deps += [ obj_backend_interface ] endif +if 'REDIS' in static_plugins + nixl_lib_deps += [ redis_backend_interface ] +endif + disable_gds_backend = get_option('disable_gds_backend') if not disable_gds_backend and 'GDS' in static_plugins nixl_lib_deps += [ gds_backend_interface, cuda_dep ] diff --git a/src/core/nixl_plugin_manager.cpp b/src/core/nixl_plugin_manager.cpp index 2a12915b33..3ce62748c3 100644 --- a/src/core/nixl_plugin_manager.cpp +++ b/src/core/nixl_plugin_manager.cpp @@ -847,6 +847,10 @@ void nixlPluginManager::registerBuiltinPlugins() { NIXL_REGISTER_STATIC_PLUGIN(Backend, OBJ) #endif +#ifdef STATIC_PLUGIN_REDIS + NIXL_REGISTER_STATIC_PLUGIN(Backend, REDIS) +#endif + #ifdef STATIC_PLUGIN_MOONCAKE NIXL_REGISTER_STATIC_PLUGIN(Backend, MOONCAKE) #endif diff --git a/src/plugins/kv/kv_engine_impl.h b/src/plugins/kv/kv_engine_impl.h index d4f26e80cd..cb6c378106 100644 --- a/src/plugins/kv/kv_engine_impl.h +++ b/src/plugins/kv/kv_engine_impl.h @@ -28,11 +28,11 @@ * nixlKVEngineImpl Vendor/backend-specific logic (this header) * ^ * | - * nixlInMemKVEngineImpl Example: in-process map via iKVStore - * nixlRedisKVEngineImpl Future: Redis via hiredis implementing iKVStore + * nixlInMemKVEngineImpl Example: in-process map via iKVStore (examples/plugins/inmemkv/) + * nixlRedisKVEngineImpl REDIS plugin: redis/engine_impl.* (under kv/redis/) * - * Storage is further factored through iKVStore (kv_store.h) so impl classes focus - * on NIXL descriptor/key mapping while iKVStore handles put/get/exists. + * Synchronous KV backends use iKVStore (kv_store.h). Redis is async and uses + * iRedisClient in kv/redis/redis_engine.h; hiredisAsyncClient in kv/redis/client.*. */ #ifndef NIXL_SRC_PLUGINS_KV_KV_ENGINE_IMPL_H @@ -46,9 +46,9 @@ * @class nixlKVEngineImpl * @brief Abstract implementation interface for KV-style backend engines. * - * Each KV plugin (INMEMKV example, REDIS src plugin, etc.) provides a concrete - * subclass that implements register/deregister, query, and synchronous transfer - * operations against its chosen iKVStore backend. + * Each KV plugin (INMEMKV example, REDIS, etc.) provides a concrete subclass + * that implements register/deregister, query, and transfer operations. Sync + * backends delegate storage to iKVStore; async backends (Redis) use iRedisClient. */ class nixlKVEngineImpl { public: diff --git a/src/plugins/kv/kv_store.h b/src/plugins/kv/kv_store.h index cfb87bc43b..34824d9308 100644 --- a/src/plugins/kv/kv_store.h +++ b/src/plugins/kv/kv_store.h @@ -19,8 +19,9 @@ * @file kv_store.h * @brief Shared minimal key-value storage interface for NIXL KV backends. * - * Implementations (e.g. INMEMKV example plugin, REDIS src plugin) provide - * backend-specific storage while sharing this synchronous put/get/exists API. + * Synchronous implementations (e.g. INMEMKV example plugin) provide + * backend-specific storage via this API. Async backends (REDIS) use iRedisClient + * in kv/redis/redis_engine.h instead — see kv_engine_impl.h. */ #ifndef NIXL_SRC_PLUGINS_KV_KV_STORE_H @@ -35,7 +36,8 @@ * @brief Minimal key-value storage interface for KV-style NIXL backends. * * This interface intentionally keeps only the smallest synchronous API - * needed by current KV backend flows (INMEMKV, REDIS, etc.). + * needed by current KV backend flows (e.g. INMEMKV). Async backends use + * iRedisClient instead — see kv/redis/redis_engine.h. */ class iKVStore { public: diff --git a/src/plugins/kv/meson.build b/src/plugins/kv/meson.build new file mode 100644 index 0000000000..43a7b27a3f --- /dev/null +++ b/src/plugins/kv/meson.build @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Shared KV headers (kv_engine, kv_engine_impl, kv_store) live in kv/. +# REDIS plugin sources are entirely under kv/redis/. + +if enabled_plugins.get('REDIS') + subdir('redis') +endif diff --git a/src/plugins/kv/redis/README.md b/src/plugins/kv/redis/README.md new file mode 100644 index 0000000000..71195624c8 --- /dev/null +++ b/src/plugins/kv/redis/README.md @@ -0,0 +1,176 @@ + + +# NIXL Redis Plugin (KV) + +All REDIS plugin code lives in `src/plugins/kv/redis/`. Shared KV abstractions stay in `src/plugins/kv/`. + +``` +kv/ # shared KV layer (PR1 interface) + kv_engine.h/.cpp nixlKVEngine + kv_engine_impl.h nixlKVEngineImpl + kv_store.h iKVStore (sync; INMEMKV only) + meson.build delegates to redis/ when REDIS enabled + +kv/redis/ # REDIS plugin (this directory) + redis_engine.h/.cpp iRedisClient + nixlRedisKVEngine + redis_plugin.cpp registers "REDIS" + redis_executor.h ASIO thread pool + client.h/.cpp hiredisAsyncClient + engine_impl.h/.cpp nixlRedisKVEngineImpl + meson.build + README.md +``` + +## Class hierarchy + +| Layer | REDIS plugin | +|-------|--------------| +| Shared KV wrapper | `nixlKVEngine` (`../kv_engine.h`) | +| Shared KV impl base | `nixlKVEngineImpl` (`../kv_engine_impl.h`) | +| Plugin entry | `nixlRedisKVEngine` (`redis_engine.h`) | +| Async storage interface | `iRedisClient` (`redis_engine.h`) | +| Client implementation | `hiredisAsyncClient` (`client.h`) | +| Engine implementation | `nixlRedisKVEngineImpl` (`engine_impl.h`) | + +## iKVStore vs iRedisClient + +| | `iKVStore` (`kv_store.h`) | `iRedisClient` (`redis_engine.h`) | +|--|---------------------------|-----------------------------------| +| Used by | INMEMKV example plugin | REDIS plugin | +| API style | Sync `put` / `get` / `exists` | Async SET/GET + sync EXISTS (separate connections) | +| `postXfer` | Returns `NIXL_SUCCESS` immediately | Returns `NIXL_IN_PROG`; `checkXfer` polls futures | +| Why separate | Blocking on hiredis event loop would deadlock | Async promise path for postXfer/checkXfer | + +## iRedisClient API + +```cpp +class iRedisClient { + void setExecutor(std::shared_ptr executor); + + // postXfer path (async connection) + void putKeyAsync(..., std::shared_ptr> promise); + void getKeyAsync(..., std::shared_ptr> promise); + + // queryMem path (sync connection) + std::optional checkKeyExistsSync(std::string_view key); +}; +``` + +Connection settings (constructor): `host`, `port`, `password`, `db` via NIXL custom params or env vars `REDIS_HOST`, `REDIS_PORT`, `REDIS_PASSWORD`. + +## Transfer call path + +``` +nixlbench / NIXL API + -> nixlRedisKVEngine::postXfer + -> nixlRedisKVEngineImpl::postXfer + creates std::promise per descriptor + -> iRedisClient::putKeyAsync / getKeyAsync (async connection) + -> hiredisAsyncClient (libevent thread + redisAsyncCommand) + promise completed on ASIO executor thread + -> checkXfer polls statusFutures_ + +queryMem (sync path): + -> nixlRedisKVEngineImpl::queryMem + -> iRedisClient::checkKeyExistsSync (separate blocking hiredis connection) + -> redisCommand EXISTS, returns before queryMem exits +``` + +## Build + +### Native (plugin only) + +```bash +meson setup build -Dplugins=REDIS +ninja -C build +# -> build/src/plugins/kv/redis/libplugin_REDIS.so +``` + +### Docker image (nixlbench + REDIS plugin) + +On the **host**, from `benchmark/nixlbench/contrib/`: + +```bash +cd benchmark/nixlbench/contrib + +# Build image; tag defaults to nixlbench:v.dev. +./build.sh --nixl --redis-only + +# Or pin an explicit tag: +./build.sh --nixl --redis-only --tag nixlbench:v0.1.0.dev.9da7bd1 +``` + +`--redis-only` passes `-Denable_plugins=REDIS -Dstatic_plugins=REDIS` into the image build. +The image installs `nixlbench` to `/usr/local/nixlbench/bin` and sets the default +working directory to `/workspace/nixl/benchmark/kvbench`. + +## Test (Docker container, step by step) + +This matches the workflow: build image on the host, then run `nixlbench` **inside** the container. + +### Step 1 — Start Redis on the host + +```bash +docker run -d --name nixlbench-redis -p 6379:6379 redis:7-alpine +``` + +Redis listens on host port `6379`. Use `--network host` when starting the bench container +(step 2) so `127.0.0.1:6379` inside the container reaches this Redis instance. + +### Step 2 — Start the nixlbench container + +```bash +docker run -it --rm --network host \ + nixlbench:v0.1.0.dev.9da7bd1 \ + bash +``` + +Replace the image tag with the one printed by `build.sh`. + +### Step 3 — Inside the container: set Redis env and run bench + +The image default working directory is `/workspace/nixl/benchmark/kvbench`. +`nixlbench` is already on `PATH`. + +```bash +cd /workspace/nixl/benchmark/kvbench +export REDIS_HOST=127.0.0.1 REDIS_PORT=6379 + +# READ — nixlbench seeds the Redis key automatically before the transfer loop +nixlbench --backend REDIS --start_block_size 4096 --max_block_size 4096 \ + --num_iter 50 --warmup_iter 5 --op_type READ + +# WRITE +nixlbench --backend REDIS --start_block_size 4096 --max_block_size 4096 \ + --num_iter 100 --warmup_iter 10 --op_type WRITE +``` + +Expected log lines: + +- `REDIS backend configured (using defaults or environment variables)` +- READ only: `Seeded Redis key for READ: nixlbench_redis0_0_...` +- Stats table with bandwidth and latency (example: ~0.03 GB/s, ~120 µs at 4096 B) + +**Notes:** + +- `REDIS_HOST` / `REDIS_PORT` / `REDIS_PASSWORD` are read by `hiredisAsyncClient` at plugin init. +- nixlbench may adjust `--num_iter` / `--warmup_iter` for thread alignment (warnings are normal). +- For a one-shot run without an interactive shell, pass `nixlbench ...` as the `docker run` command + instead of `bash`. + +### Step 4 — Native bench (no Docker) + +If `nixlbench` is built locally with REDIS support: + +```bash +export REDIS_HOST=127.0.0.1 REDIS_PORT=6379 + +nixlbench --backend REDIS --start_block_size 4096 --max_block_size 4096 \ + --num_iter 100 --warmup_iter 10 --op_type WRITE + +nixlbench --backend REDIS --start_block_size 4096 --max_block_size 4096 \ + --num_iter 50 --warmup_iter 5 --op_type READ +``` diff --git a/src/plugins/kv/redis/client.cpp b/src/plugins/kv/redis/client.cpp new file mode 100644 index 0000000000..dc88cc65d3 --- /dev/null +++ b/src/plugins/kv/redis/client.cpp @@ -0,0 +1,458 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "client.h" +#include "common/nixl_log.h" +#include +#include +#include +#include +#include + +#ifdef HAVE_HIREDIS_ASYNC + +namespace { + +std::string +getRedisHost(nixl_b_params_t *custom_params) { + if (custom_params && custom_params->count("host") > 0) { + return custom_params->at("host"); + } + const char *env_host = getenv("REDIS_HOST"); + return env_host ? std::string(env_host) : "localhost"; +} + +int +getRedisPort(nixl_b_params_t *custom_params) { + if (custom_params && custom_params->count("port") > 0) { + try { + return std::stoi(custom_params->at("port")); + } + catch (const std::exception &) { + NIXL_WARN << "Invalid port value, using default 6379"; + } + } + const char *env_port = getenv("REDIS_PORT"); + if (env_port) { + try { + return std::stoi(env_port); + } + catch (const std::exception &) { + NIXL_WARN << "Invalid REDIS_PORT, using default 6379"; + } + } + return 6379; +} + +std::string +getRedisPassword(nixl_b_params_t *custom_params) { + if (custom_params && custom_params->count("password") > 0) { + return custom_params->at("password"); + } + const char *env_password = getenv("REDIS_PASSWORD"); + return env_password ? std::string(env_password) : ""; +} + +int +getRedisDB(nixl_b_params_t *custom_params) { + if (custom_params && custom_params->count("db") > 0) { + try { + return std::stoi(custom_params->at("db")); + } + catch (const std::exception &) { + NIXL_WARN << "Invalid db value, using default 0"; + } + } + return 0; +} + +bool +checkRedisReplyOk(redisReply *reply, const char *command) { + if (!reply) { + NIXL_ERROR << absl::StrFormat("Redis %s: no reply", command); + return false; + } + if (reply->type == REDIS_REPLY_ERROR) { + NIXL_ERROR << absl::StrFormat("Redis %s error: %s", command, reply->str); + return false; + } + if (reply->type == REDIS_REPLY_STATUS && strcmp(reply->str, "OK") != 0) { + NIXL_ERROR << absl::StrFormat("Redis %s unexpected status: %s", command, reply->str); + return false; + } + return true; +} + +void +setPromiseStatus(const std::shared_ptr &executor, + const std::shared_ptr> &promise, + bool success) { + if (!promise) { + return; + } + if (executor) { + executor->post([promise, success]() { + promise->set_value(success ? NIXL_SUCCESS : NIXL_ERR_BACKEND); + }); + } else { + promise->set_value(success ? NIXL_SUCCESS : NIXL_ERR_BACKEND); + } +} + +} // namespace + +hiredisAsyncClient::hiredisAsyncClient(nixl_b_params_t *custom_params, + std::shared_ptr executor) + : host_(getRedisHost(custom_params)), + port_(getRedisPort(custom_params)), + password_(getRedisPassword(custom_params)), + db_(getRedisDB(custom_params)), + asyncContext_(nullptr), + syncContext_(nullptr), + eventBase_(nullptr), + executor_(executor), + connected_(false) { + evthread_use_pthreads(); + + eventBase_ = event_base_new(); + if (!eventBase_) { + throw std::runtime_error("Failed to create event base"); + } + + asyncContext_ = redisAsyncConnect(host_.c_str(), port_); + if (!asyncContext_ || asyncContext_->err) { + if (asyncContext_) { + std::string err_msg = + absl::StrFormat("Failed to connect to Redis: %s", asyncContext_->errstr); + redisAsyncFree(asyncContext_); + event_base_free(eventBase_); + throw std::runtime_error(err_msg); + } + event_base_free(eventBase_); + throw std::runtime_error("Failed to allocate Redis async context"); + } + + asyncContext_->data = this; + + if (redisLibeventAttach(asyncContext_, eventBase_) != REDIS_OK) { + std::string err_msg = + absl::StrFormat("Failed to attach Redis to event base: %s", asyncContext_->errstr); + redisAsyncFree(asyncContext_); + event_base_free(eventBase_); + throw std::runtime_error(err_msg); + } + + redisAsyncSetConnectCallback(asyncContext_, connectCallback); + redisAsyncSetDisconnectCallback(asyncContext_, disconnectCallback); + + if (!password_.empty()) { + redisAsyncCommand(asyncContext_, nullptr, nullptr, "AUTH %s", password_.c_str()); + } + if (db_ != 0) { + redisAsyncCommand(asyncContext_, nullptr, nullptr, "SELECT %d", db_); + } + + eventLoopThread_ = std::thread(&hiredisAsyncClient::processEventLoop, this); + + int retries = 100; + while (!connected_ && retries-- > 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + + if (!connected_) { + redisAsyncDisconnect(asyncContext_); + event_base_loopbreak(eventBase_); + if (eventLoopThread_.joinable()) { + eventLoopThread_.join(); + } + redisAsyncFree(asyncContext_); + event_base_free(eventBase_); + throw std::runtime_error("Failed to connect to Redis within timeout"); + } + + connectSyncContext(); + + NIXL_INFO << absl::StrFormat("Connected to Redis at %s:%d (db=%d)", host_, port_, db_); +} + +void +hiredisAsyncClient::connectSyncContext() { + struct timeval timeout = {5, 0}; + syncContext_ = redisConnectWithTimeout(host_.c_str(), port_, timeout); + if (!syncContext_ || syncContext_->err) { + std::string err_msg = syncContext_ ? syncContext_->errstr : "allocation failed"; + if (syncContext_) { + redisFree(syncContext_); + syncContext_ = nullptr; + } + NIXL_WARN << absl::StrFormat( + "Sync Redis connection for EXISTS failed (%s:%d): %s; queryMem will return errors", + host_, + port_, + err_msg); + return; + } + + if (!password_.empty()) { + redisReply *reply = + static_cast(redisCommand(syncContext_, "AUTH %s", password_.c_str())); + if (!checkRedisReplyOk(reply, "AUTH")) { + freeReplyObject(reply); + redisFree(syncContext_); + syncContext_ = nullptr; + NIXL_WARN << "Sync Redis AUTH failed; queryMem will return errors"; + return; + } + freeReplyObject(reply); + } + + if (db_ != 0) { + redisReply *reply = + static_cast(redisCommand(syncContext_, "SELECT %d", db_)); + if (!checkRedisReplyOk(reply, "SELECT")) { + freeReplyObject(reply); + redisFree(syncContext_); + syncContext_ = nullptr; + NIXL_WARN << "Sync Redis SELECT failed; queryMem will return errors"; + return; + } + freeReplyObject(reply); + } + + NIXL_INFO << absl::StrFormat( + "Sync Redis connection ready for EXISTS at %s:%d (db=%d)", host_, port_, db_); +} + +hiredisAsyncClient::~hiredisAsyncClient() { + if (asyncContext_) { + redisAsyncDisconnect(asyncContext_); + } + if (eventBase_) { + event_base_loopbreak(eventBase_); + if (eventLoopThread_.joinable()) { + eventLoopThread_.join(); + } + event_base_free(eventBase_); + } + if (asyncContext_) { + redisAsyncFree(asyncContext_); + } + if (syncContext_) { + redisFree(syncContext_); + } +} + +void +hiredisAsyncClient::setExecutor(std::shared_ptr executor) { + executor_ = executor; +} + +void +hiredisAsyncClient::processEventLoop() { + event_base_dispatch(eventBase_); +} + +void +hiredisAsyncClient::connectCallback(const redisAsyncContext *c, int status) { + auto *client = static_cast(c->data); + if (status != REDIS_OK) { + NIXL_ERROR << absl::StrFormat("Redis connection error: %s", c->errstr); + client->connected_ = false; + } else { + client->connected_ = true; + } +} + +void +hiredisAsyncClient::disconnectCallback(const redisAsyncContext *c, int status) { + auto *client = static_cast(c->data); + if (status != REDIS_OK) { + NIXL_WARN << absl::StrFormat("Redis disconnection error: %s", c->errstr); + } + client->connected_ = false; +} + +void +hiredisAsyncClient::setCallback(redisAsyncContext *c, void *reply, void *privdata) { + auto *ctx = static_cast(privdata); + auto *r = static_cast(reply); + + bool success = false; + if (r && r->type == REDIS_REPLY_STATUS) { + success = (strcmp(r->str, "OK") == 0); + } else if (r && r->type == REDIS_REPLY_ERROR) { + NIXL_ERROR << absl::StrFormat("Redis SET error: %s", r->str); + } + + auto promise_ptr = ctx->promise_ptr; + auto executor = ctx->executor; + delete ctx; + setPromiseStatus(executor, promise_ptr, success); +} + +void +hiredisAsyncClient::getCallback(redisAsyncContext *c, void *reply, void *privdata) { + auto *ctx = static_cast(privdata); + auto *r = static_cast(reply); + + bool success = false; + if (r && r->type == REDIS_REPLY_STRING) { + size_t copy_len = std::min(static_cast(r->len), ctx->data_len); + if (ctx->data_ptr && copy_len > 0) { + std::memcpy(reinterpret_cast(ctx->data_ptr), r->str, copy_len); + success = true; + } + } else if (r && r->type == REDIS_REPLY_NIL) { + NIXL_WARN << "Redis GET: key not found"; + } else if (r && r->type == REDIS_REPLY_ERROR) { + NIXL_ERROR << absl::StrFormat("Redis GET error: %s", r->str); + } + + auto promise_ptr = ctx->promise_ptr; + auto executor = ctx->executor; + delete ctx; + setPromiseStatus(executor, promise_ptr, success); +} + +void +hiredisAsyncClient::putKeyAsync(std::string_view key, + uintptr_t data_ptr, + size_t data_len, + std::shared_ptr> promise) { + if (!connected_) { + setPromiseStatus(executor_, promise, false); + return; + } + + auto *ctx = new CallbackContext; + ctx->executor = executor_; + ctx->promise_ptr = std::move(promise); + + int ret = redisAsyncCommand(asyncContext_, + setCallback, + ctx, + "SET %b %b", + key.data(), + key.size(), + reinterpret_cast(data_ptr), + data_len); + + if (ret != REDIS_OK) { + auto promise_ptr = ctx->promise_ptr; + delete ctx; + setPromiseStatus(executor_, promise_ptr, false); + } +} + +void +hiredisAsyncClient::getKeyAsync(std::string_view key, + uintptr_t data_ptr, + size_t data_len, + std::shared_ptr> promise) { + if (!connected_) { + setPromiseStatus(executor_, promise, false); + return; + } + + auto *ctx = new CallbackContext; + ctx->executor = executor_; + ctx->data_ptr = data_ptr; + ctx->data_len = data_len; + ctx->promise_ptr = std::move(promise); + + int ret = redisAsyncCommand( + asyncContext_, getCallback, ctx, "GET %b", key.data(), key.size()); + + if (ret != REDIS_OK) { + auto promise_ptr = ctx->promise_ptr; + delete ctx; + setPromiseStatus(executor_, promise_ptr, false); + } +} + +std::optional +hiredisAsyncClient::checkKeyExistsSync(std::string_view key) { + std::lock_guard lock(syncMutex_); + + if (!syncContext_ || syncContext_->err) { + NIXL_ERROR << "Sync Redis connection unavailable for EXISTS"; + return std::nullopt; + } + + redisReply *reply = static_cast( + redisCommand(syncContext_, "EXISTS %b", key.data(), key.size())); + + if (!reply) { + NIXL_ERROR << "Redis EXISTS: no reply"; + return std::nullopt; + } + + if (reply->type == REDIS_REPLY_ERROR) { + NIXL_ERROR << absl::StrFormat("Redis EXISTS error: %s", reply->str); + freeReplyObject(reply); + return std::nullopt; + } + + if (reply->type != REDIS_REPLY_INTEGER) { + NIXL_ERROR << absl::StrFormat("Redis EXISTS unexpected reply type: %d", reply->type); + freeReplyObject(reply); + return std::nullopt; + } + + bool exists = (reply->integer == 1); + freeReplyObject(reply); + return exists; +} + +#else // HAVE_HIREDIS_ASYNC + +hiredisAsyncClient::hiredisAsyncClient(nixl_b_params_t *custom_params, + std::shared_ptr executor) { + throw std::runtime_error("hiredis-async not available"); +} + +hiredisAsyncClient::~hiredisAsyncClient() {} + +void +hiredisAsyncClient::setExecutor(std::shared_ptr executor) {} + +void +hiredisAsyncClient::putKeyAsync(std::string_view key, + uintptr_t data_ptr, + size_t data_len, + std::shared_ptr> promise) { + if (promise) { + promise->set_value(NIXL_ERR_BACKEND); + } +} + +void +hiredisAsyncClient::getKeyAsync(std::string_view key, + uintptr_t data_ptr, + size_t data_len, + std::shared_ptr> promise) { + if (promise) { + promise->set_value(NIXL_ERR_BACKEND); + } +} + +std::optional +hiredisAsyncClient::checkKeyExistsSync(std::string_view key) { + return std::nullopt; +} + +#endif // HAVE_HIREDIS_ASYNC diff --git a/src/plugins/kv/redis/client.h b/src/plugins/kv/redis/client.h new file mode 100644 index 0000000000..baad2d7faa --- /dev/null +++ b/src/plugins/kv/redis/client.h @@ -0,0 +1,122 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @file client.h + * @brief hiredis Redis client implementing iRedisClient. + * + * Two connections: + * - async (hiredis-async + libevent): SET/GET for postXfer + * - sync (blocking hiredis): EXISTS for queryMem + */ + +#ifndef KV_PLUGIN_REDIS_CLIENT_H +#define KV_PLUGIN_REDIS_CLIENT_H + +#include "redis_engine.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include "nixl_types.h" + +#ifdef HAVE_HIREDIS_ASYNC +#include +#include +#include +#include +#include +#endif + +/** + * @class hiredisAsyncClient + * @brief iRedisClient with async SET/GET and a separate sync EXISTS connection. + */ +class hiredisAsyncClient : public iRedisClient { +public: + /** + * @brief Construct a client from NIXL custom params and environment variables. + * @param custom_params Optional host/port/password/db overrides. + * @param executor Executor used to complete async promises. + */ + hiredisAsyncClient(nixl_b_params_t *custom_params, + std::shared_ptr executor = nullptr); + + ~hiredisAsyncClient() override; + + void + setExecutor(std::shared_ptr executor) override; + + void + putKeyAsync(std::string_view key, + uintptr_t data_ptr, + size_t data_len, + std::shared_ptr> promise) override; + + void + getKeyAsync(std::string_view key, + uintptr_t data_ptr, + size_t data_len, + std::shared_ptr> promise) override; + + std::optional + checkKeyExistsSync(std::string_view key) override; + +private: + struct CallbackContext { + std::shared_ptr executor; + uintptr_t data_ptr; + size_t data_len; + std::shared_ptr> promise_ptr; + }; + + static void + connectCallback(const redisAsyncContext *c, int status); + + static void + disconnectCallback(const redisAsyncContext *c, int status); + + static void + setCallback(redisAsyncContext *c, void *reply, void *privdata); + + static void + getCallback(redisAsyncContext *c, void *reply, void *privdata); + + void + processEventLoop(); + + void + connectSyncContext(); + + std::string host_; + int port_; + std::string password_; + int db_; + redisAsyncContext *asyncContext_; + redisContext *syncContext_; + struct event_base *eventBase_; + std::shared_ptr executor_; + std::thread eventLoopThread_; + bool connected_; + mutable std::mutex syncMutex_; +}; + +#endif // KV_PLUGIN_REDIS_CLIENT_H diff --git a/src/plugins/kv/redis/engine_impl.cpp b/src/plugins/kv/redis/engine_impl.cpp new file mode 100644 index 0000000000..10cb0b23b8 --- /dev/null +++ b/src/plugins/kv/redis/engine_impl.cpp @@ -0,0 +1,303 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @file engine_impl.cpp + * @brief nixlRedisKVEngineImpl — postXfer/checkXfer over async Redis SET/GET. + */ + +#include "engine_impl.h" +#include "client.h" +#include "common/nixl_log.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +std::size_t +getNumThreads(nixl_b_params_t *custom_params) { + return custom_params && custom_params->count("num_threads") > 0 ? + std::stoul(custom_params->at("num_threads")) : + std::max(1u, std::thread::hardware_concurrency() / 2); +} + +bool +isValidPrepXferParams(const nixl_xfer_op_t &operation, + const nixl_meta_dlist_t &local, + const nixl_meta_dlist_t &remote, + const std::string &remote_agent, + const std::string &local_agent) { + if (operation != NIXL_WRITE && operation != NIXL_READ) { + NIXL_ERROR << absl::StrFormat("Error: Invalid operation type: %d", operation); + return false; + } + + if (remote_agent != local_agent) + NIXL_WARN << absl::StrFormat( + "Warning: Remote agent doesn't match the requesting agent (%s). Got %s", + local_agent, + remote_agent); + + if (local.getType() != DRAM_SEG) { + NIXL_ERROR << absl::StrFormat("Error: Local memory type must be DRAM_SEG, got %d", + local.getType()); + return false; + } + + if (remote.getType() != DRAM_SEG && remote.getType() != OBJ_SEG) { + NIXL_ERROR << absl::StrFormat("Error: Remote memory type must be DRAM_SEG or OBJ_SEG, got %d", + remote.getType()); + return false; + } + + return true; +} + +class nixlRedisBackendReqH : public nixlBackendReqH { +public: + std::vector>> statusPromises_; + std::vector> statusFutures_; + + nixl_status_t + getOverallStatus() { + while (!statusFutures_.empty()) { + if (statusFutures_.back().wait_for(std::chrono::seconds(0)) == + std::future_status::ready) { + auto current_status = statusFutures_.back().get(); + if (current_status != NIXL_SUCCESS) { + statusFutures_.clear(); + statusPromises_.clear(); + return current_status; + } + statusFutures_.pop_back(); + statusPromises_.pop_back(); + } else { + return NIXL_IN_PROG; + } + } + return NIXL_SUCCESS; + } +}; + +class nixlRedisMetadata : public nixlBackendMD { +public: + nixlRedisMetadata(nixl_mem_t nixl_mem, uint64_t dev_id, std::string redis_key) + : nixlBackendMD(true), + nixlMem(nixl_mem), + devId(dev_id), + redisKey(std::move(redis_key)) {} + + nixl_mem_t nixlMem; + uint64_t devId; + std::string redisKey; +}; + +} // namespace + +nixlRedisKVEngineImpl::nixlRedisKVEngineImpl(const nixlBackendInitParams *init_params) + : executor_(std::make_shared(getNumThreads(init_params->customParams))) { + redisClient_ = std::make_shared(init_params->customParams, executor_); + NIXL_INFO << "Redis KV backend initialized"; +} + +nixlRedisKVEngineImpl::nixlRedisKVEngineImpl(const nixlBackendInitParams *init_params, + std::shared_ptr redis_client) + : executor_(std::make_shared(std::thread::hardware_concurrency())), + redisClient_(std::move(redis_client)) { + if (redisClient_) { + redisClient_->setExecutor(executor_); + } +} + +nixlRedisKVEngineImpl::~nixlRedisKVEngineImpl() { + if (executor_) { + executor_->WaitUntilStopped(); + } +} + +nixl_status_t +nixlRedisKVEngineImpl::registerMem(const nixlBlobDesc &mem, + const nixl_mem_t &nixl_mem, + nixlBackendMD *&out) { + auto supported_mems = getSupportedMems(); + if (std::find(supported_mems.begin(), supported_mems.end(), nixl_mem) == supported_mems.end()) { + return NIXL_ERR_NOT_SUPPORTED; + } + + std::string redis_key = mem.metaInfo.empty() ? std::to_string(mem.devId) : mem.metaInfo; + + if (nixl_mem == OBJ_SEG || nixl_mem == DRAM_SEG) { + auto redis_md = std::make_unique(nixl_mem, mem.devId, redis_key); + devIdToRedisKey_[mem.devId] = redis_key; + out = redis_md.release(); + } else { + out = nullptr; + } + + return NIXL_SUCCESS; +} + +nixl_status_t +nixlRedisKVEngineImpl::deregisterMem(nixlBackendMD *meta) { + auto *redis_md = static_cast(meta); + if (redis_md) { + std::unique_ptr redis_md_ptr(redis_md); + devIdToRedisKey_.erase(redis_md->devId); + } + return NIXL_SUCCESS; +} + +nixl_status_t +nixlRedisKVEngineImpl::queryMem(const nixl_reg_dlist_t &descs, + std::vector &resp) const { + iRedisClient *client = getClient(); + if (!client) { + NIXL_ERROR << "Failed to query memory: no Redis client available"; + return NIXL_ERR_BACKEND; + } + + resp.clear(); + resp.reserve(descs.descCount()); + + bool has_error = false; + + try { + for (int i = 0; i < descs.descCount(); ++i) { + const auto &desc = descs[i]; + const std::string key = + desc.metaInfo.empty() ? std::to_string(desc.devId) : desc.metaInfo; + + std::optional exists = client->checkKeyExistsSync(key); + if (!exists.has_value()) { + resp.emplace_back(std::nullopt); + has_error = true; + } else { + resp.emplace_back(*exists ? nixl_query_resp_t{nixl_b_params_t{}} : std::nullopt); + } + } + } + catch (const std::exception &e) { + NIXL_ERROR << "Failed to query memory: " << e.what(); + return NIXL_ERR_BACKEND; + } + + if (has_error) { + return NIXL_ERR_BACKEND; + } + + return NIXL_SUCCESS; +} + +nixl_status_t +nixlRedisKVEngineImpl::prepXfer(const nixl_xfer_op_t &operation, + const nixl_meta_dlist_t &local, + const nixl_meta_dlist_t &remote, + const std::string &remote_agent, + const std::string &local_agent, + nixlBackendReqH *&handle, + const nixl_opt_b_args_t *opt_args) const { + if (!isValidPrepXferParams(operation, local, remote, remote_agent, local_agent)) { + return NIXL_ERR_INVALID_PARAM; + } + + auto req_h = std::make_unique(); + handle = req_h.release(); + return NIXL_SUCCESS; +} + +nixl_status_t +nixlRedisKVEngineImpl::postXfer(const nixl_xfer_op_t &operation, + const nixl_meta_dlist_t &local, + const nixl_meta_dlist_t &remote, + const std::string &remote_agent, + const std::string &local_agent, + nixlBackendReqH *&handle, + const nixl_opt_b_args_t *opt_args) const { + if (!handle) { + return NIXL_ERR_INVALID_PARAM; + } + + auto *req_h = static_cast(handle); + iRedisClient *client = getClient(); + if (!client) { + return NIXL_ERR_BACKEND; + } + + req_h->statusFutures_.clear(); + req_h->statusPromises_.clear(); + + for (int i = 0; i < local.descCount(); ++i) { + const auto &local_desc = local[i]; + const auto &remote_desc = remote[i]; + + std::string redis_key; + if (remote_desc.metadataP) { + auto *redis_md = dynamic_cast(remote_desc.metadataP); + if (redis_md) { + redis_key = redis_md->redisKey; + } else { + auto it = devIdToRedisKey_.find(remote_desc.devId); + if (it == devIdToRedisKey_.end()) { + return NIXL_ERR_INVALID_PARAM; + } + redis_key = it->second; + } + } else { + auto it = devIdToRedisKey_.find(remote_desc.devId); + if (it == devIdToRedisKey_.end()) { + return NIXL_ERR_INVALID_PARAM; + } + redis_key = it->second; + } + + auto status_promise = std::make_shared>(); + req_h->statusPromises_.push_back(status_promise); + req_h->statusFutures_.push_back(status_promise->get_future()); + + uintptr_t data_ptr = local_desc.addr; + size_t data_len = local_desc.len; + + if (operation == NIXL_WRITE) { + client->putKeyAsync(redis_key, data_ptr, data_len, status_promise); + } else { + client->getKeyAsync(redis_key, data_ptr, data_len, status_promise); + } + } + + return NIXL_IN_PROG; +} + +nixl_status_t +nixlRedisKVEngineImpl::checkXfer(nixlBackendReqH *handle) const { + if (!handle) { + return NIXL_ERR_INVALID_PARAM; + } + return static_cast(handle)->getOverallStatus(); +} + +nixl_status_t +nixlRedisKVEngineImpl::releaseReqH(nixlBackendReqH *handle) const { + delete static_cast(handle); + return NIXL_SUCCESS; +} diff --git a/src/plugins/kv/redis/engine_impl.h b/src/plugins/kv/redis/engine_impl.h new file mode 100644 index 0000000000..b8c3334bca --- /dev/null +++ b/src/plugins/kv/redis/engine_impl.h @@ -0,0 +1,90 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @file engine_impl.h + * @brief nixlRedisKVEngineImpl — NIXL protocol layer over iRedisClient. + */ + +#ifndef KV_PLUGIN_REDIS_ENGINE_IMPL_H +#define KV_PLUGIN_REDIS_ENGINE_IMPL_H + +#include "../kv_engine_impl.h" +#include "redis_engine.h" +#include +#include +#include + +/** + * @class nixlRedisKVEngineImpl + * @brief REDIS KV engine implementation — NIXL transfer protocol over iRedisClient. + */ +class nixlRedisKVEngineImpl : public nixlKVEngineImpl { +public: + explicit nixlRedisKVEngineImpl(const nixlBackendInitParams *init_params); + + nixlRedisKVEngineImpl(const nixlBackendInitParams *init_params, + std::shared_ptr redis_client); + + ~nixlRedisKVEngineImpl() override; + + nixl_mem_list_t getSupportedMems() const override { + return {OBJ_SEG, DRAM_SEG}; + } + + nixl_status_t + registerMem(const nixlBlobDesc &mem, const nixl_mem_t &nixl_mem, nixlBackendMD *&out) override; + + nixl_status_t deregisterMem(nixlBackendMD *meta) override; + + nixl_status_t + queryMem(const nixl_reg_dlist_t &descs, std::vector &resp) const override; + + nixl_status_t + prepXfer(const nixl_xfer_op_t &operation, + const nixl_meta_dlist_t &local, + const nixl_meta_dlist_t &remote, + const std::string &remote_agent, + const std::string &local_agent, + nixlBackendReqH *&handle, + const nixl_opt_b_args_t *opt_args) const override; + + nixl_status_t + postXfer(const nixl_xfer_op_t &operation, + const nixl_meta_dlist_t &local, + const nixl_meta_dlist_t &remote, + const std::string &remote_agent, + const std::string &local_agent, + nixlBackendReqH *&handle, + const nixl_opt_b_args_t *opt_args) const override; + + nixl_status_t checkXfer(nixlBackendReqH *handle) const override; + + nixl_status_t releaseReqH(nixlBackendReqH *handle) const override; + +protected: + virtual iRedisClient * + getClient() const { + return redisClient_.get(); + } + + std::shared_ptr executor_; + std::shared_ptr redisClient_; + std::unordered_map devIdToRedisKey_; +}; + +#endif // KV_PLUGIN_REDIS_ENGINE_IMPL_H diff --git a/src/plugins/kv/redis/meson.build b/src/plugins/kv/redis/meson.build new file mode 100644 index 0000000000..8d328c2529 --- /dev/null +++ b/src/plugins/kv/redis/meson.build @@ -0,0 +1,122 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# All REDIS plugin sources live in this directory (kv/redis/). + +if not enabled_plugins.get('REDIS') + subdir_done() +endif + +redis_sources = [ + '../kv_engine.cpp', + 'redis_engine.cpp', + 'redis_engine.h', + 'redis_plugin.cpp', + 'redis_executor.h', + 'client.cpp', + 'client.h', + 'engine_impl.cpp', + 'engine_impl.h', +] + +hiredis_async_dep = dependency('hiredis_async', required: false) +libevent_dep = dependency('libevent', required: false) +libevent_pthreads_dep = dependency('libevent_pthreads', required: false) + +if not hiredis_async_dep.found() + hiredis_async_dep = dependency('hiredis', required: false) +endif + +if not hiredis_async_dep.found() or not libevent_dep.found() + cc = meson.get_compiler('cpp') + hiredis_lib = cc.find_library('hiredis', required: false) + libevent_lib = cc.find_library('event', required: false) + libevent_pthreads_lib = cc.find_library('event_pthreads', required: false) + + if hiredis_lib.found() and libevent_lib.found() + if not hiredis_async_dep.found() + hiredis_async_dep = hiredis_lib + endif + if not libevent_dep.found() + libevent_dep = libevent_lib + endif + if libevent_pthreads_lib.found() and not libevent_pthreads_dep.found() + libevent_pthreads_dep = libevent_pthreads_lib + endif + endif +endif + +if not hiredis_async_dep.found() or not libevent_dep.found() + if 'REDIS' in static_plugins + error('REDIS static plugin requested but hiredis and libevent are required. Install libhiredis-dev and libevent-dev (or equivalent), then reconfigure.') + endif + warning('hiredis-async or libevent not available, skipping REDIS plugin build') + subdir_done() +endif + +compile_defs_redis = [] +if is_variable('compile_defs') + compile_defs_redis = compile_defs +endif +if hiredis_async_dep.found() and libevent_dep.found() + compile_defs_redis += ['-DHAVE_HIREDIS_ASYNC'] +endif + +redis_plugin_deps = [nixl_infra, nixl_common_dep, hiredis_async_dep, libevent_dep, dependency('asio', required: true)] +if libevent_pthreads_dep.found() + redis_plugin_deps += [libevent_pthreads_dep] +else + cc = meson.get_compiler('cpp') + event_pthreads = cc.find_library('event_pthreads', required: false) + if event_pthreads.found() + redis_plugin_deps += [event_pthreads] + else + warning('libevent_pthreads not found; REDIS plugin may fail to link (evthread_use_pthreads)') + endif +endif + +if not is_variable('compile_flags') + compile_flags = [] +endif + +if 'REDIS' in static_plugins + redis_backend_lib = static_library( + 'REDIS', + redis_sources, + dependencies: redis_plugin_deps, + cpp_args: compile_defs_redis + compile_flags, + include_directories: [nixl_inc_dirs, utils_inc_dirs], + install: false, + name_prefix: 'libplugin_') +else + redis_backend_lib = shared_library( + 'REDIS', + redis_sources, + dependencies: redis_plugin_deps, + cpp_args: compile_defs_redis + ['-fPIC'], + include_directories: [nixl_inc_dirs, utils_inc_dirs], + install: true, + name_prefix: 'libplugin_', + install_dir: plugin_install_dir, + install_rpath: '$ORIGIN/..') + if get_option('buildtype') == 'debug' + run_command('sh', '-c', + 'echo "REDIS=' + redis_backend_lib.full_path() + '" >> ' + plugin_build_dir + '/pluginlist', + check: true + ) + endif +endif + +redis_backend_interface = declare_dependency(link_with: redis_backend_lib) diff --git a/src/plugins/kv/redis/redis_engine.cpp b/src/plugins/kv/redis/redis_engine.cpp new file mode 100644 index 0000000000..99f0d6a49d --- /dev/null +++ b/src/plugins/kv/redis/redis_engine.cpp @@ -0,0 +1,37 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @file redis_engine.cpp + * @brief Factory wiring for nixlRedisKVEngine; creates nixlRedisKVEngineImpl at construction. + */ + +#include "redis_engine.h" +#include "engine_impl.h" +#include + +namespace { + +std::unique_ptr +createRedisKVEngineImpl(const nixlBackendInitParams *init_params) { + return std::make_unique(init_params); +} + +} // namespace + +nixlRedisKVEngine::nixlRedisKVEngine(const nixlBackendInitParams *init_params) + : nixlKVEngine(init_params, createRedisKVEngineImpl(init_params)) {} diff --git a/src/plugins/kv/redis/redis_engine.h b/src/plugins/kv/redis/redis_engine.h new file mode 100644 index 0000000000..3efc1f122a --- /dev/null +++ b/src/plugins/kv/redis/redis_engine.h @@ -0,0 +1,114 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @file redis_engine.h + * @brief REDIS KV plugin — iRedisClient interface and nixlRedisKVEngine entry. + * + * KV plugin layout (under src/plugins/kv/): + * - nixlKVEngine / nixlKVEngineImpl (../kv_engine.h, ../kv_engine_impl.h) + * shared NIXL protocol wrapper and implementation base for KV backends + * - iRedisClient (this file) + * async Redis storage interface used by the REDIS backend + * - nixlRedisKVEngine (this file) + * thin nixlKVEngine subclass registered with the NIXL plugin loader + * - hiredisAsyncClient (client.h) + * async SET/GET (hiredis-async) + sync EXISTS (blocking hiredis) + * - nixlRedisKVEngineImpl (engine_impl.h) + * concrete nixlKVEngineImpl; maps registerMem/postXfer to iRedisClient calls + */ + +#ifndef NIXL_SRC_PLUGINS_KV_REDIS_ENGINE_H +#define NIXL_SRC_PLUGINS_KV_REDIS_ENGINE_H + +#include "../kv_engine.h" +#include "redis_executor.h" +#include +#include +#include +#include +#include + +/** + * @class iRedisClient + * @brief Redis client for the REDIS KV backend. + * + * SET/GET are async (std::promise) for postXfer/checkXfer. + * EXISTS uses a separate synchronous hiredis connection for queryMem(). + */ +class iRedisClient { +public: + virtual ~iRedisClient() = default; + + /** + * @brief Set the executor used to complete async promises. + * @param executor ASIO thread pool shared with the engine implementation. + */ + virtual void + setExecutor(std::shared_ptr executor) = 0; + + /** + * @brief Asynchronously store a value under key (Redis SET). + * @param key Redis key. + * @param data_ptr Pointer to value bytes. + * @param data_len Number of bytes to write. + * @param promise Promise set to NIXL_SUCCESS or NIXL_ERR_BACKEND on completion. + */ + virtual void + putKeyAsync(std::string_view key, + uintptr_t data_ptr, + size_t data_len, + std::shared_ptr> promise) = 0; + + /** + * @brief Asynchronously read a value into buffer (Redis GET). + * @param key Redis key. + * @param data_ptr Output buffer pointer. + * @param data_len Maximum number of bytes to read. + * @param promise Promise set to NIXL_SUCCESS or NIXL_ERR_BACKEND on completion. + */ + virtual void + getKeyAsync(std::string_view key, + uintptr_t data_ptr, + size_t data_len, + std::shared_ptr> promise) = 0; + + /** + * @brief Synchronously check whether key exists (Redis EXISTS). + * + * Uses a dedicated blocking hiredis connection, independent of the async + * SET/GET path. Intended for queryMem() only. + * + * @param key Redis key. + * @return true if key exists, false if not found, std::nullopt on error. + */ + virtual std::optional + checkKeyExistsSync(std::string_view key) = 0; +}; + +/** + * @class nixlRedisKVEngine + * @brief REDIS backend engine registered with the NIXL plugin loader. + */ +class nixlRedisKVEngine : public nixlKVEngine { +public: + explicit nixlRedisKVEngine(const nixlBackendInitParams *init_params); + + ~nixlRedisKVEngine() override = default; +}; + +#endif // NIXL_SRC_PLUGINS_KV_REDIS_ENGINE_H diff --git a/src/plugins/kv/redis/redis_executor.h b/src/plugins/kv/redis/redis_executor.h new file mode 100644 index 0000000000..68c1a7582a --- /dev/null +++ b/src/plugins/kv/redis/redis_executor.h @@ -0,0 +1,123 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef NIXL_SRC_PLUGINS_KV_REDIS_EXECUTOR_H +#define NIXL_SRC_PLUGINS_KV_REDIS_EXECUTOR_H + +#include +#include +#include + +/** + * @class asioThreadPoolExecutor + * @brief ASIO-based thread pool executor for asynchronous task execution + * + * This executor uses ASIO's thread_pool to execute tasks asynchronously across + * multiple threads. It provides a simple interface for posting tasks that will + * be executed by worker threads in the pool. + * + * ASIO Design Pattern: + * - ASIO (Asio C++ library) provides asynchronous I/O operations + * - thread_pool creates a pool of worker threads that process tasks + * - post() submits tasks to the pool's work queue + * - Tasks are executed by worker threads asynchronously (non-blocking) + * + * Why use ASIO executor here? + * - Redis async callbacks need to be executed in a separate thread context + * - Prevents blocking the main thread or hiredis event loop thread + * - Allows concurrent execution of multiple Redis operations + * - Provides thread-safe task queuing and execution + */ +class asioThreadPoolExecutor { +public: + /** + * @brief Construct a thread pool executor with specified number of threads + * @param num_threads Number of worker threads in the pool + * + * Example: asioThreadPoolExecutor(4) creates a pool with 4 worker threads + * These threads will continuously process tasks from the work queue + */ + explicit asioThreadPoolExecutor(std::size_t num_threads) : pool_(num_threads) { + // ASIO thread_pool automatically starts worker threads + // Each thread runs an event loop that processes tasks from the queue + } + + /** + * @brief Stop the thread pool and wait for all threads to finish + * + * This is called during Redis backend destruction to ensure all + * pending tasks complete before cleanup. + */ + void + WaitUntilStopped() { + pool_.stop(); // Signal all threads to stop processing new tasks + pool_.join(); // Wait for all threads to finish current tasks and exit + } + + /** + * @brief Wait until all queued tasks are completed + * + * Blocks until the work queue is empty and all tasks have been processed. + * Useful for synchronization during shutdown. + */ + void + waitUntilIdle() { + pool_.wait(); // Wait for all tasks in the queue to complete + } + + /** + * @brief Post a task to the thread pool for asynchronous execution + * @param task Function object to execute asynchronously + * + * ASIO post() semantics: + * - Non-blocking: returns immediately after queuing the task + * - Asynchronous: task executes in a worker thread, not the caller's thread + * - Thread-safe: multiple threads can post tasks concurrently + * - Ordering: tasks are executed in FIFO order per thread + * + * Execution flow: + * 1. Caller thread: post(task) -> returns immediately + * 2. Task is queued in ASIO's internal work queue + * 3. Worker thread picks up task from queue + * 4. Worker thread executes task asynchronously + * + * Example usage: + * executor->post([promise_ptr, success]() { + * promise_ptr->set_value(success ? NIXL_SUCCESS : NIXL_ERR_BACKEND); + * }); + * + * This lambda will execute in a worker thread, not the caller's thread. + */ + void + post(std::function &&task) { + // asio::post schedules the task for execution in the thread pool + // The task will be executed by one of the worker threads + asio::post(pool_, std::move(task)); + } + +private: + /** + * ASIO thread_pool manages a pool of worker threads + * - Automatically creates and manages worker threads + * - Provides a work queue for task distribution + * - Handles thread lifecycle (creation, execution, cleanup) + * - Ensures thread-safe task execution + */ + asio::thread_pool pool_; +}; + +#endif // NIXL_SRC_PLUGINS_KV_REDIS_EXECUTOR_H diff --git a/src/plugins/kv/redis/redis_plugin.cpp b/src/plugins/kv/redis/redis_plugin.cpp new file mode 100644 index 0000000000..59434458a0 --- /dev/null +++ b/src/plugins/kv/redis/redis_plugin.cpp @@ -0,0 +1,45 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @file redis_plugin.cpp + * @brief Registers the REDIS backend with NIXL (static or shared plugin). + */ + +#include "nixl_types.h" +#include "redis_engine.h" +#include "backend/backend_plugin.h" +#include "common/nixl_log.h" + +using redis_plugin_t = nixlBackendPluginCreator; + +#ifdef STATIC_PLUGIN_REDIS +nixlBackendPlugin * +createStaticREDISPlugin() { + return redis_plugin_t::create( + NIXL_PLUGIN_API_VERSION, "REDIS", "0.1.0", {}, {DRAM_SEG, OBJ_SEG}); +} +#else +extern "C" NIXL_PLUGIN_EXPORT nixlBackendPlugin * +nixl_plugin_init() { + return redis_plugin_t::create( + NIXL_PLUGIN_API_VERSION, "REDIS", "0.1.0", {}, {DRAM_SEG, OBJ_SEG}); +} + +extern "C" NIXL_PLUGIN_EXPORT void +nixl_plugin_fini() {} +#endif diff --git a/src/plugins/meson.build b/src/plugins/meson.build index ce7103cb63..26b42c1962 100644 --- a/src/plugins/meson.build +++ b/src/plugins/meson.build @@ -29,6 +29,10 @@ if enabled_plugins.get('OBJ') subdir('obj') endif +if enabled_plugins.get('REDIS') + subdir('kv') +endif + if enabled_plugins.get('AZURE_BLOB') subdir('azure_blob') endif From b3643b15f850ae6c9890f3ee3481c0ee1c300f38 Mon Sep 17 00:00:00 2001 From: Harry Xie Date: Wed, 17 Jun 2026 03:15:19 +0300 Subject: [PATCH 08/24] refactor(kv): consolidate backend interface --- meson.build | 4 +- .../kv/{kv_engine.cpp => kv_backend.cpp} | 7 +- src/plugins/kv/{kv_engine.h => kv_backend.h} | 95 +++++++---- src/plugins/kv/kv_engine_impl.h | 157 ------------------ src/plugins/kv/kv_store.h | 79 --------- src/plugins/kv/meson.build | 2 +- src/plugins/kv/redis/engine_impl.h | 2 +- src/plugins/kv/redis/meson.build | 2 +- src/plugins/kv/redis/redis_engine.h | 13 +- 9 files changed, 65 insertions(+), 296 deletions(-) rename src/plugins/kv/{kv_engine.cpp => kv_backend.cpp} (95%) rename src/plugins/kv/{kv_engine.h => kv_backend.h} (60%) delete mode 100644 src/plugins/kv/kv_engine_impl.h delete mode 100644 src/plugins/kv/kv_store.h diff --git a/meson.build b/meson.build index 7d52019ac0..2b0070a042 100644 --- a/meson.build +++ b/meson.build @@ -538,9 +538,7 @@ if get_option('install_headers') install_headers('src/core/transfer_request.h', install_dir: prefix_inc) install_headers('src/core/agent_data.h', install_dir: prefix_inc) install_headers('src/infra/mem_section.h', install_dir: prefix_inc) - install_headers('src/plugins/kv/kv_store.h', install_dir: prefix_inc + '/plugins/kv') - install_headers('src/plugins/kv/kv_engine_impl.h', install_dir: prefix_inc + '/plugins/kv') - install_headers('src/plugins/kv/kv_engine.h', install_dir: prefix_inc + '/plugins/kv') + install_headers('src/plugins/kv/kv_backend.h', install_dir: prefix_inc + '/plugins/kv') install_headers('src/core/telemetry/telemetry_event.h', install_dir: prefix_inc) if ucx_gpu_device_api_available diff --git a/src/plugins/kv/kv_engine.cpp b/src/plugins/kv/kv_backend.cpp similarity index 95% rename from src/plugins/kv/kv_engine.cpp rename to src/plugins/kv/kv_backend.cpp index 12191e8da8..9dfb70e5fd 100644 --- a/src/plugins/kv/kv_engine.cpp +++ b/src/plugins/kv/kv_backend.cpp @@ -15,12 +15,7 @@ * limitations under the License. */ -/** - * @file kv_engine.cpp - * @brief nixlKVEngine delegation layer; forwards nixlBackendEngine calls to impl_. - */ - -#include "kv_engine.h" +#include "kv_backend.h" #include nixlKVEngine::nixlKVEngine(const nixlBackendInitParams *init_params, diff --git a/src/plugins/kv/kv_engine.h b/src/plugins/kv/kv_backend.h similarity index 60% rename from src/plugins/kv/kv_engine.h rename to src/plugins/kv/kv_backend.h index 6de1280ff4..273d9a12a8 100644 --- a/src/plugins/kv/kv_engine.h +++ b/src/plugins/kv/kv_backend.h @@ -15,40 +15,70 @@ * limitations under the License. */ -/** - * @file kv_engine.h - * @brief Shared thin wrapper base class for KV-style NIXL backend engines. - * - * nixlKVEngine implements nixlBackendEngine and forwards all data-plane calls - * to a nixlKVEngineImpl instance supplied by each plugin. - * - * Plugin authors subclass nixlKVEngine (or use it directly with a factory) and - * register the resulting engine with nixlBackendPluginCreator. - */ - -#ifndef NIXL_SRC_PLUGINS_KV_KV_ENGINE_H -#define NIXL_SRC_PLUGINS_KV_KV_ENGINE_H +#ifndef KV_BACKEND_H +#define KV_BACKEND_H -#include "kv_engine_impl.h" +#include "backend/backend_engine.h" +#include +#include #include +#include +#include +#include + +class iKVStore { +public: + virtual ~iKVStore() = default; + + virtual nixl_status_t + put(std::string_view key, const uint8_t *data, size_t len) = 0; + + virtual nixl_status_t + get(std::string_view key, uint8_t *buffer, size_t len, size_t &bytes_read) const = 0; + + virtual bool + exists(std::string_view key) const = 0; +}; + +class nixlKVEngineImpl { +public: + virtual ~nixlKVEngineImpl() = default; + + virtual nixl_mem_list_t + getSupportedMems() const = 0; + + virtual nixl_status_t + registerMem(const nixlBlobDesc &mem, const nixl_mem_t &nixl_mem, nixlBackendMD *&out) = 0; + virtual nixl_status_t + deregisterMem(nixlBackendMD *meta) = 0; + virtual nixl_status_t + queryMem(const nixl_reg_dlist_t &descs, std::vector &resp) const = 0; + virtual nixl_status_t + prepXfer(const nixl_xfer_op_t &operation, + const nixl_meta_dlist_t &local, + const nixl_meta_dlist_t &remote, + const std::string &remote_agent, + const std::string &local_agent, + nixlBackendReqH *&handle, + const nixl_opt_b_args_t *opt_args) const = 0; + virtual nixl_status_t + postXfer(const nixl_xfer_op_t &operation, + const nixl_meta_dlist_t &local, + const nixl_meta_dlist_t &remote, + const std::string &remote_agent, + const std::string &local_agent, + nixlBackendReqH *&handle, + const nixl_opt_b_args_t *opt_args) const = 0; + virtual nixl_status_t + checkXfer(nixlBackendReqH *handle) const = 0; + virtual nixl_status_t + releaseReqH(nixlBackendReqH *handle) const = 0; +}; -/** - * @class nixlKVEngine - * @brief Thin nixlBackendEngine wrapper that delegates to nixlKVEngineImpl. - * - * Common KV backend capabilities (local-only, no notifications) are declared here - * so individual plugins only need to supply a concrete nixlKVEngineImpl. - */ class nixlKVEngine : public nixlBackendEngine { public: - /** - * @brief Construct a KV engine with a plugin-specific implementation. - * @param init_params NIXL initialization parameters. - * @param impl Concrete nixlKVEngineImpl (ownership transferred). - */ nixlKVEngine(const nixlBackendInitParams *init_params, std::unique_ptr impl); - - ~nixlKVEngine() override; + virtual ~nixlKVEngine(); bool supportsRemote() const override { @@ -92,12 +122,6 @@ class nixlKVEngine : public nixlBackendEngine { return NIXL_SUCCESS; } - /** - * @brief Passes local metadata through unchanged. - * - * Local-only KV backends do not transform metadata. input must be the - * nixlBackendMD* returned by registerMem(). - */ nixl_status_t loadLocalMD(nixlBackendMD *input, nixlBackendMD *&output) override { if (input == nullptr) { @@ -126,7 +150,6 @@ class nixlKVEngine : public nixlBackendEngine { nixl_status_t checkXfer(nixlBackendReqH *handle) const override; - nixl_status_t releaseReqH(nixlBackendReqH *handle) const override; @@ -134,4 +157,4 @@ class nixlKVEngine : public nixlBackendEngine { std::unique_ptr impl_; }; -#endif // NIXL_SRC_PLUGINS_KV_KV_ENGINE_H +#endif // KV_BACKEND_H diff --git a/src/plugins/kv/kv_engine_impl.h b/src/plugins/kv/kv_engine_impl.h deleted file mode 100644 index cb6c378106..0000000000 --- a/src/plugins/kv/kv_engine_impl.h +++ /dev/null @@ -1,157 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * @file kv_engine_impl.h - * @brief Abstract implementation interface for KV-style NIXL backend engines. - * - * Architecture: - * - * nixlBackendEngine NIXL agent-facing protocol (registerMem, prepXfer, ...) - * ^ - * | nixlKVEngine Thin wrapper; delegates lifecycle calls to impl_ - * | - * nixlKVEngineImpl Vendor/backend-specific logic (this header) - * ^ - * | - * nixlInMemKVEngineImpl Example: in-process map via iKVStore (examples/plugins/inmemkv/) - * nixlRedisKVEngineImpl REDIS plugin: redis/engine_impl.* (under kv/redis/) - * - * Synchronous KV backends use iKVStore (kv_store.h). Redis is async and uses - * iRedisClient in kv/redis/redis_engine.h; hiredisAsyncClient in kv/redis/client.*. - */ - -#ifndef NIXL_SRC_PLUGINS_KV_KV_ENGINE_IMPL_H -#define NIXL_SRC_PLUGINS_KV_KV_ENGINE_IMPL_H - -#include "backend/backend_engine.h" -#include -#include - -/** - * @class nixlKVEngineImpl - * @brief Abstract implementation interface for KV-style backend engines. - * - * Each KV plugin (INMEMKV example, REDIS, etc.) provides a concrete subclass - * that implements register/deregister, query, and transfer operations. Sync - * backends delegate storage to iKVStore; async backends (Redis) use iRedisClient. - */ -class nixlKVEngineImpl { -public: - virtual ~nixlKVEngineImpl() = default; - - /** @brief Memory segment types supported by this KV backend. @return Segment list (typically - * DRAM_SEG). */ - virtual nixl_mem_list_t - getSupportedMems() const = 0; - - /** - * @brief Registers a local memory descriptor as a KV key. - * - * @param mem Descriptor blob including key metadata (metaInfo or devId). - * @param nixl_mem Memory segment type; must be supported by getSupportedMems(). - * @param out On success, receives a newly allocated nixlBackendMD* owned by the caller - * until deregisterMem(). - * @return NIXL_SUCCESS on success, or an error status on invalid input or backend failure. - */ - virtual nixl_status_t - registerMem(const nixlBlobDesc &mem, const nixl_mem_t &nixl_mem, nixlBackendMD *&out) = 0; - - /** - * @brief Deregisters backend metadata created by registerMem(). - * - * @param meta Metadata pointer returned by registerMem(); must be non-null. - * @return NIXL_SUCCESS on success, or an error status if meta is invalid. - */ - virtual nixl_status_t - deregisterMem(nixlBackendMD *meta) = 0; - - /** - * @brief Queries whether registered keys exist in the backing store. - * - * @param descs Registered descriptors to query. - * @param resp Output vector; each entry is set when the key exists, or std::nullopt otherwise. - * @return NIXL_SUCCESS on success, or an error status on backend failure. - */ - virtual nixl_status_t - queryMem(const nixl_reg_dlist_t &descs, std::vector &resp) const = 0; - - /** - * @brief Prepares a transfer request and allocates a backend request handle. - * - * @param operation Transfer operation (NIXL_WRITE or NIXL_READ). - * @param local Local descriptor metadata list. - * @param remote Remote descriptor metadata list. - * @param remote_agent Remote agent name (unused for local-only KV backends). - * @param local_agent Local agent name from nixlKVEngine::localAgent (passed explicitly - * so impl does not depend on nixlBackendEngine protected members). - * @param handle On success, receives a newly allocated nixlBackendReqH* owned by the caller - * until releaseReqH(). - * @param opt_args Optional backend arguments; may be nullptr. - * @return NIXL_SUCCESS on success, or an error status on invalid parameters. - */ - virtual nixl_status_t - prepXfer(const nixl_xfer_op_t &operation, - const nixl_meta_dlist_t &local, - const nixl_meta_dlist_t &remote, - const std::string &remote_agent, - const std::string &local_agent, - nixlBackendReqH *&handle, - const nixl_opt_b_args_t *opt_args) const = 0; - - /** - * @brief Executes a prepared transfer request. - * - * @param operation Transfer operation (NIXL_WRITE or NIXL_READ). - * @param local Local descriptor metadata list. - * @param remote Remote descriptor metadata list. - * @param remote_agent Remote agent name (unused for local-only KV backends). - * @param local_agent Local agent name from nixlKVEngine::localAgent (passed explicitly - * so impl does not depend on nixlBackendEngine protected members). - * @param handle Request handle allocated by prepXfer(); must be non-null. - * @param opt_args Optional backend arguments; may be nullptr. - * @return NIXL_SUCCESS on success, or an error status on transfer or backend failure. - */ - virtual nixl_status_t - postXfer(const nixl_xfer_op_t &operation, - const nixl_meta_dlist_t &local, - const nixl_meta_dlist_t &remote, - const std::string &remote_agent, - const std::string &local_agent, - nixlBackendReqH *&handle, - const nixl_opt_b_args_t *opt_args) const = 0; - - /** - * @brief Checks transfer completion status for a request handle. - * - * @param handle Request handle from prepXfer(); must be non-null. - * @return NIXL_SUCCESS when the transfer is complete, or an in-progress/error status. - */ - virtual nixl_status_t - checkXfer(nixlBackendReqH *handle) const = 0; - - /** - * @brief Releases a request handle allocated by prepXfer(). - * - * @param handle Request handle to release; must be non-null. - * @return NIXL_SUCCESS on success. - */ - virtual nixl_status_t - releaseReqH(nixlBackendReqH *handle) const = 0; -}; - -#endif // NIXL_SRC_PLUGINS_KV_KV_ENGINE_IMPL_H diff --git a/src/plugins/kv/kv_store.h b/src/plugins/kv/kv_store.h deleted file mode 100644 index 34824d9308..0000000000 --- a/src/plugins/kv/kv_store.h +++ /dev/null @@ -1,79 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * @file kv_store.h - * @brief Shared minimal key-value storage interface for NIXL KV backends. - * - * Synchronous implementations (e.g. INMEMKV example plugin) provide - * backend-specific storage via this API. Async backends (REDIS) use iRedisClient - * in kv/redis/redis_engine.h instead — see kv_engine_impl.h. - */ - -#ifndef NIXL_SRC_PLUGINS_KV_KV_STORE_H -#define NIXL_SRC_PLUGINS_KV_KV_STORE_H - -#include "nixl_types.h" -#include -#include -#include - -/** - * @brief Minimal key-value storage interface for KV-style NIXL backends. - * - * This interface intentionally keeps only the smallest synchronous API - * needed by current KV backend flows (e.g. INMEMKV). Async backends use - * iRedisClient instead — see kv/redis/redis_engine.h. - */ -class iKVStore { -public: - virtual ~iKVStore() = default; - - /** - * @brief Stores value bytes for key. - * - * @param key Storage key. - * @param data Pointer to value bytes to store. - * @param len Number of bytes to copy from data. - * @return NIXL_SUCCESS on success, or a backend error code on failure. - */ - virtual nixl_status_t - put(std::string_view key, const uint8_t *data, size_t len) = 0; - - /** - * @brief Reads value for key into buffer. - * - * @param key Storage key. - * @param buffer Output buffer for stored value bytes. - * @param len Maximum number of bytes to read into buffer. - * @param bytes_read Set to min(stored_len, len) on success. - * @return NIXL_SUCCESS on success, or NIXL_ERR_NOT_FOUND when key does not exist. - */ - virtual nixl_status_t - get(std::string_view key, uint8_t *buffer, size_t len, size_t &bytes_read) const = 0; - - /** - * @brief Checks whether key is present in the store. - * - * @param key Storage key. - * @return true if key exists, false otherwise. - */ - virtual bool - exists(std::string_view key) const = 0; -}; - -#endif // NIXL_SRC_PLUGINS_KV_KV_STORE_H diff --git a/src/plugins/kv/meson.build b/src/plugins/kv/meson.build index 43a7b27a3f..89f197efe3 100644 --- a/src/plugins/kv/meson.build +++ b/src/plugins/kv/meson.build @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Shared KV headers (kv_engine, kv_engine_impl, kv_store) live in kv/. +# Shared KV backend interfaces live in kv/. # REDIS plugin sources are entirely under kv/redis/. if enabled_plugins.get('REDIS') diff --git a/src/plugins/kv/redis/engine_impl.h b/src/plugins/kv/redis/engine_impl.h index b8c3334bca..9a247879e4 100644 --- a/src/plugins/kv/redis/engine_impl.h +++ b/src/plugins/kv/redis/engine_impl.h @@ -23,7 +23,7 @@ #ifndef KV_PLUGIN_REDIS_ENGINE_IMPL_H #define KV_PLUGIN_REDIS_ENGINE_IMPL_H -#include "../kv_engine_impl.h" +#include "../kv_backend.h" #include "redis_engine.h" #include #include diff --git a/src/plugins/kv/redis/meson.build b/src/plugins/kv/redis/meson.build index 8d328c2529..f4cc10de1d 100644 --- a/src/plugins/kv/redis/meson.build +++ b/src/plugins/kv/redis/meson.build @@ -20,7 +20,7 @@ if not enabled_plugins.get('REDIS') endif redis_sources = [ - '../kv_engine.cpp', + '../kv_backend.cpp', 'redis_engine.cpp', 'redis_engine.h', 'redis_plugin.cpp', diff --git a/src/plugins/kv/redis/redis_engine.h b/src/plugins/kv/redis/redis_engine.h index 3efc1f122a..2192089cf3 100644 --- a/src/plugins/kv/redis/redis_engine.h +++ b/src/plugins/kv/redis/redis_engine.h @@ -19,23 +19,12 @@ * @file redis_engine.h * @brief REDIS KV plugin — iRedisClient interface and nixlRedisKVEngine entry. * - * KV plugin layout (under src/plugins/kv/): - * - nixlKVEngine / nixlKVEngineImpl (../kv_engine.h, ../kv_engine_impl.h) - * shared NIXL protocol wrapper and implementation base for KV backends - * - iRedisClient (this file) - * async Redis storage interface used by the REDIS backend - * - nixlRedisKVEngine (this file) - * thin nixlKVEngine subclass registered with the NIXL plugin loader - * - hiredisAsyncClient (client.h) - * async SET/GET (hiredis-async) + sync EXISTS (blocking hiredis) - * - nixlRedisKVEngineImpl (engine_impl.h) - * concrete nixlKVEngineImpl; maps registerMem/postXfer to iRedisClient calls */ #ifndef NIXL_SRC_PLUGINS_KV_REDIS_ENGINE_H #define NIXL_SRC_PLUGINS_KV_REDIS_ENGINE_H -#include "../kv_engine.h" +#include "../kv_backend.h" #include "redis_executor.h" #include #include From bf5dc6e04a403144f06cfb01d8afa51856dd4860 Mon Sep 17 00:00:00 2001 From: Harry Xie Date: Wed, 17 Jun 2026 03:15:58 +0300 Subject: [PATCH 09/24] refactor(kv): hide injected impl constructor --- src/plugins/kv/kv_backend.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/plugins/kv/kv_backend.h b/src/plugins/kv/kv_backend.h index 273d9a12a8..5db3df4c04 100644 --- a/src/plugins/kv/kv_backend.h +++ b/src/plugins/kv/kv_backend.h @@ -77,7 +77,6 @@ class nixlKVEngineImpl { class nixlKVEngine : public nixlBackendEngine { public: - nixlKVEngine(const nixlBackendInitParams *init_params, std::unique_ptr impl); virtual ~nixlKVEngine(); bool @@ -153,6 +152,9 @@ class nixlKVEngine : public nixlBackendEngine { nixl_status_t releaseReqH(nixlBackendReqH *handle) const override; +protected: + nixlKVEngine(const nixlBackendInitParams *init_params, std::unique_ptr impl); + private: std::unique_ptr impl_; }; From e3b6d2de5bea89a509240bfff536912cb17ed753 Mon Sep 17 00:00:00 2001 From: Harry Xie Date: Wed, 17 Jun 2026 03:16:46 +0300 Subject: [PATCH 10/24] refactor(kv): drop local agent from postXfer --- src/plugins/kv/kv_backend.cpp | 2 +- src/plugins/kv/kv_backend.h | 1 - src/plugins/kv/redis/engine_impl.cpp | 1 - src/plugins/kv/redis/engine_impl.h | 1 - 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/plugins/kv/kv_backend.cpp b/src/plugins/kv/kv_backend.cpp index 9dfb70e5fd..680e569388 100644 --- a/src/plugins/kv/kv_backend.cpp +++ b/src/plugins/kv/kv_backend.cpp @@ -66,7 +66,7 @@ nixlKVEngine::postXfer(const nixl_xfer_op_t &operation, const std::string &remote_agent, nixlBackendReqH *&handle, const nixl_opt_b_args_t *opt_args) const { - return impl_->postXfer(operation, local, remote, remote_agent, localAgent, handle, opt_args); + return impl_->postXfer(operation, local, remote, remote_agent, handle, opt_args); } nixl_status_t diff --git a/src/plugins/kv/kv_backend.h b/src/plugins/kv/kv_backend.h index 5db3df4c04..b6cad3b93e 100644 --- a/src/plugins/kv/kv_backend.h +++ b/src/plugins/kv/kv_backend.h @@ -66,7 +66,6 @@ class nixlKVEngineImpl { const nixl_meta_dlist_t &local, const nixl_meta_dlist_t &remote, const std::string &remote_agent, - const std::string &local_agent, nixlBackendReqH *&handle, const nixl_opt_b_args_t *opt_args) const = 0; virtual nixl_status_t diff --git a/src/plugins/kv/redis/engine_impl.cpp b/src/plugins/kv/redis/engine_impl.cpp index 10cb0b23b8..56ca779261 100644 --- a/src/plugins/kv/redis/engine_impl.cpp +++ b/src/plugins/kv/redis/engine_impl.cpp @@ -231,7 +231,6 @@ nixlRedisKVEngineImpl::postXfer(const nixl_xfer_op_t &operation, const nixl_meta_dlist_t &local, const nixl_meta_dlist_t &remote, const std::string &remote_agent, - const std::string &local_agent, nixlBackendReqH *&handle, const nixl_opt_b_args_t *opt_args) const { if (!handle) { diff --git a/src/plugins/kv/redis/engine_impl.h b/src/plugins/kv/redis/engine_impl.h index 9a247879e4..0209ebf48d 100644 --- a/src/plugins/kv/redis/engine_impl.h +++ b/src/plugins/kv/redis/engine_impl.h @@ -68,7 +68,6 @@ class nixlRedisKVEngineImpl : public nixlKVEngineImpl { const nixl_meta_dlist_t &local, const nixl_meta_dlist_t &remote, const std::string &remote_agent, - const std::string &local_agent, nixlBackendReqH *&handle, const nixl_opt_b_args_t *opt_args) const override; From 9233e92cea77303c15c2b342af1ab4b15f2b65c5 Mon Sep 17 00:00:00 2001 From: Harry Xie Date: Wed, 17 Jun 2026 03:17:16 +0300 Subject: [PATCH 11/24] docs(kv): document backend facade layout --- src/plugins/kv/README.md | 9 +++++++++ src/plugins/kv/redis/README.md | 10 ++++------ 2 files changed, 13 insertions(+), 6 deletions(-) create mode 100644 src/plugins/kv/README.md diff --git a/src/plugins/kv/README.md b/src/plugins/kv/README.md new file mode 100644 index 0000000000..8d0873cdc4 --- /dev/null +++ b/src/plugins/kv/README.md @@ -0,0 +1,9 @@ +# NIXL KV Backend Interfaces + +The KV plugin layer follows the same facade/implementation split as the object backend: + +- `nixlKVEngine` is the NIXL-facing backend engine wrapper. +- `nixlKVEngineImpl` is the implementation interface for concrete KV backends. +- `iKVStore` is the small synchronous store interface used by simple KV backends. + +Redis provides its async client and concrete implementation under `redis/`. diff --git a/src/plugins/kv/redis/README.md b/src/plugins/kv/redis/README.md index 71195624c8..eacaacf475 100644 --- a/src/plugins/kv/redis/README.md +++ b/src/plugins/kv/redis/README.md @@ -9,9 +9,7 @@ All REDIS plugin code lives in `src/plugins/kv/redis/`. Shared KV abstractions s ``` kv/ # shared KV layer (PR1 interface) - kv_engine.h/.cpp nixlKVEngine - kv_engine_impl.h nixlKVEngineImpl - kv_store.h iKVStore (sync; INMEMKV only) + kv_backend.h/.cpp nixlKVEngine, nixlKVEngineImpl, iKVStore meson.build delegates to redis/ when REDIS enabled kv/redis/ # REDIS plugin (this directory) @@ -28,8 +26,8 @@ kv/redis/ # REDIS plugin (this directory) | Layer | REDIS plugin | |-------|--------------| -| Shared KV wrapper | `nixlKVEngine` (`../kv_engine.h`) | -| Shared KV impl base | `nixlKVEngineImpl` (`../kv_engine_impl.h`) | +| Shared KV wrapper | `nixlKVEngine` (`../kv_backend.h`) | +| Shared KV impl base | `nixlKVEngineImpl` (`../kv_backend.h`) | | Plugin entry | `nixlRedisKVEngine` (`redis_engine.h`) | | Async storage interface | `iRedisClient` (`redis_engine.h`) | | Client implementation | `hiredisAsyncClient` (`client.h`) | @@ -37,7 +35,7 @@ kv/redis/ # REDIS plugin (this directory) ## iKVStore vs iRedisClient -| | `iKVStore` (`kv_store.h`) | `iRedisClient` (`redis_engine.h`) | +| | `iKVStore` (`../kv_backend.h`) | `iRedisClient` (`redis_engine.h`) | |--|---------------------------|-----------------------------------| | Used by | INMEMKV example plugin | REDIS plugin | | API style | Sync `put` / `get` / `exists` | Async SET/GET + sync EXISTS (separate connections) | From 025eb190b8ac91debfa971a5e2a02bcb383001fc Mon Sep 17 00:00:00 2001 From: Harry Xie Date: Wed, 17 Jun 2026 03:36:31 +0300 Subject: [PATCH 12/24] refactor(kv): move redis plugin entrypoint to kv layer --- .../kv/{redis/redis_plugin.cpp => kv_plugin.cpp} | 13 +++++-------- src/plugins/kv/redis/meson.build | 5 +++-- 2 files changed, 8 insertions(+), 10 deletions(-) rename src/plugins/kv/{redis/redis_plugin.cpp => kv_plugin.cpp} (80%) diff --git a/src/plugins/kv/redis/redis_plugin.cpp b/src/plugins/kv/kv_plugin.cpp similarity index 80% rename from src/plugins/kv/redis/redis_plugin.cpp rename to src/plugins/kv/kv_plugin.cpp index 59434458a0..3622672a28 100644 --- a/src/plugins/kv/redis/redis_plugin.cpp +++ b/src/plugins/kv/kv_plugin.cpp @@ -15,29 +15,26 @@ * limitations under the License. */ -/** - * @file redis_plugin.cpp - * @brief Registers the REDIS backend with NIXL (static or shared plugin). - */ - #include "nixl_types.h" -#include "redis_engine.h" +#include "redis/redis_engine.h" #include "backend/backend_plugin.h" #include "common/nixl_log.h" using redis_plugin_t = nixlBackendPluginCreator; +static const nixl_mem_list_t supported_segments = {DRAM_SEG, OBJ_SEG}; + #ifdef STATIC_PLUGIN_REDIS nixlBackendPlugin * createStaticREDISPlugin() { return redis_plugin_t::create( - NIXL_PLUGIN_API_VERSION, "REDIS", "0.1.0", {}, {DRAM_SEG, OBJ_SEG}); + NIXL_PLUGIN_API_VERSION, "REDIS", "0.1.0", {}, supported_segments); } #else extern "C" NIXL_PLUGIN_EXPORT nixlBackendPlugin * nixl_plugin_init() { return redis_plugin_t::create( - NIXL_PLUGIN_API_VERSION, "REDIS", "0.1.0", {}, {DRAM_SEG, OBJ_SEG}); + NIXL_PLUGIN_API_VERSION, "REDIS", "0.1.0", {}, supported_segments); } extern "C" NIXL_PLUGIN_EXPORT void diff --git a/src/plugins/kv/redis/meson.build b/src/plugins/kv/redis/meson.build index f4cc10de1d..02df437e90 100644 --- a/src/plugins/kv/redis/meson.build +++ b/src/plugins/kv/redis/meson.build @@ -13,7 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# All REDIS plugin sources live in this directory (kv/redis/). +# REDIS implementation sources live in this directory; the plugin entrypoint +# stays in the shared KV layer to match the OBJ plugin layout. if not enabled_plugins.get('REDIS') subdir_done() @@ -21,9 +22,9 @@ endif redis_sources = [ '../kv_backend.cpp', + '../kv_plugin.cpp', 'redis_engine.cpp', 'redis_engine.h', - 'redis_plugin.cpp', 'redis_executor.h', 'client.cpp', 'client.h', From 3ce356b9444106f789f088da8bfd5b28fbc36bb0 Mon Sep 17 00:00:00 2001 From: Harry Xie Date: Wed, 17 Jun 2026 03:37:29 +0300 Subject: [PATCH 13/24] refactor(kv): make redis executor backend-specific --- src/plugins/kv/redis/client.cpp | 10 ++-- src/plugins/kv/redis/client.h | 8 +-- src/plugins/kv/redis/engine_impl.cpp | 4 +- src/plugins/kv/redis/engine_impl.h | 2 +- src/plugins/kv/redis/redis_engine.h | 2 +- src/plugins/kv/redis/redis_executor.h | 84 +++------------------------ 6 files changed, 20 insertions(+), 90 deletions(-) diff --git a/src/plugins/kv/redis/client.cpp b/src/plugins/kv/redis/client.cpp index dc88cc65d3..adb19a5c49 100644 --- a/src/plugins/kv/redis/client.cpp +++ b/src/plugins/kv/redis/client.cpp @@ -98,7 +98,7 @@ checkRedisReplyOk(redisReply *reply, const char *command) { } void -setPromiseStatus(const std::shared_ptr &executor, +setPromiseStatus(const std::shared_ptr &executor, const std::shared_ptr> &promise, bool success) { if (!promise) { @@ -116,7 +116,7 @@ setPromiseStatus(const std::shared_ptr &executor, } // namespace hiredisAsyncClient::hiredisAsyncClient(nixl_b_params_t *custom_params, - std::shared_ptr executor) + std::shared_ptr executor) : host_(getRedisHost(custom_params)), port_(getRedisPort(custom_params)), password_(getRedisPassword(custom_params)), @@ -257,7 +257,7 @@ hiredisAsyncClient::~hiredisAsyncClient() { } void -hiredisAsyncClient::setExecutor(std::shared_ptr executor) { +hiredisAsyncClient::setExecutor(std::shared_ptr executor) { executor_ = executor; } @@ -421,14 +421,14 @@ hiredisAsyncClient::checkKeyExistsSync(std::string_view key) { #else // HAVE_HIREDIS_ASYNC hiredisAsyncClient::hiredisAsyncClient(nixl_b_params_t *custom_params, - std::shared_ptr executor) { + std::shared_ptr executor) { throw std::runtime_error("hiredis-async not available"); } hiredisAsyncClient::~hiredisAsyncClient() {} void -hiredisAsyncClient::setExecutor(std::shared_ptr executor) {} +hiredisAsyncClient::setExecutor(std::shared_ptr executor) {} void hiredisAsyncClient::putKeyAsync(std::string_view key, diff --git a/src/plugins/kv/redis/client.h b/src/plugins/kv/redis/client.h index baad2d7faa..1912b2ab7c 100644 --- a/src/plugins/kv/redis/client.h +++ b/src/plugins/kv/redis/client.h @@ -58,12 +58,12 @@ class hiredisAsyncClient : public iRedisClient { * @param executor Executor used to complete async promises. */ hiredisAsyncClient(nixl_b_params_t *custom_params, - std::shared_ptr executor = nullptr); + std::shared_ptr executor = nullptr); ~hiredisAsyncClient() override; void - setExecutor(std::shared_ptr executor) override; + setExecutor(std::shared_ptr executor) override; void putKeyAsync(std::string_view key, @@ -82,7 +82,7 @@ class hiredisAsyncClient : public iRedisClient { private: struct CallbackContext { - std::shared_ptr executor; + std::shared_ptr executor; uintptr_t data_ptr; size_t data_len; std::shared_ptr> promise_ptr; @@ -113,7 +113,7 @@ class hiredisAsyncClient : public iRedisClient { redisAsyncContext *asyncContext_; redisContext *syncContext_; struct event_base *eventBase_; - std::shared_ptr executor_; + std::shared_ptr executor_; std::thread eventLoopThread_; bool connected_; mutable std::mutex syncMutex_; diff --git a/src/plugins/kv/redis/engine_impl.cpp b/src/plugins/kv/redis/engine_impl.cpp index 56ca779261..46ac765bce 100644 --- a/src/plugins/kv/redis/engine_impl.cpp +++ b/src/plugins/kv/redis/engine_impl.cpp @@ -116,14 +116,14 @@ class nixlRedisMetadata : public nixlBackendMD { } // namespace nixlRedisKVEngineImpl::nixlRedisKVEngineImpl(const nixlBackendInitParams *init_params) - : executor_(std::make_shared(getNumThreads(init_params->customParams))) { + : executor_(std::make_shared(getNumThreads(init_params->customParams))) { redisClient_ = std::make_shared(init_params->customParams, executor_); NIXL_INFO << "Redis KV backend initialized"; } nixlRedisKVEngineImpl::nixlRedisKVEngineImpl(const nixlBackendInitParams *init_params, std::shared_ptr redis_client) - : executor_(std::make_shared(std::thread::hardware_concurrency())), + : executor_(std::make_shared(std::thread::hardware_concurrency())), redisClient_(std::move(redis_client)) { if (redisClient_) { redisClient_->setExecutor(executor_); diff --git a/src/plugins/kv/redis/engine_impl.h b/src/plugins/kv/redis/engine_impl.h index 0209ebf48d..4bf2747c13 100644 --- a/src/plugins/kv/redis/engine_impl.h +++ b/src/plugins/kv/redis/engine_impl.h @@ -81,7 +81,7 @@ class nixlRedisKVEngineImpl : public nixlKVEngineImpl { return redisClient_.get(); } - std::shared_ptr executor_; + std::shared_ptr executor_; std::shared_ptr redisClient_; std::unordered_map devIdToRedisKey_; }; diff --git a/src/plugins/kv/redis/redis_engine.h b/src/plugins/kv/redis/redis_engine.h index 2192089cf3..0055821ac2 100644 --- a/src/plugins/kv/redis/redis_engine.h +++ b/src/plugins/kv/redis/redis_engine.h @@ -48,7 +48,7 @@ class iRedisClient { * @param executor ASIO thread pool shared with the engine implementation. */ virtual void - setExecutor(std::shared_ptr executor) = 0; + setExecutor(std::shared_ptr executor) = 0; /** * @brief Asynchronously store a value under key (Redis SET). diff --git a/src/plugins/kv/redis/redis_executor.h b/src/plugins/kv/redis/redis_executor.h index 68c1a7582a..986b58cb29 100644 --- a/src/plugins/kv/redis/redis_executor.h +++ b/src/plugins/kv/redis/redis_executor.h @@ -23,100 +23,30 @@ #include /** - * @class asioThreadPoolExecutor - * @brief ASIO-based thread pool executor for asynchronous task execution - * - * This executor uses ASIO's thread_pool to execute tasks asynchronously across - * multiple threads. It provides a simple interface for posting tasks that will - * be executed by worker threads in the pool. - * - * ASIO Design Pattern: - * - ASIO (Asio C++ library) provides asynchronous I/O operations - * - thread_pool creates a pool of worker threads that process tasks - * - post() submits tasks to the pool's work queue - * - Tasks are executed by worker threads asynchronously (non-blocking) - * - * Why use ASIO executor here? - * - Redis async callbacks need to be executed in a separate thread context - * - Prevents blocking the main thread or hiredis event loop thread - * - Allows concurrent execution of multiple Redis operations - * - Provides thread-safe task queuing and execution + * @class redisThreadPoolExecutor + * @brief Small ASIO thread pool used to complete Redis async operations. */ -class asioThreadPoolExecutor { +class redisThreadPoolExecutor { public: - /** - * @brief Construct a thread pool executor with specified number of threads - * @param num_threads Number of worker threads in the pool - * - * Example: asioThreadPoolExecutor(4) creates a pool with 4 worker threads - * These threads will continuously process tasks from the work queue - */ - explicit asioThreadPoolExecutor(std::size_t num_threads) : pool_(num_threads) { - // ASIO thread_pool automatically starts worker threads - // Each thread runs an event loop that processes tasks from the queue - } + explicit redisThreadPoolExecutor(std::size_t num_threads) : pool_(num_threads) {} - /** - * @brief Stop the thread pool and wait for all threads to finish - * - * This is called during Redis backend destruction to ensure all - * pending tasks complete before cleanup. - */ void WaitUntilStopped() { - pool_.stop(); // Signal all threads to stop processing new tasks - pool_.join(); // Wait for all threads to finish current tasks and exit + pool_.stop(); + pool_.join(); } - /** - * @brief Wait until all queued tasks are completed - * - * Blocks until the work queue is empty and all tasks have been processed. - * Useful for synchronization during shutdown. - */ void waitUntilIdle() { - pool_.wait(); // Wait for all tasks in the queue to complete + pool_.wait(); } - /** - * @brief Post a task to the thread pool for asynchronous execution - * @param task Function object to execute asynchronously - * - * ASIO post() semantics: - * - Non-blocking: returns immediately after queuing the task - * - Asynchronous: task executes in a worker thread, not the caller's thread - * - Thread-safe: multiple threads can post tasks concurrently - * - Ordering: tasks are executed in FIFO order per thread - * - * Execution flow: - * 1. Caller thread: post(task) -> returns immediately - * 2. Task is queued in ASIO's internal work queue - * 3. Worker thread picks up task from queue - * 4. Worker thread executes task asynchronously - * - * Example usage: - * executor->post([promise_ptr, success]() { - * promise_ptr->set_value(success ? NIXL_SUCCESS : NIXL_ERR_BACKEND); - * }); - * - * This lambda will execute in a worker thread, not the caller's thread. - */ void post(std::function &&task) { - // asio::post schedules the task for execution in the thread pool - // The task will be executed by one of the worker threads asio::post(pool_, std::move(task)); } private: - /** - * ASIO thread_pool manages a pool of worker threads - * - Automatically creates and manages worker threads - * - Provides a work queue for task distribution - * - Handles thread lifecycle (creation, execution, cleanup) - * - Ensures thread-safe task execution - */ asio::thread_pool pool_; }; From 099be214cadf3b23c470bd423c71911f1b554717 Mon Sep 17 00:00:00 2001 From: Harry Xie Date: Wed, 17 Jun 2026 03:37:55 +0300 Subject: [PATCH 14/24] fix(kv): harden redis async transfer handling --- src/plugins/kv/redis/client.cpp | 18 ++++---- src/plugins/kv/redis/client.h | 7 +++- src/plugins/kv/redis/engine_impl.cpp | 61 +++++++++++++++++++++++++--- 3 files changed, 71 insertions(+), 15 deletions(-) diff --git a/src/plugins/kv/redis/client.cpp b/src/plugins/kv/redis/client.cpp index adb19a5c49..6fe4849dcc 100644 --- a/src/plugins/kv/redis/client.cpp +++ b/src/plugins/kv/redis/client.cpp @@ -169,11 +169,11 @@ hiredisAsyncClient::hiredisAsyncClient(nixl_b_params_t *custom_params, eventLoopThread_ = std::thread(&hiredisAsyncClient::processEventLoop, this); int retries = 100; - while (!connected_ && retries-- > 0) { + while (!connected_.load() && retries-- > 0) { std::this_thread::sleep_for(std::chrono::milliseconds(10)); } - if (!connected_) { + if (!connected_.load()) { redisAsyncDisconnect(asyncContext_); event_base_loopbreak(eventBase_); if (eventLoopThread_.joinable()) { @@ -271,9 +271,9 @@ hiredisAsyncClient::connectCallback(const redisAsyncContext *c, int status) { auto *client = static_cast(c->data); if (status != REDIS_OK) { NIXL_ERROR << absl::StrFormat("Redis connection error: %s", c->errstr); - client->connected_ = false; + client->connected_.store(false); } else { - client->connected_ = true; + client->connected_.store(true); } } @@ -283,7 +283,7 @@ hiredisAsyncClient::disconnectCallback(const redisAsyncContext *c, int status) { if (status != REDIS_OK) { NIXL_WARN << absl::StrFormat("Redis disconnection error: %s", c->errstr); } - client->connected_ = false; + client->connected_.store(false); } void @@ -312,7 +312,9 @@ hiredisAsyncClient::getCallback(redisAsyncContext *c, void *reply, void *privdat bool success = false; if (r && r->type == REDIS_REPLY_STRING) { size_t copy_len = std::min(static_cast(r->len), ctx->data_len); - if (ctx->data_ptr && copy_len > 0) { + if (copy_len == 0) { + success = true; + } else if (ctx->data_ptr) { std::memcpy(reinterpret_cast(ctx->data_ptr), r->str, copy_len); success = true; } @@ -333,7 +335,7 @@ hiredisAsyncClient::putKeyAsync(std::string_view key, uintptr_t data_ptr, size_t data_len, std::shared_ptr> promise) { - if (!connected_) { + if (!connected_.load()) { setPromiseStatus(executor_, promise, false); return; } @@ -363,7 +365,7 @@ hiredisAsyncClient::getKeyAsync(std::string_view key, uintptr_t data_ptr, size_t data_len, std::shared_ptr> promise) { - if (!connected_) { + if (!connected_.load()) { setPromiseStatus(executor_, promise, false); return; } diff --git a/src/plugins/kv/redis/client.h b/src/plugins/kv/redis/client.h index 1912b2ab7c..cb2d1be75b 100644 --- a/src/plugins/kv/redis/client.h +++ b/src/plugins/kv/redis/client.h @@ -28,6 +28,7 @@ #define KV_PLUGIN_REDIS_CLIENT_H #include "redis_engine.h" +#include #include #include #include @@ -44,6 +45,10 @@ #include #include #include +#else +struct event_base; +struct redisAsyncContext; +struct redisContext; #endif /** @@ -115,7 +120,7 @@ class hiredisAsyncClient : public iRedisClient { struct event_base *eventBase_; std::shared_ptr executor_; std::thread eventLoopThread_; - bool connected_; + std::atomic connected_; mutable std::mutex syncMutex_; }; diff --git a/src/plugins/kv/redis/engine_impl.cpp b/src/plugins/kv/redis/engine_impl.cpp index 46ac765bce..8bc697900d 100644 --- a/src/plugins/kv/redis/engine_impl.cpp +++ b/src/plugins/kv/redis/engine_impl.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -35,11 +36,32 @@ namespace { +std::size_t +defaultNumThreads() { + return std::max(1u, std::thread::hardware_concurrency() / 2); +} + std::size_t getNumThreads(nixl_b_params_t *custom_params) { - return custom_params && custom_params->count("num_threads") > 0 ? - std::stoul(custom_params->at("num_threads")) : - std::max(1u, std::thread::hardware_concurrency() / 2); + if (custom_params && custom_params->count("num_threads") > 0) { + try { + const auto num_threads = std::stoul(custom_params->at("num_threads")); + if (num_threads > 0) { + return num_threads; + } + NIXL_WARN << "Invalid num_threads value 0, using default"; + } + catch (const std::exception &) { + NIXL_WARN << "Invalid num_threads value, using default"; + } + } + + return defaultNumThreads(); +} + +nixl_b_params_t * +getCustomParams(const nixlBackendInitParams *init_params) { + return init_params ? init_params->customParams : nullptr; } bool @@ -53,6 +75,19 @@ isValidPrepXferParams(const nixl_xfer_op_t &operation, return false; } + if (local.descCount() == 0) { + NIXL_ERROR << "Error: Transfer descriptor lists must not be empty"; + return false; + } + + if (local.descCount() != remote.descCount()) { + NIXL_ERROR << absl::StrFormat( + "Error: Local and remote descriptor counts must match (%d != %d)", + local.descCount(), + remote.descCount()); + return false; + } + if (remote_agent != local_agent) NIXL_WARN << absl::StrFormat( "Warning: Remote agent doesn't match the requesting agent (%s). Got %s", @@ -116,14 +151,16 @@ class nixlRedisMetadata : public nixlBackendMD { } // namespace nixlRedisKVEngineImpl::nixlRedisKVEngineImpl(const nixlBackendInitParams *init_params) - : executor_(std::make_shared(getNumThreads(init_params->customParams))) { - redisClient_ = std::make_shared(init_params->customParams, executor_); + : executor_(std::make_shared( + getNumThreads(getCustomParams(init_params)))) { + redisClient_ = std::make_shared(getCustomParams(init_params), executor_); NIXL_INFO << "Redis KV backend initialized"; } nixlRedisKVEngineImpl::nixlRedisKVEngineImpl(const nixlBackendInitParams *init_params, std::shared_ptr redis_client) - : executor_(std::make_shared(std::thread::hardware_concurrency())), + : executor_(std::make_shared( + getNumThreads(getCustomParams(init_params)))), redisClient_(std::move(redis_client)) { if (redisClient_) { redisClient_->setExecutor(executor_); @@ -237,6 +274,18 @@ nixlRedisKVEngineImpl::postXfer(const nixl_xfer_op_t &operation, return NIXL_ERR_INVALID_PARAM; } + if (operation != NIXL_WRITE && operation != NIXL_READ) { + return NIXL_ERR_INVALID_PARAM; + } + + if (local.descCount() == 0 || local.descCount() != remote.descCount()) { + NIXL_ERROR << absl::StrFormat( + "Invalid transfer descriptor counts for Redis postXfer (%d local, %d remote)", + local.descCount(), + remote.descCount()); + return NIXL_ERR_INVALID_PARAM; + } + auto *req_h = static_cast(handle); iRedisClient *client = getClient(); if (!client) { From c997861903bc30a2de90e68f472e055b5bfc1a45 Mon Sep 17 00:00:00 2001 From: Harry Xie Date: Wed, 17 Jun 2026 03:38:20 +0300 Subject: [PATCH 15/24] docs(kv): expand kv redis plugin guide --- src/plugins/kv/README.md | 181 +++++++++++++++++++++++++++++++-- src/plugins/kv/redis/README.md | 11 +- 2 files changed, 181 insertions(+), 11 deletions(-) diff --git a/src/plugins/kv/README.md b/src/plugins/kv/README.md index 8d0873cdc4..f8d8a5f053 100644 --- a/src/plugins/kv/README.md +++ b/src/plugins/kv/README.md @@ -1,9 +1,178 @@ -# NIXL KV Backend Interfaces + + +# NIXL Key-Value Storage Plugin + +The KV plugin layer provides a NIXL backend facade for key-value storage. The shared code in this +directory follows the same facade/implementation split as the object backend, while concrete +implementations live in backend-specific subdirectories. + +The current production implementation is the `REDIS` plugin under `redis/`. + +## Architecture + +The shared KV layer owns the NIXL-facing backend wrapper and implementation interface: + +- **KV backend wrapper** (`nixlKVEngine`): The class registered with the NIXL backend plugin loader. +- **KV implementation base** (`nixlKVEngineImpl`): The backend-facing interface implemented by concrete KV stores. +- **Simple KV store interface** (`iKVStore`): A synchronous put/get/exists helper interface for simple KV implementations. + +The Redis implementation uses a richer client interface because `postXfer` is asynchronous: + +- **Redis engine** (`nixlRedisKVEngine`): The REDIS plugin engine registered by `kv_plugin.cpp`. +- **Redis implementation** (`nixlRedisKVEngineImpl`): Maps NIXL transfers to Redis SET/GET/EXISTS operations. +- **Redis client** (`iRedisClient`, `hiredisAsyncClient`): Uses an async hiredis connection for SET/GET and a + separate synchronous connection for EXISTS. +- **Redis executor** (`redisThreadPoolExecutor`): Completes async promises away from the hiredis/libevent callback path. + +``` +kv/ + kv_backend.h/.cpp shared nixlKVEngine, nixlKVEngineImpl, iKVStore + kv_plugin.cpp registers the REDIS backend with the NIXL plugin loader + meson.build includes backend subdirectories + +kv/redis/ + redis_engine.h/.cpp iRedisClient and nixlRedisKVEngine + engine_impl.h/.cpp nixlRedisKVEngineImpl transfer logic + client.h/.cpp hiredisAsyncClient + redis_executor.h Redis-specific ASIO executor + meson.build REDIS plugin target + README.md Redis-specific build and benchmark notes +``` + +## Dependencies + +The shared KV facade has no storage-system dependency. The REDIS backend requires: + +| Dependency | Used for | +|------------|----------| +| `hiredis` / `hiredis_async` | Redis command encoding, sync EXISTS, async SET/GET | +| `libevent` | Event loop integration for hiredis async | +| `libevent_pthreads` | Thread-safe libevent setup | +| `asio` | Thread pool used by the Redis executor | + +If REDIS is enabled but hiredis or libevent is not found, the dynamic plugin build is skipped. If +`REDIS` is requested as a static plugin, missing hiredis/libevent is treated as a configuration +error. + +## Configuration + +Backend parameters are passed as a `nixl_b_params_t` map when creating the backend. Backend +parameters take precedence over environment variables. + +| Parameter | Environment fallback | Default | Description | +|-----------|----------------------|---------|-------------| +| `host` | `REDIS_HOST` | `localhost` | Redis server hostname or IP address | +| `port` | `REDIS_PORT` | `6379` | Redis server TCP port | +| `password` | `REDIS_PASSWORD` | empty | Redis AUTH password | +| `db` | - | `0` | Redis logical database number | +| `num_threads` | - | half of hardware concurrency, minimum 1 | ASIO worker threads for async completion | + +### Configuration Examples + +#### Local Redis + +```cpp +nixl_b_params_t params = { + {"host", "127.0.0.1"}, + {"port", "6379"} +}; +agent.createBackend("REDIS", params); +``` + +#### Authenticated Redis + +```cpp +nixl_b_params_t params = { + {"host", "redis.internal"}, + {"port", "6379"}, + {"password", "example-password"}, + {"db", "2"}, + {"num_threads", "4"} +}; +agent.createBackend("REDIS", params); +``` + +#### Environment Variable Configuration + +```bash +export REDIS_HOST=127.0.0.1 +export REDIS_PORT=6379 +export REDIS_PASSWORD=example-password +``` + +```cpp +nixl_b_params_t params = {}; +agent.createBackend("REDIS", params); +``` + +## Transfer Operations + +The REDIS backend supports `NIXL_WRITE` and `NIXL_READ`. + +| NIXL operation | Redis operation | Behavior | +|----------------|-----------------|----------| +| `NIXL_WRITE` | `SET key bytes` | Stores local DRAM bytes under the remote Redis key | +| `NIXL_READ` | `GET key` | Reads the Redis value into the local DRAM descriptor | +| `queryMem` | `EXISTS key` | Reports whether a Redis key exists | + +Local descriptors must be `DRAM_SEG`. Remote descriptors may be `DRAM_SEG` or `OBJ_SEG`, matching +the segment list exposed by the plugin. The Redis key is chosen from descriptor metadata: + +1. Use `metaInfo` when it is present. +2. Otherwise use the descriptor `devId` converted to a string. + +`prepXfer` validates operation type, local/remote descriptor counts, and supported segment types. +`postXfer` creates one promise/future pair per descriptor and returns `NIXL_IN_PROG`. `checkXfer` +polls those futures until all Redis async operations complete. + +## Build + +Build only the REDIS plugin: + +```bash +meson setup build -Denable_plugins=REDIS +ninja -C build src/plugins/kv/redis/libplugin_REDIS.so +``` + +For static plugin builds: + +```bash +meson setup build -Denable_plugins=REDIS -Dstatic_plugins=REDIS +ninja -C build +``` + +The dynamic plugin output is: + +```text +build/src/plugins/kv/redis/libplugin_REDIS.so +``` + +## Extending KV Backends + +New KV implementations should follow the same split used by Redis: + +1. Add a backend-specific subdirectory under `src/plugins/kv/`. +2. Implement `nixlKVEngineImpl` for NIXL registration, query, prep, post, check, and request-handle cleanup. +3. Provide a small client/store interface for backend-specific storage operations. +4. Add a Meson subdirectory and plugin source list. +5. Register the plugin from a top-level `kv_plugin.cpp`-style entrypoint when the implementation should be exposed as a NIXL plugin. + +There is no generic `kv_executor.h` today. Redis keeps `redis_executor.h` because the executor is +only used by the Redis async client, and OBJ's executor inherits from an AWS SDK executor interface. +If another KV implementation needs the same ASIO-only executor, it would be reasonable to promote +that type into a shared `kv_executor.h` at that point. diff --git a/src/plugins/kv/redis/README.md b/src/plugins/kv/redis/README.md index eacaacf475..0ec70c0f24 100644 --- a/src/plugins/kv/redis/README.md +++ b/src/plugins/kv/redis/README.md @@ -5,17 +5,18 @@ SPDX-License-Identifier: Apache-2.0 # NIXL Redis Plugin (KV) -All REDIS plugin code lives in `src/plugins/kv/redis/`. Shared KV abstractions stay in `src/plugins/kv/`. +The REDIS implementation lives in `src/plugins/kv/redis/`. Shared KV abstractions and the plugin +entrypoint stay in `src/plugins/kv/`. ``` kv/ # shared KV layer (PR1 interface) kv_backend.h/.cpp nixlKVEngine, nixlKVEngineImpl, iKVStore + kv_plugin.cpp registers "REDIS" meson.build delegates to redis/ when REDIS enabled kv/redis/ # REDIS plugin (this directory) redis_engine.h/.cpp iRedisClient + nixlRedisKVEngine - redis_plugin.cpp registers "REDIS" - redis_executor.h ASIO thread pool + redis_executor.h Redis-specific ASIO thread pool client.h/.cpp hiredisAsyncClient engine_impl.h/.cpp nixlRedisKVEngineImpl meson.build @@ -46,7 +47,7 @@ kv/redis/ # REDIS plugin (this directory) ```cpp class iRedisClient { - void setExecutor(std::shared_ptr executor); + void setExecutor(std::shared_ptr executor); // postXfer path (async connection) void putKeyAsync(..., std::shared_ptr> promise); @@ -82,7 +83,7 @@ queryMem (sync path): ### Native (plugin only) ```bash -meson setup build -Dplugins=REDIS +meson setup build -Denable_plugins=REDIS ninja -C build # -> build/src/plugins/kv/redis/libplugin_REDIS.so ``` From d8515896ae8cbe67d42f05a604f3c2188ea0bb49 Mon Sep 17 00:00:00 2001 From: Harry Xie Date: Fri, 26 Jun 2026 03:13:11 +0300 Subject: [PATCH 16/24] Address Redis KV review feedback --- benchmark/nixlbench/contrib/Dockerfile | 9 +- benchmark/nixlbench/src/utils/utils.cpp | 166 ++++++++-- .../nixlbench/src/worker/nixl/nixl_worker.cpp | 4 +- src/plugins/kv/README.md | 6 +- src/plugins/kv/kv_backend.cpp | 4 + src/plugins/kv/kv_backend.h | 8 +- src/plugins/kv/redis/README.md | 3 +- src/plugins/kv/redis/client.cpp | 303 ++++++++++++++---- src/plugins/kv/redis/client.h | 33 +- src/plugins/kv/redis/engine_impl.cpp | 22 +- src/plugins/kv/redis/engine_impl.h | 18 +- src/plugins/kv/redis/meson.build | 6 +- src/plugins/kv/redis/redis_executor.h | 2 +- test/gtest/unit/meson.build | 5 + test/gtest/unit/redis/meson.build | 29 ++ test/gtest/unit/redis/redis.cpp | 134 ++++++++ 16 files changed, 614 insertions(+), 138 deletions(-) create mode 100644 test/gtest/unit/redis/meson.build create mode 100644 test/gtest/unit/redis/redis.cpp diff --git a/benchmark/nixlbench/contrib/Dockerfile b/benchmark/nixlbench/contrib/Dockerfile index 4823b1fa04..e7cc9062c4 100644 --- a/benchmark/nixlbench/contrib/Dockerfile +++ b/benchmark/nixlbench/contrib/Dockerfile @@ -27,7 +27,7 @@ FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG} AS os_setup_stage ARG ARCH="x86_64" ARG DEFAULT_PYTHON_VERSION RUN apt-get update -y && \ - DEBIAN_FRONTEND=noninteractive apt-get -y install \ + DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends \ ninja-build \ pybind11-dev \ libclang-dev \ @@ -50,7 +50,8 @@ RUN apt-get update -y && \ build-essential \ libhiredis-dev \ libevent-dev \ - redis-tools + redis-tools && \ + rm -rf /var/lib/apt/lists/* # Add DOCA repo + upgrade always so the RDMA reinstall below resolves MLNX/DOCA versions. # Skip only the DOCA SDK install when the base image already ships it. @@ -126,8 +127,8 @@ ARG LIBFABRIC_VERSION="v1.21.0" ARG NPROC ARG ABSL_TAG="lts_2025_08_14" ARG GRPC_TAG="v1.73.0" -ARG NIXL_ENABLE_PLUGINS="REDIS" -ARG NIXL_STATIC_PLUGINS="REDIS" +ARG NIXL_ENABLE_PLUGINS="" +ARG NIXL_STATIC_PLUGINS="" WORKDIR /workspace diff --git a/benchmark/nixlbench/src/utils/utils.cpp b/benchmark/nixlbench/src/utils/utils.cpp index 092454573d..2301286172 100644 --- a/benchmark/nixlbench/src/utils/utils.cpp +++ b/benchmark/nixlbench/src/utils/utils.cpp @@ -41,6 +41,125 @@ #include "utils/neuron.h" #include "utils/utils.h" +namespace { + +bool +parseRedisBenchPort(int &port) { + port = 6379; + const char *port_env = getenv("REDIS_PORT"); + if (!port_env) { + return true; + } + + try { + size_t parsed = 0; + int parsed_port = std::stoi(port_env, &parsed); + if (parsed != std::string(port_env).size() || parsed_port <= 0 || parsed_port > 65535) { + std::cerr << "Invalid REDIS_PORT value: " << port_env << std::endl; + return false; + } + port = parsed_port; + return true; + } + catch (const std::exception &) { + std::cerr << "Invalid REDIS_PORT value: " << port_env << std::endl; + return false; + } +} + +redisContext * +connectRedisBench(const char *operation) { + const char *host = getenv("REDIS_HOST"); + if (!host) { + host = "127.0.0.1"; + } + + int port = 6379; + if (!parseRedisBenchPort(port)) { + return nullptr; + } + + struct timeval timeout = {5, 0}; + redisContext *ctx = redisConnectWithTimeout(host, port, timeout); + if (!ctx || ctx->err) { + std::cerr << "Redis connect failed for " << operation << ": " + << (ctx ? ctx->errstr : "null context") << std::endl; + if (ctx) { + redisFree(ctx); + } + return nullptr; + } + + return ctx; +} + +bool +authRedisBench(redisContext *ctx, const char *operation) { + const char *password = getenv("REDIS_PASSWORD"); + if (!password || password[0] == '\0') { + return true; + } + + redisReply *auth_reply = (redisReply *)redisCommand(ctx, "AUTH %s", password); + if (!auth_reply || auth_reply->type == REDIS_REPLY_ERROR) { + std::cerr << "Redis AUTH failed during " << operation << std::endl; + if (auth_reply) { + freeReplyObject(auth_reply); + } + return false; + } + freeReplyObject(auth_reply); + return true; +} + +bool +getRedisBench(size_t buffer_size, const std::string &key, void *buffer) { + if (buffer_size > 0 && !buffer) { + std::cerr << "Invalid Redis GET output buffer for key: " << key << std::endl; + return false; + } + + redisContext *ctx = connectRedisBench("consistency GET"); + if (!ctx) { + return false; + } + + if (!authRedisBench(ctx, "consistency GET")) { + redisFree(ctx); + return false; + } + + redisReply *reply = + (redisReply *)redisCommand(ctx, "GET %b", key.data(), key.size()); + bool success = false; + if (reply && reply->type == REDIS_REPLY_STRING) { + const size_t reply_len = static_cast(reply->len); + if (reply_len == buffer_size) { + if (buffer_size > 0) { + memcpy(buffer, reply->str, buffer_size); + } + success = true; + } else { + std::cerr << "Redis GET size mismatch for key " << key << ": expected " + << buffer_size << " bytes, got " << reply_len << " bytes" << std::endl; + } + } else if (reply && reply->type == REDIS_REPLY_NIL) { + std::cerr << "Redis GET failed, key not found: " << key << std::endl; + } else if (reply && reply->type == REDIS_REPLY_ERROR) { + std::cerr << "Redis GET error for key " << key << ": " << reply->str << std::endl; + } else { + std::cerr << "Redis GET failed for key " << key << std::endl; + } + + if (reply) { + freeReplyObject(reply); + } + redisFree(ctx); + return success; +} + +} // namespace + // Define command line parameters #define NB_ARG_STRING(param_name, def_val, help_text) DEFINE_string(param_name, def_val, help_text) #define NB_ARG_BOOL(param_name, def_val, help_text) DEFINE_bool(param_name, def_val, help_text) @@ -680,8 +799,9 @@ xferBenchConfig::printConfig() { } printOption("Worker type (--worker_type=[nixl,nvshmem])", worker_type); if (worker_type == XFERBENCH_WORKER_NIXL) { - printOption("Backend (--backend=[UCX,GDS,GDS_MT,POSIX,Mooncake,HF3FS,OBJ,REDIS,AZURE_BLOB])", - backend); + printOption( + "Backend (--backend=[UCX,GDS,GDS_MT,POSIX,Mooncake,HF3FS,OBJ,REDIS,GUSLI,AZURE_BLOB])", + backend); printOption("Enable pt (--enable_pt=[0,1])", std::to_string(enable_pt)); printOption("Progress threads (--progress_threads=N)", std::to_string(progress_threads)); printOption("Device list (--device_list=dev1,dev2,...)", device_list); @@ -1080,6 +1200,11 @@ xferBenchUtils::checkConsistency(std::vector> &iov_lis exit(EXIT_FAILURE); } close(fd); + } else if (xferBenchConfig::backend == XFERBENCH_BACKEND_REDIS) { + if (!getRedisBench(len, iov.metaInfo, addr)) { + std::cerr << "Failed to get Redis key: " << iov.metaInfo << std::endl; + exit(EXIT_FAILURE); + } } else { ssize_t rc = pread(iov.devId, addr, len, iov.addr); if (rc < 0) { @@ -1312,16 +1437,6 @@ xferBenchUtils::putRedis(size_t buffer_size, const std::string &key) { return false; } - const char *host = getenv("REDIS_HOST"); - if (!host) { - host = "127.0.0.1"; - } - int port = 6379; - const char *port_env = getenv("REDIS_PORT"); - if (port_env) { - port = std::atoi(port_env); - } - void *buf = nullptr; const size_t align = xferBenchConfig::page_size > 0 ? xferBenchConfig::page_size : 4096; if (posix_memalign(&buf, align, buffer_size) != 0) { @@ -1330,31 +1445,16 @@ xferBenchUtils::putRedis(size_t buffer_size, const std::string &key) { } memset(buf, XFERBENCH_TARGET_BUFFER_ELEMENT, buffer_size); - redisContext *ctx = redisConnect(host, port); - if (!ctx || ctx->err) { - std::cerr << "Redis connect failed for seed SET: " - << (ctx ? ctx->errstr : "null context") << std::endl; - if (ctx) { - redisFree(ctx); - } + redisContext *ctx = connectRedisBench("seed SET"); + if (!ctx) { free(buf); return false; } - const char *password = getenv("REDIS_PASSWORD"); - if (password && password[0] != '\0') { - redisReply *auth_reply = - (redisReply *)redisCommand(ctx, "AUTH %s", password); - if (!auth_reply || auth_reply->type == REDIS_REPLY_ERROR) { - std::cerr << "Redis AUTH failed during seed SET" << std::endl; - if (auth_reply) { - freeReplyObject(auth_reply); - } - redisFree(ctx); - free(buf); - return false; - } - freeReplyObject(auth_reply); + if (!authRedisBench(ctx, "seed SET")) { + redisFree(ctx); + free(buf); + return false; } redisReply *reply = (redisReply *)redisCommand(ctx, diff --git a/benchmark/nixlbench/src/worker/nixl/nixl_worker.cpp b/benchmark/nixlbench/src/worker/nixl/nixl_worker.cpp index 7860f19053..d72d1a1867 100644 --- a/benchmark/nixlbench/src/worker/nixl/nixl_worker.cpp +++ b/benchmark/nixlbench/src/worker/nixl/nixl_worker.cpp @@ -1040,7 +1040,7 @@ xferBenchNixlWorker::allocateMemory(int num_threads) { const size_t seed_size = xferBenchConfig::max_block_size; if (!xferBenchUtils::putRedis(seed_size, unique_name)) { std::cerr << "Failed to seed Redis key: " << unique_name << std::endl; - continue; + exit(EXIT_FAILURE); } } @@ -1261,7 +1261,7 @@ xferBenchNixlWorker::exchangeIOV(const std::vector> &l } else if (XFERBENCH_BACKEND_REDIS == xferBenchConfig::backend) { xferBenchIOV redis_remote(iov); redis_remote.addr = 0; - redis_remote.metaInfo = iov.metaInfo; + redis_remote.len = block_size; remote_iov_list.push_back(redis_remote); } else if (XFERBENCH_BACKEND_GUSLI == xferBenchConfig::backend) { xferBenchIOV iov_remote(iov); diff --git a/src/plugins/kv/README.md b/src/plugins/kv/README.md index f8d8a5f053..e3894616da 100644 --- a/src/plugins/kv/README.md +++ b/src/plugins/kv/README.md @@ -65,9 +65,9 @@ The shared KV facade has no storage-system dependency. The REDIS backend require | `libevent_pthreads` | Thread-safe libevent setup | | `asio` | Thread pool used by the Redis executor | -If REDIS is enabled but hiredis or libevent is not found, the dynamic plugin build is skipped. If -`REDIS` is requested as a static plugin, missing hiredis/libevent is treated as a configuration -error. +If REDIS is explicitly enabled or requested as a static plugin, missing hiredis/libevent is treated +as a configuration error. When building all plugins by default, a missing Redis dependency skips the +dynamic REDIS plugin with a warning. ## Configuration diff --git a/src/plugins/kv/kv_backend.cpp b/src/plugins/kv/kv_backend.cpp index 680e569388..74af4284ce 100644 --- a/src/plugins/kv/kv_backend.cpp +++ b/src/plugins/kv/kv_backend.cpp @@ -17,11 +17,15 @@ #include "kv_backend.h" #include +#include nixlKVEngine::nixlKVEngine(const nixlBackendInitParams *init_params, std::unique_ptr impl) : nixlBackendEngine(init_params), impl_(std::move(impl)) { + if (!impl_) { + throw std::invalid_argument("nixlKVEngine requires a non-null nixlKVEngineImpl"); + } assert(impl_ && "nixlKVEngine requires a non-null nixlKVEngineImpl"); } diff --git a/src/plugins/kv/kv_backend.h b/src/plugins/kv/kv_backend.h index b6cad3b93e..fa16fae2ba 100644 --- a/src/plugins/kv/kv_backend.h +++ b/src/plugins/kv/kv_backend.h @@ -15,8 +15,8 @@ * limitations under the License. */ -#ifndef KV_BACKEND_H -#define KV_BACKEND_H +#ifndef NIXL_SRC_PLUGINS_KV_KV_BACKEND_H +#define NIXL_SRC_PLUGINS_KV_KV_BACKEND_H #include "backend/backend_engine.h" #include @@ -76,7 +76,7 @@ class nixlKVEngineImpl { class nixlKVEngine : public nixlBackendEngine { public: - virtual ~nixlKVEngine(); + ~nixlKVEngine() override; bool supportsRemote() const override { @@ -158,4 +158,4 @@ class nixlKVEngine : public nixlBackendEngine { std::unique_ptr impl_; }; -#endif // KV_BACKEND_H +#endif // NIXL_SRC_PLUGINS_KV_KV_BACKEND_H diff --git a/src/plugins/kv/redis/README.md b/src/plugins/kv/redis/README.md index 0ec70c0f24..73fcd574d2 100644 --- a/src/plugins/kv/redis/README.md +++ b/src/plugins/kv/redis/README.md @@ -58,7 +58,7 @@ class iRedisClient { }; ``` -Connection settings (constructor): `host`, `port`, `password`, `db` via NIXL custom params or env vars `REDIS_HOST`, `REDIS_PORT`, `REDIS_PASSWORD`. +Connection settings (constructor): `host`, `port`, and `password` can come from NIXL custom params or `REDIS_HOST`, `REDIS_PORT`, and `REDIS_PASSWORD`; `db` is a NIXL custom param only. ## Transfer call path @@ -156,6 +156,7 @@ Expected log lines: **Notes:** - `REDIS_HOST` / `REDIS_PORT` / `REDIS_PASSWORD` are read by `hiredisAsyncClient` at plugin init. +- Redis `db` selection is configured only through the NIXL `db` custom param. - nixlbench may adjust `--num_iter` / `--warmup_iter` for thread alignment (warnings are normal). - For a one-shot run without an interactive shell, pass `nixlbench ...` as the `docker run` command instead of `bash`. diff --git a/src/plugins/kv/redis/client.cpp b/src/plugins/kv/redis/client.cpp index 6fe4849dcc..33f2a79213 100644 --- a/src/plugins/kv/redis/client.cpp +++ b/src/plugins/kv/redis/client.cpp @@ -20,13 +20,23 @@ #include #include #include +#include #include #include +#include #ifdef HAVE_HIREDIS_ASYNC namespace { +using redis_event_task_t = std::function; + +void +runEventTask(evutil_socket_t, short, void *arg) { + std::unique_ptr task(static_cast(arg)); + (*task)(); +} + std::string getRedisHost(nixl_b_params_t *custom_params) { if (custom_params && custom_params->count("host") > 0) { @@ -36,23 +46,37 @@ getRedisHost(nixl_b_params_t *custom_params) { return env_host ? std::string(env_host) : "localhost"; } +std::optional +parseRedisPort(const std::string &value, const char *source) { + try { + size_t parsed = 0; + int port = std::stoi(value, &parsed); + if (parsed != value.size() || port <= 0 || port > 65535) { + NIXL_WARN << absl::StrFormat( + "Invalid %s value '%s', using default 6379", source, value); + return std::nullopt; + } + return port; + } + catch (const std::exception &) { + NIXL_WARN << absl::StrFormat("Invalid %s value '%s', using default 6379", source, value); + return std::nullopt; + } +} + int getRedisPort(nixl_b_params_t *custom_params) { if (custom_params && custom_params->count("port") > 0) { - try { - return std::stoi(custom_params->at("port")); - } - catch (const std::exception &) { - NIXL_WARN << "Invalid port value, using default 6379"; + auto port = parseRedisPort(custom_params->at("port"), "Redis port"); + if (port) { + return *port; } } const char *env_port = getenv("REDIS_PORT"); if (env_port) { - try { - return std::stoi(env_port); - } - catch (const std::exception &) { - NIXL_WARN << "Invalid REDIS_PORT, using default 6379"; + auto port = parseRedisPort(env_port, "REDIS_PORT"); + if (port) { + return *port; } } return 6379; @@ -125,7 +149,9 @@ hiredisAsyncClient::hiredisAsyncClient(nixl_b_params_t *custom_params, syncContext_(nullptr), eventBase_(nullptr), executor_(executor), - connected_(false) { + connected_(false), + initDone_(false), + initSucceeded_(false) { evthread_use_pthreads(); eventBase_ = event_base_new(); @@ -159,28 +185,15 @@ hiredisAsyncClient::hiredisAsyncClient(nixl_b_params_t *custom_params, redisAsyncSetConnectCallback(asyncContext_, connectCallback); redisAsyncSetDisconnectCallback(asyncContext_, disconnectCallback); - if (!password_.empty()) { - redisAsyncCommand(asyncContext_, nullptr, nullptr, "AUTH %s", password_.c_str()); - } - if (db_ != 0) { - redisAsyncCommand(asyncContext_, nullptr, nullptr, "SELECT %d", db_); - } - eventLoopThread_ = std::thread(&hiredisAsyncClient::processEventLoop, this); int retries = 100; - while (!connected_.load() && retries-- > 0) { + while (!initDone_.load() && retries-- > 0) { std::this_thread::sleep_for(std::chrono::milliseconds(10)); } - if (!connected_.load()) { - redisAsyncDisconnect(asyncContext_); - event_base_loopbreak(eventBase_); - if (eventLoopThread_.joinable()) { - eventLoopThread_.join(); - } - redisAsyncFree(asyncContext_); - event_base_free(eventBase_); + if (!initSucceeded_.load()) { + stopEventLoop(); throw std::runtime_error("Failed to connect to Redis within timeout"); } @@ -238,21 +251,10 @@ hiredisAsyncClient::connectSyncContext() { } hiredisAsyncClient::~hiredisAsyncClient() { - if (asyncContext_) { - redisAsyncDisconnect(asyncContext_); - } - if (eventBase_) { - event_base_loopbreak(eventBase_); - if (eventLoopThread_.joinable()) { - eventLoopThread_.join(); - } - event_base_free(eventBase_); - } - if (asyncContext_) { - redisAsyncFree(asyncContext_); - } + stopEventLoop(); if (syncContext_) { redisFree(syncContext_); + syncContext_ = nullptr; } } @@ -266,14 +268,102 @@ hiredisAsyncClient::processEventLoop() { event_base_dispatch(eventBase_); } +bool +hiredisAsyncClient::scheduleOnEventLoop(std::function task) { + if (!eventBase_) { + return false; + } + + auto *owned_task = new redis_event_task_t(std::move(task)); + timeval immediate = {0, 0}; + if (event_base_once(eventBase_, -1, EV_TIMEOUT, runEventTask, owned_task, &immediate) != 0) { + delete owned_task; + return false; + } + return true; +} + +void +hiredisAsyncClient::freeAsyncContextInEventLoop() { + redisAsyncContext *ctx = asyncContext_; + asyncContext_ = nullptr; + if (ctx) { + redisAsyncFree(ctx); + } +} + +void +hiredisAsyncClient::stopEventLoop() { + connected_.store(false); + + if (eventBase_ && eventLoopThread_.joinable()) { + bool scheduled = scheduleOnEventLoop([this]() { + freeAsyncContextInEventLoop(); + event_base_loopbreak(eventBase_); + }); + if (!scheduled) { + event_base_loopbreak(eventBase_); + } + eventLoopThread_.join(); + } + + if (eventBase_) { + event_base_free(eventBase_); + eventBase_ = nullptr; + } + asyncContext_ = nullptr; +} + +void +hiredisAsyncClient::completeAsyncInitialization(bool success) { + connected_.store(success); + initSucceeded_.store(success); + initDone_.store(true); +} + +void +hiredisAsyncClient::startAsyncSelect() { + if (db_ == 0) { + completeAsyncInitialization(true); + return; + } + + int ret = redisAsyncCommand(asyncContext_, selectCallback, this, "SELECT %d", db_); + if (ret != REDIS_OK) { + NIXL_ERROR << "Failed to queue Redis SELECT command"; + completeAsyncInitialization(false); + freeAsyncContextInEventLoop(); + event_base_loopbreak(eventBase_); + } +} + +void +hiredisAsyncClient::startAsyncInitialization() { + if (!password_.empty()) { + int ret = + redisAsyncCommand(asyncContext_, authCallback, this, "AUTH %s", password_.c_str()); + if (ret != REDIS_OK) { + NIXL_ERROR << "Failed to queue Redis AUTH command"; + completeAsyncInitialization(false); + freeAsyncContextInEventLoop(); + event_base_loopbreak(eventBase_); + } + return; + } + + startAsyncSelect(); +} + void hiredisAsyncClient::connectCallback(const redisAsyncContext *c, int status) { auto *client = static_cast(c->data); if (status != REDIS_OK) { NIXL_ERROR << absl::StrFormat("Redis connection error: %s", c->errstr); - client->connected_.store(false); + client->completeAsyncInitialization(false); + client->freeAsyncContextInEventLoop(); + event_base_loopbreak(client->eventBase_); } else { - client->connected_.store(true); + client->startAsyncInitialization(); } } @@ -286,6 +376,36 @@ hiredisAsyncClient::disconnectCallback(const redisAsyncContext *c, int status) { client->connected_.store(false); } +void +hiredisAsyncClient::authCallback(redisAsyncContext *c, void *reply, void *privdata) { + auto *client = static_cast(privdata); + auto *r = static_cast(reply); + + if (!checkRedisReplyOk(r, "AUTH")) { + client->completeAsyncInitialization(false); + client->freeAsyncContextInEventLoop(); + event_base_loopbreak(client->eventBase_); + return; + } + + client->startAsyncSelect(); +} + +void +hiredisAsyncClient::selectCallback(redisAsyncContext *c, void *reply, void *privdata) { + auto *client = static_cast(privdata); + auto *r = static_cast(reply); + + if (!checkRedisReplyOk(r, "SELECT")) { + client->completeAsyncInitialization(false); + client->freeAsyncContextInEventLoop(); + event_base_loopbreak(client->eventBase_); + return; + } + + client->completeAsyncInitialization(true); +} + void hiredisAsyncClient::setCallback(redisAsyncContext *c, void *reply, void *privdata) { auto *ctx = static_cast(privdata); @@ -311,11 +431,16 @@ hiredisAsyncClient::getCallback(redisAsyncContext *c, void *reply, void *privdat bool success = false; if (r && r->type == REDIS_REPLY_STRING) { - size_t copy_len = std::min(static_cast(r->len), ctx->data_len); - if (copy_len == 0) { + const size_t reply_len = static_cast(r->len); + if (reply_len != ctx->data_len) { + NIXL_ERROR << absl::StrFormat( + "Redis GET size mismatch: expected %zu bytes, got %zu bytes", + ctx->data_len, + reply_len); + } else if (ctx->data_len == 0) { success = true; } else if (ctx->data_ptr) { - std::memcpy(reinterpret_cast(ctx->data_ptr), r->str, copy_len); + std::memcpy(reinterpret_cast(ctx->data_ptr), r->str, ctx->data_len); success = true; } } else if (r && r->type == REDIS_REPLY_NIL) { @@ -340,23 +465,41 @@ hiredisAsyncClient::putKeyAsync(std::string_view key, return; } - auto *ctx = new CallbackContext; - ctx->executor = executor_; - ctx->promise_ptr = std::move(promise); + std::string key_copy(key); + auto executor = executor_; + bool scheduled = scheduleOnEventLoop([this, + key = std::move(key_copy), + data_ptr, + data_len, + promise, + executor]() mutable { + if (!connected_.load() || !asyncContext_) { + setPromiseStatus(executor, promise, false); + return; + } - int ret = redisAsyncCommand(asyncContext_, - setCallback, - ctx, - "SET %b %b", - key.data(), - key.size(), - reinterpret_cast(data_ptr), - data_len); + auto *ctx = new CallbackContext; + ctx->executor = executor; + ctx->promise_ptr = promise; + + int ret = redisAsyncCommand(asyncContext_, + setCallback, + ctx, + "SET %b %b", + key.data(), + key.size(), + reinterpret_cast(data_ptr), + data_len); + + if (ret != REDIS_OK) { + auto promise_ptr = ctx->promise_ptr; + delete ctx; + setPromiseStatus(executor, promise_ptr, false); + } + }); - if (ret != REDIS_OK) { - auto promise_ptr = ctx->promise_ptr; - delete ctx; - setPromiseStatus(executor_, promise_ptr, false); + if (!scheduled) { + setPromiseStatus(executor_, promise, false); } } @@ -370,19 +513,37 @@ hiredisAsyncClient::getKeyAsync(std::string_view key, return; } - auto *ctx = new CallbackContext; - ctx->executor = executor_; - ctx->data_ptr = data_ptr; - ctx->data_len = data_len; - ctx->promise_ptr = std::move(promise); + std::string key_copy(key); + auto executor = executor_; + bool scheduled = scheduleOnEventLoop([this, + key = std::move(key_copy), + data_ptr, + data_len, + promise, + executor]() mutable { + if (!connected_.load() || !asyncContext_) { + setPromiseStatus(executor, promise, false); + return; + } - int ret = redisAsyncCommand( - asyncContext_, getCallback, ctx, "GET %b", key.data(), key.size()); + auto *ctx = new CallbackContext; + ctx->executor = executor; + ctx->data_ptr = data_ptr; + ctx->data_len = data_len; + ctx->promise_ptr = promise; - if (ret != REDIS_OK) { - auto promise_ptr = ctx->promise_ptr; - delete ctx; - setPromiseStatus(executor_, promise_ptr, false); + int ret = + redisAsyncCommand(asyncContext_, getCallback, ctx, "GET %b", key.data(), key.size()); + + if (ret != REDIS_OK) { + auto promise_ptr = ctx->promise_ptr; + delete ctx; + setPromiseStatus(executor, promise_ptr, false); + } + }); + + if (!scheduled) { + setPromiseStatus(executor_, promise, false); } } diff --git a/src/plugins/kv/redis/client.h b/src/plugins/kv/redis/client.h index cb2d1be75b..3f816003c8 100644 --- a/src/plugins/kv/redis/client.h +++ b/src/plugins/kv/redis/client.h @@ -24,12 +24,13 @@ * - sync (blocking hiredis): EXISTS for queryMem */ -#ifndef KV_PLUGIN_REDIS_CLIENT_H -#define KV_PLUGIN_REDIS_CLIENT_H +#ifndef NIXL_SRC_PLUGINS_KV_REDIS_CLIENT_H +#define NIXL_SRC_PLUGINS_KV_REDIS_CLIENT_H #include "redis_engine.h" #include #include +#include #include #include #include @@ -99,6 +100,12 @@ class hiredisAsyncClient : public iRedisClient { static void disconnectCallback(const redisAsyncContext *c, int status); + static void + authCallback(redisAsyncContext *c, void *reply, void *privdata); + + static void + selectCallback(redisAsyncContext *c, void *reply, void *privdata); + static void setCallback(redisAsyncContext *c, void *reply, void *privdata); @@ -111,6 +118,24 @@ class hiredisAsyncClient : public iRedisClient { void connectSyncContext(); + void + startAsyncInitialization(); + + void + startAsyncSelect(); + + void + completeAsyncInitialization(bool success); + + void + freeAsyncContextInEventLoop(); + + bool + scheduleOnEventLoop(std::function task); + + void + stopEventLoop(); + std::string host_; int port_; std::string password_; @@ -121,7 +146,9 @@ class hiredisAsyncClient : public iRedisClient { std::shared_ptr executor_; std::thread eventLoopThread_; std::atomic connected_; + std::atomic initDone_; + std::atomic initSucceeded_; mutable std::mutex syncMutex_; }; -#endif // KV_PLUGIN_REDIS_CLIENT_H +#endif // NIXL_SRC_PLUGINS_KV_REDIS_CLIENT_H diff --git a/src/plugins/kv/redis/engine_impl.cpp b/src/plugins/kv/redis/engine_impl.cpp index 8bc697900d..4c92bd87b0 100644 --- a/src/plugins/kv/redis/engine_impl.cpp +++ b/src/plugins/kv/redis/engine_impl.cpp @@ -88,11 +88,12 @@ isValidPrepXferParams(const nixl_xfer_op_t &operation, return false; } - if (remote_agent != local_agent) + if (remote_agent != local_agent) { NIXL_WARN << absl::StrFormat( "Warning: Remote agent doesn't match the requesting agent (%s). Got %s", local_agent, remote_agent); + } if (local.getType() != DRAM_SEG) { NIXL_ERROR << absl::StrFormat("Error: Local memory type must be DRAM_SEG, got %d", @@ -101,8 +102,8 @@ isValidPrepXferParams(const nixl_xfer_op_t &operation, } if (remote.getType() != DRAM_SEG && remote.getType() != OBJ_SEG) { - NIXL_ERROR << absl::StrFormat("Error: Remote memory type must be DRAM_SEG or OBJ_SEG, got %d", - remote.getType()); + NIXL_ERROR << absl::StrFormat( + "Error: Remote memory type must be DRAM_SEG or OBJ_SEG, got %d", remote.getType()); return false; } @@ -169,7 +170,7 @@ nixlRedisKVEngineImpl::nixlRedisKVEngineImpl(const nixlBackendInitParams *init_p nixlRedisKVEngineImpl::~nixlRedisKVEngineImpl() { if (executor_) { - executor_->WaitUntilStopped(); + executor_->waitUntilStopped(); } } @@ -179,6 +180,7 @@ nixlRedisKVEngineImpl::registerMem(const nixlBlobDesc &mem, nixlBackendMD *&out) { auto supported_mems = getSupportedMems(); if (std::find(supported_mems.begin(), supported_mems.end(), nixl_mem) == supported_mems.end()) { + out = nullptr; return NIXL_ERR_NOT_SUPPORTED; } @@ -295,8 +297,10 @@ nixlRedisKVEngineImpl::postXfer(const nixl_xfer_op_t &operation, req_h->statusFutures_.clear(); req_h->statusPromises_.clear(); - for (int i = 0; i < local.descCount(); ++i) { - const auto &local_desc = local[i]; + std::vector redis_keys; + redis_keys.reserve(local.descCount()); + + for (int i = 0; i < remote.descCount(); ++i) { const auto &remote_desc = remote[i]; std::string redis_key; @@ -318,6 +322,12 @@ nixlRedisKVEngineImpl::postXfer(const nixl_xfer_op_t &operation, } redis_key = it->second; } + redis_keys.push_back(std::move(redis_key)); + } + + for (int i = 0; i < local.descCount(); ++i) { + const auto &local_desc = local[i]; + const auto &redis_key = redis_keys[i]; auto status_promise = std::make_shared>(); req_h->statusPromises_.push_back(status_promise); diff --git a/src/plugins/kv/redis/engine_impl.h b/src/plugins/kv/redis/engine_impl.h index 4bf2747c13..2ae02f5850 100644 --- a/src/plugins/kv/redis/engine_impl.h +++ b/src/plugins/kv/redis/engine_impl.h @@ -20,8 +20,8 @@ * @brief nixlRedisKVEngineImpl — NIXL protocol layer over iRedisClient. */ -#ifndef KV_PLUGIN_REDIS_ENGINE_IMPL_H -#define KV_PLUGIN_REDIS_ENGINE_IMPL_H +#ifndef NIXL_SRC_PLUGINS_KV_REDIS_ENGINE_IMPL_H +#define NIXL_SRC_PLUGINS_KV_REDIS_ENGINE_IMPL_H #include "../kv_backend.h" #include "redis_engine.h" @@ -42,14 +42,16 @@ class nixlRedisKVEngineImpl : public nixlKVEngineImpl { ~nixlRedisKVEngineImpl() override; - nixl_mem_list_t getSupportedMems() const override { + nixl_mem_list_t + getSupportedMems() const override { return {OBJ_SEG, DRAM_SEG}; } nixl_status_t registerMem(const nixlBlobDesc &mem, const nixl_mem_t &nixl_mem, nixlBackendMD *&out) override; - nixl_status_t deregisterMem(nixlBackendMD *meta) override; + nixl_status_t + deregisterMem(nixlBackendMD *meta) override; nixl_status_t queryMem(const nixl_reg_dlist_t &descs, std::vector &resp) const override; @@ -71,9 +73,11 @@ class nixlRedisKVEngineImpl : public nixlKVEngineImpl { nixlBackendReqH *&handle, const nixl_opt_b_args_t *opt_args) const override; - nixl_status_t checkXfer(nixlBackendReqH *handle) const override; + nixl_status_t + checkXfer(nixlBackendReqH *handle) const override; - nixl_status_t releaseReqH(nixlBackendReqH *handle) const override; + nixl_status_t + releaseReqH(nixlBackendReqH *handle) const override; protected: virtual iRedisClient * @@ -86,4 +90,4 @@ class nixlRedisKVEngineImpl : public nixlKVEngineImpl { std::unordered_map devIdToRedisKey_; }; -#endif // KV_PLUGIN_REDIS_ENGINE_IMPL_H +#endif // NIXL_SRC_PLUGINS_KV_REDIS_ENGINE_IMPL_H diff --git a/src/plugins/kv/redis/meson.build b/src/plugins/kv/redis/meson.build index 02df437e90..a69bfcf2d5 100644 --- a/src/plugins/kv/redis/meson.build +++ b/src/plugins/kv/redis/meson.build @@ -60,8 +60,8 @@ if not hiredis_async_dep.found() or not libevent_dep.found() endif if not hiredis_async_dep.found() or not libevent_dep.found() - if 'REDIS' in static_plugins - error('REDIS static plugin requested but hiredis and libevent are required. Install libhiredis-dev and libevent-dev (or equivalent), then reconfigure.') + if is_explicit_enable or 'REDIS' in static_plugins + error('REDIS plugin requested but hiredis and libevent are required. Install libhiredis-dev and libevent-dev (or equivalent), then reconfigure.') endif warning('hiredis-async or libevent not available, skipping REDIS plugin build') subdir_done() @@ -106,7 +106,7 @@ else 'REDIS', redis_sources, dependencies: redis_plugin_deps, - cpp_args: compile_defs_redis + ['-fPIC'], + cpp_args: compile_defs_redis + compile_flags + ['-fPIC'], include_directories: [nixl_inc_dirs, utils_inc_dirs], install: true, name_prefix: 'libplugin_', diff --git a/src/plugins/kv/redis/redis_executor.h b/src/plugins/kv/redis/redis_executor.h index 986b58cb29..7e1803f4ac 100644 --- a/src/plugins/kv/redis/redis_executor.h +++ b/src/plugins/kv/redis/redis_executor.h @@ -31,7 +31,7 @@ class redisThreadPoolExecutor { explicit redisThreadPoolExecutor(std::size_t num_threads) : pool_(num_threads) {} void - WaitUntilStopped() { + waitUntilStopped() { pool_.stop(); pool_.join(); } diff --git a/test/gtest/unit/meson.build b/test/gtest/unit/meson.build index b68a5b96ef..032fd42aa1 100644 --- a/test/gtest/unit/meson.build +++ b/test/gtest/unit/meson.build @@ -23,6 +23,11 @@ if is_variable('obj_backend_interface') unit_test_deps += [obj_unit_test_dep] endif +if is_variable('redis_backend_interface') + subdir('redis') + unit_test_deps += [redis_unit_test_dep] +endif + subdir('agent') unit_test_deps += [agent_unit_test_dep] diff --git a/test/gtest/unit/redis/meson.build b/test/gtest/unit/redis/meson.build new file mode 100644 index 0000000000..18eabd022e --- /dev/null +++ b/test/gtest/unit/redis/meson.build @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +redis_unit_test_dep = declare_dependency( + sources: [ + 'redis.cpp', + ], + include_directories: [ + nixl_inc_dirs, utils_inc_dirs, + '../../../../src/plugins/kv', + '../../../../src/plugins/kv/redis', + ], + dependencies: [ + redis_backend_interface, + dependency('asio', required: true), + ], +) diff --git a/test/gtest/unit/redis/redis.cpp b/test/gtest/unit/redis/redis.cpp new file mode 100644 index 0000000000..bc7da6e07b --- /dev/null +++ b/test/gtest/unit/redis/redis.cpp @@ -0,0 +1,134 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "engine_impl.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace gtest::redis { + +class mockRedisClient : public iRedisClient { +public: + void + setExecutor(std::shared_ptr executor) override { + executor_ = std::move(executor); + } + + void + putKeyAsync(std::string_view key, + uintptr_t, + size_t, + std::shared_ptr> promise) override { + putKeys_.emplace_back(key); + promise->set_value(NIXL_SUCCESS); + } + + void + getKeyAsync(std::string_view key, + uintptr_t, + size_t, + std::shared_ptr> promise) override { + getKeys_.emplace_back(key); + promise->set_value(NIXL_SUCCESS); + } + + std::optional + checkKeyExistsSync(std::string_view key) override { + checkedKeys_.emplace_back(key); + return true; + } + + const std::vector & + putKeys() const { + return putKeys_; + } + +private: + std::shared_ptr executor_; + std::vector putKeys_; + std::vector getKeys_; + std::vector checkedKeys_; +}; + +class redisEngineTest : public ::testing::Test { +protected: + void + SetUp() override { + initParams_.localAgent = "redis-test-agent"; + initParams_.type = "REDIS"; + initParams_.customParams = &customParams_; + initParams_.enableProgTh = false; + initParams_.pthrDelay = 0; + initParams_.syncMode = nixl_thread_sync_t::NIXL_THREAD_SYNC_RW; + + mockClient_ = std::make_shared(); + engine_ = std::make_unique(&initParams_, mockClient_); + } + + nixlBackendInitParams initParams_; + nixl_b_params_t customParams_; + std::shared_ptr mockClient_; + std::unique_ptr engine_; +}; + +TEST_F(redisEngineTest, PostXferDoesNotDispatchPartialCommandsWhenLaterKeyIsMissing) { + std::vector firstBuffer(16, 'a'); + std::vector secondBuffer(16, 'b'); + + nixlBlobDesc registeredRemote(0, firstBuffer.size(), 11, "registered-key"); + nixlBackendMD *registeredMetadata = nullptr; + ASSERT_EQ(engine_->registerMem(registeredRemote, DRAM_SEG, registeredMetadata), NIXL_SUCCESS); + ASSERT_NE(registeredMetadata, nullptr); + + nixl_meta_dlist_t localDescs(DRAM_SEG); + localDescs.addDesc(nixlMetaDesc( + reinterpret_cast(firstBuffer.data()), firstBuffer.size(), 1, nullptr)); + localDescs.addDesc(nixlMetaDesc( + reinterpret_cast(secondBuffer.data()), secondBuffer.size(), 2, nullptr)); + + nixl_meta_dlist_t remoteDescs(DRAM_SEG); + remoteDescs.addDesc(nixlMetaDesc(0, firstBuffer.size(), 11, nullptr)); + remoteDescs.addDesc(nixlMetaDesc(0, secondBuffer.size(), 22, nullptr)); + + nixlBackendReqH *handle = nullptr; + ASSERT_EQ(engine_->prepXfer(NIXL_WRITE, + localDescs, + remoteDescs, + initParams_.localAgent, + initParams_.localAgent, + handle, + nullptr), + NIXL_SUCCESS); + + EXPECT_EQ(engine_->postXfer( + NIXL_WRITE, localDescs, remoteDescs, initParams_.localAgent, handle, nullptr), + NIXL_ERR_INVALID_PARAM); + EXPECT_TRUE(mockClient_->putKeys().empty()); + + EXPECT_EQ(engine_->releaseReqH(handle), NIXL_SUCCESS); + EXPECT_EQ(engine_->deregisterMem(registeredMetadata), NIXL_SUCCESS); +} + +} // namespace gtest::redis From d97b41319fc1e025e648635097da0afb3aafbf6b Mon Sep 17 00:00:00 2001 From: Harry Xie Date: Fri, 26 Jun 2026 03:40:31 +0300 Subject: [PATCH 17/24] Address KV review docs and Redis deps --- src/plugins/kv/kv_backend.h | 111 +++++++++++++++++++++++++++++++ src/plugins/kv/redis/meson.build | 5 +- 2 files changed, 115 insertions(+), 1 deletion(-) diff --git a/src/plugins/kv/kv_backend.h b/src/plugins/kv/kv_backend.h index fa16fae2ba..e299a938fc 100644 --- a/src/plugins/kv/kv_backend.h +++ b/src/plugins/kv/kv_backend.h @@ -26,33 +26,110 @@ #include #include +/** + * @brief Synchronous key-value store contract used by KV backend implementations. + * + * Implementations store byte ranges under string keys. The caller retains + * ownership of input and output buffers; implementations must copy data during + * the call and must not retain raw buffer pointers after returning. + */ class iKVStore { public: virtual ~iKVStore() = default; + /** + * @brief Store a byte range under a key. + * @param key Backend key identifying the object. + * @param data Pointer to the data to store. Must be valid for len bytes. + * @param len Number of bytes to store. + * @return NIXL_SUCCESS on success, otherwise an error status. + */ virtual nixl_status_t put(std::string_view key, const uint8_t *data, size_t len) = 0; + /** + * @brief Load a byte range from a key into a caller-owned buffer. + * @param key Backend key identifying the object. + * @param buffer Destination buffer. Must be valid for len bytes. + * @param len Destination buffer capacity in bytes. + * @param bytes_read Set to the number of bytes copied into buffer. + * @return NIXL_SUCCESS on success, otherwise an error status. + */ virtual nixl_status_t get(std::string_view key, uint8_t *buffer, size_t len, size_t &bytes_read) const = 0; + /** + * @brief Check whether a key exists in the backing store. + * @param key Backend key identifying the object. + * @return true when the key exists, false otherwise. + */ virtual bool exists(std::string_view key) const = 0; }; +/** + * @brief Storage-specific implementation contract behind nixlKVEngine. + * + * The public nixlKVEngine wrapper delegates backend operations to one owned + * implementation object. Memory metadata returned from registerMem() remains + * backend-owned in the sense that callers must release it through + * deregisterMem(); callers must not delete it directly. Request handles + * returned from prepXfer() remain valid until releaseReqH() is called exactly + * once, even after postXfer() and checkXfer() complete. + */ class nixlKVEngineImpl { public: virtual ~nixlKVEngineImpl() = default; + /** + * @brief Return memory segment types supported by this implementation. + */ virtual nixl_mem_list_t getSupportedMems() const = 0; + /** + * @brief Register backend metadata for a memory or object descriptor. + * @param mem Descriptor carrying address, length, device id, and optional key/path metadata. + * @param nixl_mem Segment type being registered. + * @param out Set to backend metadata on success, or nullptr when no metadata is required. + * @return NIXL_SUCCESS on success, otherwise an error status. + * + * When out is non-null, the caller must pass it back to deregisterMem(). + */ virtual nixl_status_t registerMem(const nixlBlobDesc &mem, const nixl_mem_t &nixl_mem, nixlBackendMD *&out) = 0; + + /** + * @brief Release metadata previously returned by registerMem(). + * @param meta Metadata pointer returned by registerMem(), or nullptr. + * @return NIXL_SUCCESS on success, otherwise an error status. + */ virtual nixl_status_t deregisterMem(nixlBackendMD *meta) = 0; + + /** + * @brief Query backend visibility or availability for registered descriptors. + * @param descs Descriptors to query. + * @param resp Filled with one response per descriptor. + * @return NIXL_SUCCESS when all queries complete, otherwise an error status. + */ virtual nixl_status_t queryMem(const nixl_reg_dlist_t &descs, std::vector &resp) const = 0; + + /** + * @brief Validate and prepare a transfer request handle. + * @param operation Transfer operation, such as NIXL_READ or NIXL_WRITE. + * @param local Local descriptors participating in the transfer. + * @param remote Remote descriptors participating in the transfer. + * @param remote_agent Remote agent name supplied by the caller. + * @param local_agent Local agent name for validation. + * @param handle Set to a new request handle on success; set to nullptr on failure. + * @param opt_args Optional backend arguments. + * @return NIXL_SUCCESS on successful preparation, otherwise an error status. + * + * The caller owns the returned handle lifecycle and must release it with + * releaseReqH() exactly once. + */ virtual nixl_status_t prepXfer(const nixl_xfer_op_t &operation, const nixl_meta_dlist_t &local, @@ -61,6 +138,20 @@ class nixlKVEngineImpl { const std::string &local_agent, nixlBackendReqH *&handle, const nixl_opt_b_args_t *opt_args) const = 0; + + /** + * @brief Start work for a prepared transfer request. + * @param operation Transfer operation matching prepXfer(). + * @param local Local descriptors participating in the transfer. + * @param remote Remote descriptors participating in the transfer. + * @param remote_agent Remote agent name supplied by the caller. + * @param handle Request handle returned by prepXfer(). + * @param opt_args Optional backend arguments. + * @return NIXL_IN_PROG for asynchronous progress, NIXL_SUCCESS when complete, or an error. + * + * The handle remains owned by the caller and must still be released with + * releaseReqH(). + */ virtual nixl_status_t postXfer(const nixl_xfer_op_t &operation, const nixl_meta_dlist_t &local, @@ -68,12 +159,32 @@ class nixlKVEngineImpl { const std::string &remote_agent, nixlBackendReqH *&handle, const nixl_opt_b_args_t *opt_args) const = 0; + + /** + * @brief Poll completion status for a prepared transfer request. + * @param handle Request handle returned by prepXfer(). + * @return NIXL_IN_PROG while work remains, NIXL_SUCCESS on completion, or an error. + */ virtual nixl_status_t checkXfer(nixlBackendReqH *handle) const = 0; + + /** + * @brief Destroy a request handle returned by prepXfer(). + * @param handle Request handle to release. + * @return NIXL_SUCCESS on success, otherwise an error status. + */ virtual nixl_status_t releaseReqH(nixlBackendReqH *handle) const = 0; }; +/** + * @brief NIXL backend wrapper for key-value storage plugins. + * + * nixlKVEngine owns a concrete nixlKVEngineImpl through a unique_ptr and + * delegates all storage-specific behavior to it. Concrete KV plugins derive + * from this wrapper, pass their implementation to the protected constructor, + * and inherit the NIXL backend interface glue. + */ class nixlKVEngine : public nixlBackendEngine { public: ~nixlKVEngine() override; diff --git a/src/plugins/kv/redis/meson.build b/src/plugins/kv/redis/meson.build index a69bfcf2d5..5eaeee782d 100644 --- a/src/plugins/kv/redis/meson.build +++ b/src/plugins/kv/redis/meson.build @@ -120,4 +120,7 @@ else endif endif -redis_backend_interface = declare_dependency(link_with: redis_backend_lib) +redis_backend_interface = declare_dependency( + link_with: redis_backend_lib, + dependencies: redis_plugin_deps, +) From ce0c697ca92a6d491f5c98c0ef21ccb597f7b078 Mon Sep 17 00:00:00 2001 From: Harry Xie Date: Sat, 4 Jul 2026 03:58:39 +0300 Subject: [PATCH 18/24] refactor(redis): simplify backend architecture --- meson.build | 3 +- src/plugins/kv/README.md | 178 ------------ src/plugins/kv/kv_backend.cpp | 84 ------ src/plugins/kv/kv_backend.h | 272 ------------------ src/plugins/kv/meson.build | 21 -- src/plugins/kv/redis/README.md | 176 ------------ src/plugins/kv/redis/engine_impl.h | 93 ------ src/plugins/kv/redis/redis_engine.cpp | 37 --- src/plugins/kv/redis/redis_engine.h | 103 ------- src/plugins/kv/redis/redis_executor.h | 53 ---- src/plugins/meson.build | 2 +- src/plugins/redis/README.md | 109 +++++++ src/plugins/{kv => }/redis/meson.build | 35 +-- .../redis_backend.cpp} | 210 +++++--------- src/plugins/redis/redis_backend.h | 109 +++++++ .../client.cpp => redis/redis_client.cpp} | 61 ++-- .../redis/client.h => redis/redis_client.h} | 87 ++---- .../kv_plugin.cpp => redis/redis_plugin.cpp} | 18 +- test/gtest/unit/redis/meson.build | 4 +- test/gtest/unit/redis/redis.cpp | 265 ++++++++++++++--- 20 files changed, 580 insertions(+), 1340 deletions(-) delete mode 100644 src/plugins/kv/README.md delete mode 100644 src/plugins/kv/kv_backend.cpp delete mode 100644 src/plugins/kv/kv_backend.h delete mode 100644 src/plugins/kv/meson.build delete mode 100644 src/plugins/kv/redis/README.md delete mode 100644 src/plugins/kv/redis/engine_impl.h delete mode 100644 src/plugins/kv/redis/redis_engine.cpp delete mode 100644 src/plugins/kv/redis/redis_engine.h delete mode 100644 src/plugins/kv/redis/redis_executor.h create mode 100644 src/plugins/redis/README.md rename src/plugins/{kv => }/redis/meson.build (75%) rename src/plugins/{kv/redis/engine_impl.cpp => redis/redis_backend.cpp} (53%) create mode 100644 src/plugins/redis/redis_backend.h rename src/plugins/{kv/redis/client.cpp => redis/redis_client.cpp} (90%) rename src/plugins/{kv/redis/client.h => redis/redis_client.h} (60%) rename src/plugins/{kv/kv_plugin.cpp => redis/redis_plugin.cpp} (66%) diff --git a/meson.build b/meson.build index 2b0070a042..64dc3bd3ad 100644 --- a/meson.build +++ b/meson.build @@ -508,7 +508,7 @@ if get_option('buildtype') == 'debug' run_command('truncate', '-s 0', plugfile, check: true) endif -nixl_inc_dirs = include_directories('src/api/cpp', 'src/api/cpp/backend', 'src/infra', 'src/core', 'src/core/telemetry', 'src/plugins/kv') +nixl_inc_dirs = include_directories('src/api/cpp', 'src/api/cpp/backend', 'src/infra', 'src/core', 'src/core/telemetry') nixl_gpu_inc_dirs = include_directories('src/api/gpu/ucx') plugins_inc_dirs = include_directories('src/plugins') utils_inc_dirs = include_directories('src/utils') @@ -538,7 +538,6 @@ if get_option('install_headers') install_headers('src/core/transfer_request.h', install_dir: prefix_inc) install_headers('src/core/agent_data.h', install_dir: prefix_inc) install_headers('src/infra/mem_section.h', install_dir: prefix_inc) - install_headers('src/plugins/kv/kv_backend.h', install_dir: prefix_inc + '/plugins/kv') install_headers('src/core/telemetry/telemetry_event.h', install_dir: prefix_inc) if ucx_gpu_device_api_available diff --git a/src/plugins/kv/README.md b/src/plugins/kv/README.md deleted file mode 100644 index e3894616da..0000000000 --- a/src/plugins/kv/README.md +++ /dev/null @@ -1,178 +0,0 @@ - - -# NIXL Key-Value Storage Plugin - -The KV plugin layer provides a NIXL backend facade for key-value storage. The shared code in this -directory follows the same facade/implementation split as the object backend, while concrete -implementations live in backend-specific subdirectories. - -The current production implementation is the `REDIS` plugin under `redis/`. - -## Architecture - -The shared KV layer owns the NIXL-facing backend wrapper and implementation interface: - -- **KV backend wrapper** (`nixlKVEngine`): The class registered with the NIXL backend plugin loader. -- **KV implementation base** (`nixlKVEngineImpl`): The backend-facing interface implemented by concrete KV stores. -- **Simple KV store interface** (`iKVStore`): A synchronous put/get/exists helper interface for simple KV implementations. - -The Redis implementation uses a richer client interface because `postXfer` is asynchronous: - -- **Redis engine** (`nixlRedisKVEngine`): The REDIS plugin engine registered by `kv_plugin.cpp`. -- **Redis implementation** (`nixlRedisKVEngineImpl`): Maps NIXL transfers to Redis SET/GET/EXISTS operations. -- **Redis client** (`iRedisClient`, `hiredisAsyncClient`): Uses an async hiredis connection for SET/GET and a - separate synchronous connection for EXISTS. -- **Redis executor** (`redisThreadPoolExecutor`): Completes async promises away from the hiredis/libevent callback path. - -``` -kv/ - kv_backend.h/.cpp shared nixlKVEngine, nixlKVEngineImpl, iKVStore - kv_plugin.cpp registers the REDIS backend with the NIXL plugin loader - meson.build includes backend subdirectories - -kv/redis/ - redis_engine.h/.cpp iRedisClient and nixlRedisKVEngine - engine_impl.h/.cpp nixlRedisKVEngineImpl transfer logic - client.h/.cpp hiredisAsyncClient - redis_executor.h Redis-specific ASIO executor - meson.build REDIS plugin target - README.md Redis-specific build and benchmark notes -``` - -## Dependencies - -The shared KV facade has no storage-system dependency. The REDIS backend requires: - -| Dependency | Used for | -|------------|----------| -| `hiredis` / `hiredis_async` | Redis command encoding, sync EXISTS, async SET/GET | -| `libevent` | Event loop integration for hiredis async | -| `libevent_pthreads` | Thread-safe libevent setup | -| `asio` | Thread pool used by the Redis executor | - -If REDIS is explicitly enabled or requested as a static plugin, missing hiredis/libevent is treated -as a configuration error. When building all plugins by default, a missing Redis dependency skips the -dynamic REDIS plugin with a warning. - -## Configuration - -Backend parameters are passed as a `nixl_b_params_t` map when creating the backend. Backend -parameters take precedence over environment variables. - -| Parameter | Environment fallback | Default | Description | -|-----------|----------------------|---------|-------------| -| `host` | `REDIS_HOST` | `localhost` | Redis server hostname or IP address | -| `port` | `REDIS_PORT` | `6379` | Redis server TCP port | -| `password` | `REDIS_PASSWORD` | empty | Redis AUTH password | -| `db` | - | `0` | Redis logical database number | -| `num_threads` | - | half of hardware concurrency, minimum 1 | ASIO worker threads for async completion | - -### Configuration Examples - -#### Local Redis - -```cpp -nixl_b_params_t params = { - {"host", "127.0.0.1"}, - {"port", "6379"} -}; -agent.createBackend("REDIS", params); -``` - -#### Authenticated Redis - -```cpp -nixl_b_params_t params = { - {"host", "redis.internal"}, - {"port", "6379"}, - {"password", "example-password"}, - {"db", "2"}, - {"num_threads", "4"} -}; -agent.createBackend("REDIS", params); -``` - -#### Environment Variable Configuration - -```bash -export REDIS_HOST=127.0.0.1 -export REDIS_PORT=6379 -export REDIS_PASSWORD=example-password -``` - -```cpp -nixl_b_params_t params = {}; -agent.createBackend("REDIS", params); -``` - -## Transfer Operations - -The REDIS backend supports `NIXL_WRITE` and `NIXL_READ`. - -| NIXL operation | Redis operation | Behavior | -|----------------|-----------------|----------| -| `NIXL_WRITE` | `SET key bytes` | Stores local DRAM bytes under the remote Redis key | -| `NIXL_READ` | `GET key` | Reads the Redis value into the local DRAM descriptor | -| `queryMem` | `EXISTS key` | Reports whether a Redis key exists | - -Local descriptors must be `DRAM_SEG`. Remote descriptors may be `DRAM_SEG` or `OBJ_SEG`, matching -the segment list exposed by the plugin. The Redis key is chosen from descriptor metadata: - -1. Use `metaInfo` when it is present. -2. Otherwise use the descriptor `devId` converted to a string. - -`prepXfer` validates operation type, local/remote descriptor counts, and supported segment types. -`postXfer` creates one promise/future pair per descriptor and returns `NIXL_IN_PROG`. `checkXfer` -polls those futures until all Redis async operations complete. - -## Build - -Build only the REDIS plugin: - -```bash -meson setup build -Denable_plugins=REDIS -ninja -C build src/plugins/kv/redis/libplugin_REDIS.so -``` - -For static plugin builds: - -```bash -meson setup build -Denable_plugins=REDIS -Dstatic_plugins=REDIS -ninja -C build -``` - -The dynamic plugin output is: - -```text -build/src/plugins/kv/redis/libplugin_REDIS.so -``` - -## Extending KV Backends - -New KV implementations should follow the same split used by Redis: - -1. Add a backend-specific subdirectory under `src/plugins/kv/`. -2. Implement `nixlKVEngineImpl` for NIXL registration, query, prep, post, check, and request-handle cleanup. -3. Provide a small client/store interface for backend-specific storage operations. -4. Add a Meson subdirectory and plugin source list. -5. Register the plugin from a top-level `kv_plugin.cpp`-style entrypoint when the implementation should be exposed as a NIXL plugin. - -There is no generic `kv_executor.h` today. Redis keeps `redis_executor.h` because the executor is -only used by the Redis async client, and OBJ's executor inherits from an AWS SDK executor interface. -If another KV implementation needs the same ASIO-only executor, it would be reasonable to promote -that type into a shared `kv_executor.h` at that point. diff --git a/src/plugins/kv/kv_backend.cpp b/src/plugins/kv/kv_backend.cpp deleted file mode 100644 index 74af4284ce..0000000000 --- a/src/plugins/kv/kv_backend.cpp +++ /dev/null @@ -1,84 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "kv_backend.h" -#include -#include - -nixlKVEngine::nixlKVEngine(const nixlBackendInitParams *init_params, - std::unique_ptr impl) - : nixlBackendEngine(init_params), - impl_(std::move(impl)) { - if (!impl_) { - throw std::invalid_argument("nixlKVEngine requires a non-null nixlKVEngineImpl"); - } - assert(impl_ && "nixlKVEngine requires a non-null nixlKVEngineImpl"); -} - -nixlKVEngine::~nixlKVEngine() = default; - -nixl_mem_list_t -nixlKVEngine::getSupportedMems() const { - return impl_->getSupportedMems(); -} - -nixl_status_t -nixlKVEngine::registerMem(const nixlBlobDesc &mem, - const nixl_mem_t &nixl_mem, - nixlBackendMD *&out) { - return impl_->registerMem(mem, nixl_mem, out); -} - -nixl_status_t -nixlKVEngine::deregisterMem(nixlBackendMD *meta) { - return impl_->deregisterMem(meta); -} - -nixl_status_t -nixlKVEngine::queryMem(const nixl_reg_dlist_t &descs, std::vector &resp) const { - return impl_->queryMem(descs, resp); -} - -nixl_status_t -nixlKVEngine::prepXfer(const nixl_xfer_op_t &operation, - const nixl_meta_dlist_t &local, - const nixl_meta_dlist_t &remote, - const std::string &remote_agent, - nixlBackendReqH *&handle, - const nixl_opt_b_args_t *opt_args) const { - return impl_->prepXfer(operation, local, remote, remote_agent, localAgent, handle, opt_args); -} - -nixl_status_t -nixlKVEngine::postXfer(const nixl_xfer_op_t &operation, - const nixl_meta_dlist_t &local, - const nixl_meta_dlist_t &remote, - const std::string &remote_agent, - nixlBackendReqH *&handle, - const nixl_opt_b_args_t *opt_args) const { - return impl_->postXfer(operation, local, remote, remote_agent, handle, opt_args); -} - -nixl_status_t -nixlKVEngine::checkXfer(nixlBackendReqH *handle) const { - return impl_->checkXfer(handle); -} - -nixl_status_t -nixlKVEngine::releaseReqH(nixlBackendReqH *handle) const { - return impl_->releaseReqH(handle); -} diff --git a/src/plugins/kv/kv_backend.h b/src/plugins/kv/kv_backend.h deleted file mode 100644 index e299a938fc..0000000000 --- a/src/plugins/kv/kv_backend.h +++ /dev/null @@ -1,272 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef NIXL_SRC_PLUGINS_KV_KV_BACKEND_H -#define NIXL_SRC_PLUGINS_KV_KV_BACKEND_H - -#include "backend/backend_engine.h" -#include -#include -#include -#include -#include -#include - -/** - * @brief Synchronous key-value store contract used by KV backend implementations. - * - * Implementations store byte ranges under string keys. The caller retains - * ownership of input and output buffers; implementations must copy data during - * the call and must not retain raw buffer pointers after returning. - */ -class iKVStore { -public: - virtual ~iKVStore() = default; - - /** - * @brief Store a byte range under a key. - * @param key Backend key identifying the object. - * @param data Pointer to the data to store. Must be valid for len bytes. - * @param len Number of bytes to store. - * @return NIXL_SUCCESS on success, otherwise an error status. - */ - virtual nixl_status_t - put(std::string_view key, const uint8_t *data, size_t len) = 0; - - /** - * @brief Load a byte range from a key into a caller-owned buffer. - * @param key Backend key identifying the object. - * @param buffer Destination buffer. Must be valid for len bytes. - * @param len Destination buffer capacity in bytes. - * @param bytes_read Set to the number of bytes copied into buffer. - * @return NIXL_SUCCESS on success, otherwise an error status. - */ - virtual nixl_status_t - get(std::string_view key, uint8_t *buffer, size_t len, size_t &bytes_read) const = 0; - - /** - * @brief Check whether a key exists in the backing store. - * @param key Backend key identifying the object. - * @return true when the key exists, false otherwise. - */ - virtual bool - exists(std::string_view key) const = 0; -}; - -/** - * @brief Storage-specific implementation contract behind nixlKVEngine. - * - * The public nixlKVEngine wrapper delegates backend operations to one owned - * implementation object. Memory metadata returned from registerMem() remains - * backend-owned in the sense that callers must release it through - * deregisterMem(); callers must not delete it directly. Request handles - * returned from prepXfer() remain valid until releaseReqH() is called exactly - * once, even after postXfer() and checkXfer() complete. - */ -class nixlKVEngineImpl { -public: - virtual ~nixlKVEngineImpl() = default; - - /** - * @brief Return memory segment types supported by this implementation. - */ - virtual nixl_mem_list_t - getSupportedMems() const = 0; - - /** - * @brief Register backend metadata for a memory or object descriptor. - * @param mem Descriptor carrying address, length, device id, and optional key/path metadata. - * @param nixl_mem Segment type being registered. - * @param out Set to backend metadata on success, or nullptr when no metadata is required. - * @return NIXL_SUCCESS on success, otherwise an error status. - * - * When out is non-null, the caller must pass it back to deregisterMem(). - */ - virtual nixl_status_t - registerMem(const nixlBlobDesc &mem, const nixl_mem_t &nixl_mem, nixlBackendMD *&out) = 0; - - /** - * @brief Release metadata previously returned by registerMem(). - * @param meta Metadata pointer returned by registerMem(), or nullptr. - * @return NIXL_SUCCESS on success, otherwise an error status. - */ - virtual nixl_status_t - deregisterMem(nixlBackendMD *meta) = 0; - - /** - * @brief Query backend visibility or availability for registered descriptors. - * @param descs Descriptors to query. - * @param resp Filled with one response per descriptor. - * @return NIXL_SUCCESS when all queries complete, otherwise an error status. - */ - virtual nixl_status_t - queryMem(const nixl_reg_dlist_t &descs, std::vector &resp) const = 0; - - /** - * @brief Validate and prepare a transfer request handle. - * @param operation Transfer operation, such as NIXL_READ or NIXL_WRITE. - * @param local Local descriptors participating in the transfer. - * @param remote Remote descriptors participating in the transfer. - * @param remote_agent Remote agent name supplied by the caller. - * @param local_agent Local agent name for validation. - * @param handle Set to a new request handle on success; set to nullptr on failure. - * @param opt_args Optional backend arguments. - * @return NIXL_SUCCESS on successful preparation, otherwise an error status. - * - * The caller owns the returned handle lifecycle and must release it with - * releaseReqH() exactly once. - */ - virtual nixl_status_t - prepXfer(const nixl_xfer_op_t &operation, - const nixl_meta_dlist_t &local, - const nixl_meta_dlist_t &remote, - const std::string &remote_agent, - const std::string &local_agent, - nixlBackendReqH *&handle, - const nixl_opt_b_args_t *opt_args) const = 0; - - /** - * @brief Start work for a prepared transfer request. - * @param operation Transfer operation matching prepXfer(). - * @param local Local descriptors participating in the transfer. - * @param remote Remote descriptors participating in the transfer. - * @param remote_agent Remote agent name supplied by the caller. - * @param handle Request handle returned by prepXfer(). - * @param opt_args Optional backend arguments. - * @return NIXL_IN_PROG for asynchronous progress, NIXL_SUCCESS when complete, or an error. - * - * The handle remains owned by the caller and must still be released with - * releaseReqH(). - */ - virtual nixl_status_t - postXfer(const nixl_xfer_op_t &operation, - const nixl_meta_dlist_t &local, - const nixl_meta_dlist_t &remote, - const std::string &remote_agent, - nixlBackendReqH *&handle, - const nixl_opt_b_args_t *opt_args) const = 0; - - /** - * @brief Poll completion status for a prepared transfer request. - * @param handle Request handle returned by prepXfer(). - * @return NIXL_IN_PROG while work remains, NIXL_SUCCESS on completion, or an error. - */ - virtual nixl_status_t - checkXfer(nixlBackendReqH *handle) const = 0; - - /** - * @brief Destroy a request handle returned by prepXfer(). - * @param handle Request handle to release. - * @return NIXL_SUCCESS on success, otherwise an error status. - */ - virtual nixl_status_t - releaseReqH(nixlBackendReqH *handle) const = 0; -}; - -/** - * @brief NIXL backend wrapper for key-value storage plugins. - * - * nixlKVEngine owns a concrete nixlKVEngineImpl through a unique_ptr and - * delegates all storage-specific behavior to it. Concrete KV plugins derive - * from this wrapper, pass their implementation to the protected constructor, - * and inherit the NIXL backend interface glue. - */ -class nixlKVEngine : public nixlBackendEngine { -public: - ~nixlKVEngine() override; - - bool - supportsRemote() const override { - return false; - } - - bool - supportsLocal() const override { - return true; - } - - bool - supportsNotif() const override { - return false; - } - - nixl_mem_list_t - getSupportedMems() const override; - - nixl_status_t - registerMem(const nixlBlobDesc &mem, const nixl_mem_t &nixl_mem, nixlBackendMD *&out) override; - - nixl_status_t - deregisterMem(nixlBackendMD *meta) override; - - nixl_status_t - queryMem(const nixl_reg_dlist_t &descs, std::vector &resp) const override; - - nixl_status_t - connect(const std::string &remote_agent) override { - return NIXL_SUCCESS; - } - - nixl_status_t - disconnect(const std::string &remote_agent) override { - return NIXL_SUCCESS; - } - - nixl_status_t - unloadMD(nixlBackendMD *input) override { - return NIXL_SUCCESS; - } - - nixl_status_t - loadLocalMD(nixlBackendMD *input, nixlBackendMD *&output) override { - if (input == nullptr) { - output = nullptr; - return NIXL_ERR_INVALID_PARAM; - } - output = input; - return NIXL_SUCCESS; - } - - nixl_status_t - prepXfer(const nixl_xfer_op_t &operation, - const nixl_meta_dlist_t &local, - const nixl_meta_dlist_t &remote, - const std::string &remote_agent, - nixlBackendReqH *&handle, - const nixl_opt_b_args_t *opt_args = nullptr) const override; - - nixl_status_t - postXfer(const nixl_xfer_op_t &operation, - const nixl_meta_dlist_t &local, - const nixl_meta_dlist_t &remote, - const std::string &remote_agent, - nixlBackendReqH *&handle, - const nixl_opt_b_args_t *opt_args = nullptr) const override; - - nixl_status_t - checkXfer(nixlBackendReqH *handle) const override; - nixl_status_t - releaseReqH(nixlBackendReqH *handle) const override; - -protected: - nixlKVEngine(const nixlBackendInitParams *init_params, std::unique_ptr impl); - -private: - std::unique_ptr impl_; -}; - -#endif // NIXL_SRC_PLUGINS_KV_KV_BACKEND_H diff --git a/src/plugins/kv/meson.build b/src/plugins/kv/meson.build deleted file mode 100644 index 89f197efe3..0000000000 --- a/src/plugins/kv/meson.build +++ /dev/null @@ -1,21 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Shared KV backend interfaces live in kv/. -# REDIS plugin sources are entirely under kv/redis/. - -if enabled_plugins.get('REDIS') - subdir('redis') -endif diff --git a/src/plugins/kv/redis/README.md b/src/plugins/kv/redis/README.md deleted file mode 100644 index 73fcd574d2..0000000000 --- a/src/plugins/kv/redis/README.md +++ /dev/null @@ -1,176 +0,0 @@ - - -# NIXL Redis Plugin (KV) - -The REDIS implementation lives in `src/plugins/kv/redis/`. Shared KV abstractions and the plugin -entrypoint stay in `src/plugins/kv/`. - -``` -kv/ # shared KV layer (PR1 interface) - kv_backend.h/.cpp nixlKVEngine, nixlKVEngineImpl, iKVStore - kv_plugin.cpp registers "REDIS" - meson.build delegates to redis/ when REDIS enabled - -kv/redis/ # REDIS plugin (this directory) - redis_engine.h/.cpp iRedisClient + nixlRedisKVEngine - redis_executor.h Redis-specific ASIO thread pool - client.h/.cpp hiredisAsyncClient - engine_impl.h/.cpp nixlRedisKVEngineImpl - meson.build - README.md -``` - -## Class hierarchy - -| Layer | REDIS plugin | -|-------|--------------| -| Shared KV wrapper | `nixlKVEngine` (`../kv_backend.h`) | -| Shared KV impl base | `nixlKVEngineImpl` (`../kv_backend.h`) | -| Plugin entry | `nixlRedisKVEngine` (`redis_engine.h`) | -| Async storage interface | `iRedisClient` (`redis_engine.h`) | -| Client implementation | `hiredisAsyncClient` (`client.h`) | -| Engine implementation | `nixlRedisKVEngineImpl` (`engine_impl.h`) | - -## iKVStore vs iRedisClient - -| | `iKVStore` (`../kv_backend.h`) | `iRedisClient` (`redis_engine.h`) | -|--|---------------------------|-----------------------------------| -| Used by | INMEMKV example plugin | REDIS plugin | -| API style | Sync `put` / `get` / `exists` | Async SET/GET + sync EXISTS (separate connections) | -| `postXfer` | Returns `NIXL_SUCCESS` immediately | Returns `NIXL_IN_PROG`; `checkXfer` polls futures | -| Why separate | Blocking on hiredis event loop would deadlock | Async promise path for postXfer/checkXfer | - -## iRedisClient API - -```cpp -class iRedisClient { - void setExecutor(std::shared_ptr executor); - - // postXfer path (async connection) - void putKeyAsync(..., std::shared_ptr> promise); - void getKeyAsync(..., std::shared_ptr> promise); - - // queryMem path (sync connection) - std::optional checkKeyExistsSync(std::string_view key); -}; -``` - -Connection settings (constructor): `host`, `port`, and `password` can come from NIXL custom params or `REDIS_HOST`, `REDIS_PORT`, and `REDIS_PASSWORD`; `db` is a NIXL custom param only. - -## Transfer call path - -``` -nixlbench / NIXL API - -> nixlRedisKVEngine::postXfer - -> nixlRedisKVEngineImpl::postXfer - creates std::promise per descriptor - -> iRedisClient::putKeyAsync / getKeyAsync (async connection) - -> hiredisAsyncClient (libevent thread + redisAsyncCommand) - promise completed on ASIO executor thread - -> checkXfer polls statusFutures_ - -queryMem (sync path): - -> nixlRedisKVEngineImpl::queryMem - -> iRedisClient::checkKeyExistsSync (separate blocking hiredis connection) - -> redisCommand EXISTS, returns before queryMem exits -``` - -## Build - -### Native (plugin only) - -```bash -meson setup build -Denable_plugins=REDIS -ninja -C build -# -> build/src/plugins/kv/redis/libplugin_REDIS.so -``` - -### Docker image (nixlbench + REDIS plugin) - -On the **host**, from `benchmark/nixlbench/contrib/`: - -```bash -cd benchmark/nixlbench/contrib - -# Build image; tag defaults to nixlbench:v.dev. -./build.sh --nixl --redis-only - -# Or pin an explicit tag: -./build.sh --nixl --redis-only --tag nixlbench:v0.1.0.dev.9da7bd1 -``` - -`--redis-only` passes `-Denable_plugins=REDIS -Dstatic_plugins=REDIS` into the image build. -The image installs `nixlbench` to `/usr/local/nixlbench/bin` and sets the default -working directory to `/workspace/nixl/benchmark/kvbench`. - -## Test (Docker container, step by step) - -This matches the workflow: build image on the host, then run `nixlbench` **inside** the container. - -### Step 1 — Start Redis on the host - -```bash -docker run -d --name nixlbench-redis -p 6379:6379 redis:7-alpine -``` - -Redis listens on host port `6379`. Use `--network host` when starting the bench container -(step 2) so `127.0.0.1:6379` inside the container reaches this Redis instance. - -### Step 2 — Start the nixlbench container - -```bash -docker run -it --rm --network host \ - nixlbench:v0.1.0.dev.9da7bd1 \ - bash -``` - -Replace the image tag with the one printed by `build.sh`. - -### Step 3 — Inside the container: set Redis env and run bench - -The image default working directory is `/workspace/nixl/benchmark/kvbench`. -`nixlbench` is already on `PATH`. - -```bash -cd /workspace/nixl/benchmark/kvbench -export REDIS_HOST=127.0.0.1 REDIS_PORT=6379 - -# READ — nixlbench seeds the Redis key automatically before the transfer loop -nixlbench --backend REDIS --start_block_size 4096 --max_block_size 4096 \ - --num_iter 50 --warmup_iter 5 --op_type READ - -# WRITE -nixlbench --backend REDIS --start_block_size 4096 --max_block_size 4096 \ - --num_iter 100 --warmup_iter 10 --op_type WRITE -``` - -Expected log lines: - -- `REDIS backend configured (using defaults or environment variables)` -- READ only: `Seeded Redis key for READ: nixlbench_redis0_0_...` -- Stats table with bandwidth and latency (example: ~0.03 GB/s, ~120 µs at 4096 B) - -**Notes:** - -- `REDIS_HOST` / `REDIS_PORT` / `REDIS_PASSWORD` are read by `hiredisAsyncClient` at plugin init. -- Redis `db` selection is configured only through the NIXL `db` custom param. -- nixlbench may adjust `--num_iter` / `--warmup_iter` for thread alignment (warnings are normal). -- For a one-shot run without an interactive shell, pass `nixlbench ...` as the `docker run` command - instead of `bash`. - -### Step 4 — Native bench (no Docker) - -If `nixlbench` is built locally with REDIS support: - -```bash -export REDIS_HOST=127.0.0.1 REDIS_PORT=6379 - -nixlbench --backend REDIS --start_block_size 4096 --max_block_size 4096 \ - --num_iter 100 --warmup_iter 10 --op_type WRITE - -nixlbench --backend REDIS --start_block_size 4096 --max_block_size 4096 \ - --num_iter 50 --warmup_iter 5 --op_type READ -``` diff --git a/src/plugins/kv/redis/engine_impl.h b/src/plugins/kv/redis/engine_impl.h deleted file mode 100644 index 2ae02f5850..0000000000 --- a/src/plugins/kv/redis/engine_impl.h +++ /dev/null @@ -1,93 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * @file engine_impl.h - * @brief nixlRedisKVEngineImpl — NIXL protocol layer over iRedisClient. - */ - -#ifndef NIXL_SRC_PLUGINS_KV_REDIS_ENGINE_IMPL_H -#define NIXL_SRC_PLUGINS_KV_REDIS_ENGINE_IMPL_H - -#include "../kv_backend.h" -#include "redis_engine.h" -#include -#include -#include - -/** - * @class nixlRedisKVEngineImpl - * @brief REDIS KV engine implementation — NIXL transfer protocol over iRedisClient. - */ -class nixlRedisKVEngineImpl : public nixlKVEngineImpl { -public: - explicit nixlRedisKVEngineImpl(const nixlBackendInitParams *init_params); - - nixlRedisKVEngineImpl(const nixlBackendInitParams *init_params, - std::shared_ptr redis_client); - - ~nixlRedisKVEngineImpl() override; - - nixl_mem_list_t - getSupportedMems() const override { - return {OBJ_SEG, DRAM_SEG}; - } - - nixl_status_t - registerMem(const nixlBlobDesc &mem, const nixl_mem_t &nixl_mem, nixlBackendMD *&out) override; - - nixl_status_t - deregisterMem(nixlBackendMD *meta) override; - - nixl_status_t - queryMem(const nixl_reg_dlist_t &descs, std::vector &resp) const override; - - nixl_status_t - prepXfer(const nixl_xfer_op_t &operation, - const nixl_meta_dlist_t &local, - const nixl_meta_dlist_t &remote, - const std::string &remote_agent, - const std::string &local_agent, - nixlBackendReqH *&handle, - const nixl_opt_b_args_t *opt_args) const override; - - nixl_status_t - postXfer(const nixl_xfer_op_t &operation, - const nixl_meta_dlist_t &local, - const nixl_meta_dlist_t &remote, - const std::string &remote_agent, - nixlBackendReqH *&handle, - const nixl_opt_b_args_t *opt_args) const override; - - nixl_status_t - checkXfer(nixlBackendReqH *handle) const override; - - nixl_status_t - releaseReqH(nixlBackendReqH *handle) const override; - -protected: - virtual iRedisClient * - getClient() const { - return redisClient_.get(); - } - - std::shared_ptr executor_; - std::shared_ptr redisClient_; - std::unordered_map devIdToRedisKey_; -}; - -#endif // NIXL_SRC_PLUGINS_KV_REDIS_ENGINE_IMPL_H diff --git a/src/plugins/kv/redis/redis_engine.cpp b/src/plugins/kv/redis/redis_engine.cpp deleted file mode 100644 index 99f0d6a49d..0000000000 --- a/src/plugins/kv/redis/redis_engine.cpp +++ /dev/null @@ -1,37 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * @file redis_engine.cpp - * @brief Factory wiring for nixlRedisKVEngine; creates nixlRedisKVEngineImpl at construction. - */ - -#include "redis_engine.h" -#include "engine_impl.h" -#include - -namespace { - -std::unique_ptr -createRedisKVEngineImpl(const nixlBackendInitParams *init_params) { - return std::make_unique(init_params); -} - -} // namespace - -nixlRedisKVEngine::nixlRedisKVEngine(const nixlBackendInitParams *init_params) - : nixlKVEngine(init_params, createRedisKVEngineImpl(init_params)) {} diff --git a/src/plugins/kv/redis/redis_engine.h b/src/plugins/kv/redis/redis_engine.h deleted file mode 100644 index 0055821ac2..0000000000 --- a/src/plugins/kv/redis/redis_engine.h +++ /dev/null @@ -1,103 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * @file redis_engine.h - * @brief REDIS KV plugin — iRedisClient interface and nixlRedisKVEngine entry. - * - */ - -#ifndef NIXL_SRC_PLUGINS_KV_REDIS_ENGINE_H -#define NIXL_SRC_PLUGINS_KV_REDIS_ENGINE_H - -#include "../kv_backend.h" -#include "redis_executor.h" -#include -#include -#include -#include -#include - -/** - * @class iRedisClient - * @brief Redis client for the REDIS KV backend. - * - * SET/GET are async (std::promise) for postXfer/checkXfer. - * EXISTS uses a separate synchronous hiredis connection for queryMem(). - */ -class iRedisClient { -public: - virtual ~iRedisClient() = default; - - /** - * @brief Set the executor used to complete async promises. - * @param executor ASIO thread pool shared with the engine implementation. - */ - virtual void - setExecutor(std::shared_ptr executor) = 0; - - /** - * @brief Asynchronously store a value under key (Redis SET). - * @param key Redis key. - * @param data_ptr Pointer to value bytes. - * @param data_len Number of bytes to write. - * @param promise Promise set to NIXL_SUCCESS or NIXL_ERR_BACKEND on completion. - */ - virtual void - putKeyAsync(std::string_view key, - uintptr_t data_ptr, - size_t data_len, - std::shared_ptr> promise) = 0; - - /** - * @brief Asynchronously read a value into buffer (Redis GET). - * @param key Redis key. - * @param data_ptr Output buffer pointer. - * @param data_len Maximum number of bytes to read. - * @param promise Promise set to NIXL_SUCCESS or NIXL_ERR_BACKEND on completion. - */ - virtual void - getKeyAsync(std::string_view key, - uintptr_t data_ptr, - size_t data_len, - std::shared_ptr> promise) = 0; - - /** - * @brief Synchronously check whether key exists (Redis EXISTS). - * - * Uses a dedicated blocking hiredis connection, independent of the async - * SET/GET path. Intended for queryMem() only. - * - * @param key Redis key. - * @return true if key exists, false if not found, std::nullopt on error. - */ - virtual std::optional - checkKeyExistsSync(std::string_view key) = 0; -}; - -/** - * @class nixlRedisKVEngine - * @brief REDIS backend engine registered with the NIXL plugin loader. - */ -class nixlRedisKVEngine : public nixlKVEngine { -public: - explicit nixlRedisKVEngine(const nixlBackendInitParams *init_params); - - ~nixlRedisKVEngine() override = default; -}; - -#endif // NIXL_SRC_PLUGINS_KV_REDIS_ENGINE_H diff --git a/src/plugins/kv/redis/redis_executor.h b/src/plugins/kv/redis/redis_executor.h deleted file mode 100644 index 7e1803f4ac..0000000000 --- a/src/plugins/kv/redis/redis_executor.h +++ /dev/null @@ -1,53 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef NIXL_SRC_PLUGINS_KV_REDIS_EXECUTOR_H -#define NIXL_SRC_PLUGINS_KV_REDIS_EXECUTOR_H - -#include -#include -#include - -/** - * @class redisThreadPoolExecutor - * @brief Small ASIO thread pool used to complete Redis async operations. - */ -class redisThreadPoolExecutor { -public: - explicit redisThreadPoolExecutor(std::size_t num_threads) : pool_(num_threads) {} - - void - waitUntilStopped() { - pool_.stop(); - pool_.join(); - } - - void - waitUntilIdle() { - pool_.wait(); - } - - void - post(std::function &&task) { - asio::post(pool_, std::move(task)); - } - -private: - asio::thread_pool pool_; -}; - -#endif // NIXL_SRC_PLUGINS_KV_REDIS_EXECUTOR_H diff --git a/src/plugins/meson.build b/src/plugins/meson.build index 26b42c1962..a7b2eb15dd 100644 --- a/src/plugins/meson.build +++ b/src/plugins/meson.build @@ -30,7 +30,7 @@ if enabled_plugins.get('OBJ') endif if enabled_plugins.get('REDIS') - subdir('kv') + subdir('redis') endif if enabled_plugins.get('AZURE_BLOB') diff --git a/src/plugins/redis/README.md b/src/plugins/redis/README.md new file mode 100644 index 0000000000..958def4211 --- /dev/null +++ b/src/plugins/redis/README.md @@ -0,0 +1,109 @@ + + +# NIXL Redis Plugin + +The `REDIS` plugin exposes Redis as a local NIXL storage backend. It supports asynchronous +`NIXL_WRITE`/`NIXL_READ` transfers and synchronous `queryMem()` checks. + +## Architecture + +The plugin follows the same direct-backend layout as `src/plugins/infinia`: + +```text +src/plugins/redis/ + redis_backend.h/.cpp nixlRedisKVEngine and NIXL transfer logic + redis_client.h/.cpp hiredis client and its unit-test interface + redis_plugin.cpp NIXL plugin registration + meson.build REDIS plugin target + README.md +``` + +`nixlRedisKVEngine` derives directly from `nixlBackendEngine`. The internal `iRedisClient` +interface isolates hiredis/libevent and permits Redis-free unit tests; it is not a generic KV +extension API. + +The production client uses: + +- One hiredis/libevent async connection for `SET` and `GET`. +- One blocking hiredis connection for `EXISTS`. +- One libevent thread. Hiredis callbacks complete NIXL request promises directly. + +## Dependencies + +- `hiredis` with async API support +- `libevent` +- `libevent_pthreads` + +If REDIS is explicitly enabled or selected as a static plugin, missing hiredis/libevent is a +configuration error. During a default all-plugin build, missing dependencies cause REDIS to be +skipped with a warning. + +## Configuration + +Backend parameters take precedence over environment variables. + +| Parameter | Environment fallback | Default | Description | +|-----------|----------------------|---------|-------------| +| `host` | `REDIS_HOST` | `localhost` | Redis hostname or IP address | +| `port` | `REDIS_PORT` | `6379` | Redis TCP port | +| `password` | `REDIS_PASSWORD` | empty | Redis AUTH password | +| `db` | none | `0` | Redis logical database | + +```cpp +nixl_b_params_t params = { + {"host", "127.0.0.1"}, + {"port", "6379"}, + {"db", "2"}, +}; +agent.createBackend("REDIS", params); +``` + +## Transfer behavior + +| NIXL operation | Redis operation | Result | +|----------------|-----------------|--------| +| `NIXL_WRITE` | `SET key bytes` | Stores local DRAM bytes | +| `NIXL_READ` | `GET key` | Copies the exact Redis value into local DRAM | +| `queryMem` | `EXISTS key` | Reports whether the key exists | + +Local descriptors must be `DRAM_SEG`; remote descriptors may be `DRAM_SEG` or `OBJ_SEG`. A Redis +key comes from descriptor `metaInfo` when present, otherwise from the decimal `devId`. `postXfer()` +resolves every remote key before dispatching commands, so an invalid descriptor cannot produce a +partially submitted transfer. + +`postXfer()` returns `NIXL_IN_PROG`; `checkXfer()` polls the request futures and returns success or +the first backend error once all completed work has been observed. + +## Build and test + +```bash +meson setup build -Denable_plugins=REDIS +meson compile -C build REDIS +meson test -C build unit --print-errorlogs +``` + +The dynamic plugin is produced at: + +```text +build/src/plugins/redis/libplugin_REDIS.so +``` + +For a static build: + +```bash +meson setup build -Denable_plugins=REDIS -Dstatic_plugins=REDIS +meson compile -C build +``` + +For a live smoke test, start Redis and run one write and one read: + +```bash +export REDIS_HOST=127.0.0.1 REDIS_PORT=6379 +nixlbench --backend REDIS --start_block_size 4096 --max_block_size 4096 \ + --num_iter 10 --warmup_iter 1 --op_type WRITE +nixlbench --backend REDIS --start_block_size 4096 --max_block_size 4096 \ + --num_iter 10 --warmup_iter 1 --op_type READ +``` diff --git a/src/plugins/kv/redis/meson.build b/src/plugins/redis/meson.build similarity index 75% rename from src/plugins/kv/redis/meson.build rename to src/plugins/redis/meson.build index 5eaeee782d..3b01d8304a 100644 --- a/src/plugins/kv/redis/meson.build +++ b/src/plugins/redis/meson.build @@ -1,35 +1,16 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# REDIS implementation sources live in this directory; the plugin entrypoint -# stays in the shared KV layer to match the OBJ plugin layout. if not enabled_plugins.get('REDIS') subdir_done() endif redis_sources = [ - '../kv_backend.cpp', - '../kv_plugin.cpp', - 'redis_engine.cpp', - 'redis_engine.h', - 'redis_executor.h', - 'client.cpp', - 'client.h', - 'engine_impl.cpp', - 'engine_impl.h', + 'redis_backend.cpp', + 'redis_backend.h', + 'redis_client.cpp', + 'redis_client.h', + 'redis_plugin.cpp', ] hiredis_async_dep = dependency('hiredis_async', required: false) @@ -71,11 +52,9 @@ compile_defs_redis = [] if is_variable('compile_defs') compile_defs_redis = compile_defs endif -if hiredis_async_dep.found() and libevent_dep.found() - compile_defs_redis += ['-DHAVE_HIREDIS_ASYNC'] -endif +compile_defs_redis += ['-DHAVE_HIREDIS_ASYNC'] -redis_plugin_deps = [nixl_infra, nixl_common_dep, hiredis_async_dep, libevent_dep, dependency('asio', required: true)] +redis_plugin_deps = [nixl_infra, nixl_common_dep, hiredis_async_dep, libevent_dep] if libevent_pthreads_dep.found() redis_plugin_deps += [libevent_pthreads_dep] else diff --git a/src/plugins/kv/redis/engine_impl.cpp b/src/plugins/redis/redis_backend.cpp similarity index 53% rename from src/plugins/kv/redis/engine_impl.cpp rename to src/plugins/redis/redis_backend.cpp index 4c92bd87b0..b63f565a3d 100644 --- a/src/plugins/kv/redis/engine_impl.cpp +++ b/src/plugins/redis/redis_backend.cpp @@ -1,28 +1,12 @@ /* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. */ -/** - * @file engine_impl.cpp - * @brief nixlRedisKVEngineImpl — postXfer/checkXfer over async Redis SET/GET. - */ +#include "redis_backend.h" -#include "engine_impl.h" -#include "client.h" #include "common/nixl_log.h" + #include #include #include @@ -31,36 +15,13 @@ #include #include #include -#include +#include #include namespace { -std::size_t -defaultNumThreads() { - return std::max(1u, std::thread::hardware_concurrency() / 2); -} - -std::size_t -getNumThreads(nixl_b_params_t *custom_params) { - if (custom_params && custom_params->count("num_threads") > 0) { - try { - const auto num_threads = std::stoul(custom_params->at("num_threads")); - if (num_threads > 0) { - return num_threads; - } - NIXL_WARN << "Invalid num_threads value 0, using default"; - } - catch (const std::exception &) { - NIXL_WARN << "Invalid num_threads value, using default"; - } - } - - return defaultNumThreads(); -} - nixl_b_params_t * -getCustomParams(const nixlBackendInitParams *init_params) { +getInitCustomParams(const nixlBackendInitParams *init_params) { return init_params ? init_params->customParams : nullptr; } @@ -118,19 +79,19 @@ class nixlRedisBackendReqH : public nixlBackendReqH { nixl_status_t getOverallStatus() { while (!statusFutures_.empty()) { - if (statusFutures_.back().wait_for(std::chrono::seconds(0)) == + if (statusFutures_.back().wait_for(std::chrono::seconds(0)) != std::future_status::ready) { - auto current_status = statusFutures_.back().get(); - if (current_status != NIXL_SUCCESS) { - statusFutures_.clear(); - statusPromises_.clear(); - return current_status; - } - statusFutures_.pop_back(); - statusPromises_.pop_back(); - } else { return NIXL_IN_PROG; } + + auto current_status = statusFutures_.back().get(); + if (current_status != NIXL_SUCCESS) { + statusFutures_.clear(); + statusPromises_.clear(); + return current_status; + } + statusFutures_.pop_back(); + statusPromises_.pop_back(); } return NIXL_SUCCESS; } @@ -151,54 +112,37 @@ class nixlRedisMetadata : public nixlBackendMD { } // namespace -nixlRedisKVEngineImpl::nixlRedisKVEngineImpl(const nixlBackendInitParams *init_params) - : executor_(std::make_shared( - getNumThreads(getCustomParams(init_params)))) { - redisClient_ = std::make_shared(getCustomParams(init_params), executor_); - NIXL_INFO << "Redis KV backend initialized"; +nixlRedisKVEngine::nixlRedisKVEngine(const nixlBackendInitParams *init_params) + : nixlBackendEngine(init_params), + redisClient_(std::make_shared(getInitCustomParams(init_params))) { + NIXL_INFO << "Redis backend initialized"; } -nixlRedisKVEngineImpl::nixlRedisKVEngineImpl(const nixlBackendInitParams *init_params, - std::shared_ptr redis_client) - : executor_(std::make_shared( - getNumThreads(getCustomParams(init_params)))), - redisClient_(std::move(redis_client)) { - if (redisClient_) { - redisClient_->setExecutor(executor_); - } -} - -nixlRedisKVEngineImpl::~nixlRedisKVEngineImpl() { - if (executor_) { - executor_->waitUntilStopped(); - } -} +nixlRedisKVEngine::nixlRedisKVEngine(const nixlBackendInitParams *init_params, + std::shared_ptr redis_client) + : nixlBackendEngine(init_params), + redisClient_(std::move(redis_client)) {} nixl_status_t -nixlRedisKVEngineImpl::registerMem(const nixlBlobDesc &mem, - const nixl_mem_t &nixl_mem, - nixlBackendMD *&out) { - auto supported_mems = getSupportedMems(); - if (std::find(supported_mems.begin(), supported_mems.end(), nixl_mem) == supported_mems.end()) { +nixlRedisKVEngine::registerMem(const nixlBlobDesc &mem, + const nixl_mem_t &nixl_mem, + nixlBackendMD *&out) { + const auto supported_mems = getSupportedMems(); + if (std::find(supported_mems.begin(), supported_mems.end(), nixl_mem) == + supported_mems.end()) { out = nullptr; return NIXL_ERR_NOT_SUPPORTED; } std::string redis_key = mem.metaInfo.empty() ? std::to_string(mem.devId) : mem.metaInfo; - - if (nixl_mem == OBJ_SEG || nixl_mem == DRAM_SEG) { - auto redis_md = std::make_unique(nixl_mem, mem.devId, redis_key); - devIdToRedisKey_[mem.devId] = redis_key; - out = redis_md.release(); - } else { - out = nullptr; - } - + auto redis_md = std::make_unique(nixl_mem, mem.devId, redis_key); + devIdToRedisKey_[mem.devId] = redis_key; + out = redis_md.release(); return NIXL_SUCCESS; } nixl_status_t -nixlRedisKVEngineImpl::deregisterMem(nixlBackendMD *meta) { +nixlRedisKVEngine::deregisterMem(nixlBackendMD *meta) { auto *redis_md = static_cast(meta); if (redis_md) { std::unique_ptr redis_md_ptr(redis_md); @@ -208,17 +152,15 @@ nixlRedisKVEngineImpl::deregisterMem(nixlBackendMD *meta) { } nixl_status_t -nixlRedisKVEngineImpl::queryMem(const nixl_reg_dlist_t &descs, - std::vector &resp) const { - iRedisClient *client = getClient(); - if (!client) { +nixlRedisKVEngine::queryMem(const nixl_reg_dlist_t &descs, + std::vector &resp) const { + if (!redisClient_) { NIXL_ERROR << "Failed to query memory: no Redis client available"; return NIXL_ERR_BACKEND; } resp.clear(); resp.reserve(descs.descCount()); - bool has_error = false; try { @@ -226,8 +168,7 @@ nixlRedisKVEngineImpl::queryMem(const nixl_reg_dlist_t &descs, const auto &desc = descs[i]; const std::string key = desc.metaInfo.empty() ? std::to_string(desc.devId) : desc.metaInfo; - - std::optional exists = client->checkKeyExistsSync(key); + const auto exists = redisClient_->checkKeyExistsSync(key); if (!exists.has_value()) { resp.emplace_back(std::nullopt); has_error = true; @@ -241,42 +182,32 @@ nixlRedisKVEngineImpl::queryMem(const nixl_reg_dlist_t &descs, return NIXL_ERR_BACKEND; } - if (has_error) { - return NIXL_ERR_BACKEND; - } - - return NIXL_SUCCESS; + return has_error ? NIXL_ERR_BACKEND : NIXL_SUCCESS; } nixl_status_t -nixlRedisKVEngineImpl::prepXfer(const nixl_xfer_op_t &operation, - const nixl_meta_dlist_t &local, - const nixl_meta_dlist_t &remote, - const std::string &remote_agent, - const std::string &local_agent, - nixlBackendReqH *&handle, - const nixl_opt_b_args_t *opt_args) const { - if (!isValidPrepXferParams(operation, local, remote, remote_agent, local_agent)) { +nixlRedisKVEngine::prepXfer(const nixl_xfer_op_t &operation, + const nixl_meta_dlist_t &local, + const nixl_meta_dlist_t &remote, + const std::string &remote_agent, + nixlBackendReqH *&handle, + const nixl_opt_b_args_t *) const { + if (!isValidPrepXferParams(operation, local, remote, remote_agent, localAgent)) { return NIXL_ERR_INVALID_PARAM; } - auto req_h = std::make_unique(); - handle = req_h.release(); + handle = new nixlRedisBackendReqH(); return NIXL_SUCCESS; } nixl_status_t -nixlRedisKVEngineImpl::postXfer(const nixl_xfer_op_t &operation, - const nixl_meta_dlist_t &local, - const nixl_meta_dlist_t &remote, - const std::string &remote_agent, - nixlBackendReqH *&handle, - const nixl_opt_b_args_t *opt_args) const { - if (!handle) { - return NIXL_ERR_INVALID_PARAM; - } - - if (operation != NIXL_WRITE && operation != NIXL_READ) { +nixlRedisKVEngine::postXfer(const nixl_xfer_op_t &operation, + const nixl_meta_dlist_t &local, + const nixl_meta_dlist_t &remote, + const std::string &, + nixlBackendReqH *&handle, + const nixl_opt_b_args_t *) const { + if (!handle || (operation != NIXL_WRITE && operation != NIXL_READ)) { return NIXL_ERR_INVALID_PARAM; } @@ -288,34 +219,30 @@ nixlRedisKVEngineImpl::postXfer(const nixl_xfer_op_t &operation, return NIXL_ERR_INVALID_PARAM; } - auto *req_h = static_cast(handle); - iRedisClient *client = getClient(); - if (!client) { + if (!redisClient_) { return NIXL_ERR_BACKEND; } + auto *req_h = static_cast(handle); req_h->statusFutures_.clear(); req_h->statusPromises_.clear(); + // Resolve every key before dispatching any command so invalid descriptors cannot + // produce a partially submitted Redis transfer. std::vector redis_keys; - redis_keys.reserve(local.descCount()); - + redis_keys.reserve(remote.descCount()); for (int i = 0; i < remote.descCount(); ++i) { const auto &remote_desc = remote[i]; - std::string redis_key; + if (remote_desc.metadataP) { auto *redis_md = dynamic_cast(remote_desc.metadataP); if (redis_md) { redis_key = redis_md->redisKey; - } else { - auto it = devIdToRedisKey_.find(remote_desc.devId); - if (it == devIdToRedisKey_.end()) { - return NIXL_ERR_INVALID_PARAM; - } - redis_key = it->second; } - } else { + } + + if (redis_key.empty()) { auto it = devIdToRedisKey_.find(remote_desc.devId); if (it == devIdToRedisKey_.end()) { return NIXL_ERR_INVALID_PARAM; @@ -327,19 +254,16 @@ nixlRedisKVEngineImpl::postXfer(const nixl_xfer_op_t &operation, for (int i = 0; i < local.descCount(); ++i) { const auto &local_desc = local[i]; - const auto &redis_key = redis_keys[i]; - auto status_promise = std::make_shared>(); req_h->statusPromises_.push_back(status_promise); req_h->statusFutures_.push_back(status_promise->get_future()); - uintptr_t data_ptr = local_desc.addr; - size_t data_len = local_desc.len; - if (operation == NIXL_WRITE) { - client->putKeyAsync(redis_key, data_ptr, data_len, status_promise); + redisClient_->putKeyAsync( + redis_keys[i], local_desc.addr, local_desc.len, status_promise); } else { - client->getKeyAsync(redis_key, data_ptr, data_len, status_promise); + redisClient_->getKeyAsync( + redis_keys[i], local_desc.addr, local_desc.len, status_promise); } } @@ -347,7 +271,7 @@ nixlRedisKVEngineImpl::postXfer(const nixl_xfer_op_t &operation, } nixl_status_t -nixlRedisKVEngineImpl::checkXfer(nixlBackendReqH *handle) const { +nixlRedisKVEngine::checkXfer(nixlBackendReqH *handle) const { if (!handle) { return NIXL_ERR_INVALID_PARAM; } @@ -355,7 +279,7 @@ nixlRedisKVEngineImpl::checkXfer(nixlBackendReqH *handle) const { } nixl_status_t -nixlRedisKVEngineImpl::releaseReqH(nixlBackendReqH *handle) const { +nixlRedisKVEngine::releaseReqH(nixlBackendReqH *handle) const { delete static_cast(handle); return NIXL_SUCCESS; } diff --git a/src/plugins/redis/redis_backend.h b/src/plugins/redis/redis_backend.h new file mode 100644 index 0000000000..5d879449e7 --- /dev/null +++ b/src/plugins/redis/redis_backend.h @@ -0,0 +1,109 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef NIXL_SRC_PLUGINS_REDIS_REDIS_BACKEND_H +#define NIXL_SRC_PLUGINS_REDIS_REDIS_BACKEND_H + +#include "backend/backend_engine.h" +#include "redis_client.h" + +#include +#include +#include +#include + +inline constexpr const char *REDIS_PLUGIN_NAME = "REDIS"; +inline constexpr const char *REDIS_PLUGIN_VERSION = "0.1.0"; + +/** NIXL backend engine for Redis key-value storage. */ +class nixlRedisKVEngine : public nixlBackendEngine { +public: + explicit nixlRedisKVEngine(const nixlBackendInitParams *init_params); + nixlRedisKVEngine(const nixlBackendInitParams *init_params, + std::shared_ptr redis_client); + ~nixlRedisKVEngine() override = default; + + bool + supportsRemote() const override { + return false; + } + + bool + supportsLocal() const override { + return true; + } + + bool + supportsNotif() const override { + return false; + } + + nixl_mem_list_t + getSupportedMems() const override { + return {OBJ_SEG, DRAM_SEG}; + } + + nixl_status_t + registerMem(const nixlBlobDesc &mem, const nixl_mem_t &nixl_mem, nixlBackendMD *&out) override; + + nixl_status_t + deregisterMem(nixlBackendMD *meta) override; + + nixl_status_t + queryMem(const nixl_reg_dlist_t &descs, std::vector &resp) const override; + + nixl_status_t + connect(const std::string &) override { + return NIXL_SUCCESS; + } + + nixl_status_t + disconnect(const std::string &) override { + return NIXL_SUCCESS; + } + + nixl_status_t + unloadMD(nixlBackendMD *) override { + return NIXL_SUCCESS; + } + + nixl_status_t + loadLocalMD(nixlBackendMD *input, nixlBackendMD *&output) override { + if (!input) { + output = nullptr; + return NIXL_ERR_INVALID_PARAM; + } + output = input; + return NIXL_SUCCESS; + } + + nixl_status_t + prepXfer(const nixl_xfer_op_t &operation, + const nixl_meta_dlist_t &local, + const nixl_meta_dlist_t &remote, + const std::string &remote_agent, + nixlBackendReqH *&handle, + const nixl_opt_b_args_t *opt_args = nullptr) const override; + + nixl_status_t + postXfer(const nixl_xfer_op_t &operation, + const nixl_meta_dlist_t &local, + const nixl_meta_dlist_t &remote, + const std::string &remote_agent, + nixlBackendReqH *&handle, + const nixl_opt_b_args_t *opt_args = nullptr) const override; + + nixl_status_t + checkXfer(nixlBackendReqH *handle) const override; + + nixl_status_t + releaseReqH(nixlBackendReqH *handle) const override; + +private: + std::shared_ptr redisClient_; + std::unordered_map devIdToRedisKey_; +}; + +#endif // NIXL_SRC_PLUGINS_REDIS_REDIS_BACKEND_H diff --git a/src/plugins/kv/redis/client.cpp b/src/plugins/redis/redis_client.cpp similarity index 90% rename from src/plugins/kv/redis/client.cpp rename to src/plugins/redis/redis_client.cpp index 33f2a79213..8319993d2f 100644 --- a/src/plugins/kv/redis/client.cpp +++ b/src/plugins/redis/redis_client.cpp @@ -15,7 +15,7 @@ * limitations under the License. */ -#include "client.h" +#include "redis_client.h" #include "common/nixl_log.h" #include #include @@ -122,25 +122,16 @@ checkRedisReplyOk(redisReply *reply, const char *command) { } void -setPromiseStatus(const std::shared_ptr &executor, - const std::shared_ptr> &promise, - bool success) { +setPromiseStatus(const std::shared_ptr> &promise, bool success) { if (!promise) { return; } - if (executor) { - executor->post([promise, success]() { - promise->set_value(success ? NIXL_SUCCESS : NIXL_ERR_BACKEND); - }); - } else { - promise->set_value(success ? NIXL_SUCCESS : NIXL_ERR_BACKEND); - } + promise->set_value(success ? NIXL_SUCCESS : NIXL_ERR_BACKEND); } } // namespace -hiredisAsyncClient::hiredisAsyncClient(nixl_b_params_t *custom_params, - std::shared_ptr executor) +hiredisAsyncClient::hiredisAsyncClient(nixl_b_params_t *custom_params) : host_(getRedisHost(custom_params)), port_(getRedisPort(custom_params)), password_(getRedisPassword(custom_params)), @@ -148,7 +139,6 @@ hiredisAsyncClient::hiredisAsyncClient(nixl_b_params_t *custom_params, asyncContext_(nullptr), syncContext_(nullptr), eventBase_(nullptr), - executor_(executor), connected_(false), initDone_(false), initSucceeded_(false) { @@ -258,11 +248,6 @@ hiredisAsyncClient::~hiredisAsyncClient() { } } -void -hiredisAsyncClient::setExecutor(std::shared_ptr executor) { - executor_ = executor; -} - void hiredisAsyncClient::processEventLoop() { event_base_dispatch(eventBase_); @@ -419,9 +404,8 @@ hiredisAsyncClient::setCallback(redisAsyncContext *c, void *reply, void *privdat } auto promise_ptr = ctx->promise_ptr; - auto executor = ctx->executor; delete ctx; - setPromiseStatus(executor, promise_ptr, success); + setPromiseStatus(promise_ptr, success); } void @@ -450,9 +434,8 @@ hiredisAsyncClient::getCallback(redisAsyncContext *c, void *reply, void *privdat } auto promise_ptr = ctx->promise_ptr; - auto executor = ctx->executor; delete ctx; - setPromiseStatus(executor, promise_ptr, success); + setPromiseStatus(promise_ptr, success); } void @@ -461,25 +444,22 @@ hiredisAsyncClient::putKeyAsync(std::string_view key, size_t data_len, std::shared_ptr> promise) { if (!connected_.load()) { - setPromiseStatus(executor_, promise, false); + setPromiseStatus(promise, false); return; } std::string key_copy(key); - auto executor = executor_; bool scheduled = scheduleOnEventLoop([this, key = std::move(key_copy), data_ptr, data_len, - promise, - executor]() mutable { + promise]() mutable { if (!connected_.load() || !asyncContext_) { - setPromiseStatus(executor, promise, false); + setPromiseStatus(promise, false); return; } auto *ctx = new CallbackContext; - ctx->executor = executor; ctx->promise_ptr = promise; int ret = redisAsyncCommand(asyncContext_, @@ -494,12 +474,12 @@ hiredisAsyncClient::putKeyAsync(std::string_view key, if (ret != REDIS_OK) { auto promise_ptr = ctx->promise_ptr; delete ctx; - setPromiseStatus(executor, promise_ptr, false); + setPromiseStatus(promise_ptr, false); } }); if (!scheduled) { - setPromiseStatus(executor_, promise, false); + setPromiseStatus(promise, false); } } @@ -509,25 +489,22 @@ hiredisAsyncClient::getKeyAsync(std::string_view key, size_t data_len, std::shared_ptr> promise) { if (!connected_.load()) { - setPromiseStatus(executor_, promise, false); + setPromiseStatus(promise, false); return; } std::string key_copy(key); - auto executor = executor_; bool scheduled = scheduleOnEventLoop([this, key = std::move(key_copy), data_ptr, data_len, - promise, - executor]() mutable { + promise]() mutable { if (!connected_.load() || !asyncContext_) { - setPromiseStatus(executor, promise, false); + setPromiseStatus(promise, false); return; } auto *ctx = new CallbackContext; - ctx->executor = executor; ctx->data_ptr = data_ptr; ctx->data_len = data_len; ctx->promise_ptr = promise; @@ -538,12 +515,12 @@ hiredisAsyncClient::getKeyAsync(std::string_view key, if (ret != REDIS_OK) { auto promise_ptr = ctx->promise_ptr; delete ctx; - setPromiseStatus(executor, promise_ptr, false); + setPromiseStatus(promise_ptr, false); } }); if (!scheduled) { - setPromiseStatus(executor_, promise, false); + setPromiseStatus(promise, false); } } @@ -583,16 +560,12 @@ hiredisAsyncClient::checkKeyExistsSync(std::string_view key) { #else // HAVE_HIREDIS_ASYNC -hiredisAsyncClient::hiredisAsyncClient(nixl_b_params_t *custom_params, - std::shared_ptr executor) { +hiredisAsyncClient::hiredisAsyncClient(nixl_b_params_t *custom_params) { throw std::runtime_error("hiredis-async not available"); } hiredisAsyncClient::~hiredisAsyncClient() {} -void -hiredisAsyncClient::setExecutor(std::shared_ptr executor) {} - void hiredisAsyncClient::putKeyAsync(std::string_view key, uintptr_t data_ptr, diff --git a/src/plugins/kv/redis/client.h b/src/plugins/redis/redis_client.h similarity index 60% rename from src/plugins/kv/redis/client.h rename to src/plugins/redis/redis_client.h index 3f816003c8..a494a920e5 100644 --- a/src/plugins/kv/redis/client.h +++ b/src/plugins/redis/redis_client.h @@ -1,33 +1,13 @@ /* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. */ -/** - * @file client.h - * @brief hiredis Redis client implementing iRedisClient. - * - * Two connections: - * - async (hiredis-async + libevent): SET/GET for postXfer - * - sync (blocking hiredis): EXISTS for queryMem - */ +#ifndef NIXL_SRC_PLUGINS_REDIS_REDIS_CLIENT_H +#define NIXL_SRC_PLUGINS_REDIS_REDIS_CLIENT_H -#ifndef NIXL_SRC_PLUGINS_KV_REDIS_CLIENT_H -#define NIXL_SRC_PLUGINS_KV_REDIS_CLIENT_H +#include "nixl_types.h" -#include "redis_engine.h" #include #include #include @@ -38,38 +18,45 @@ #include #include #include -#include "nixl_types.h" #ifdef HAVE_HIREDIS_ASYNC -#include -#include -#include #include #include +#include +#include +#include #else struct event_base; struct redisAsyncContext; struct redisContext; #endif -/** - * @class hiredisAsyncClient - * @brief iRedisClient with async SET/GET and a separate sync EXISTS connection. - */ -class hiredisAsyncClient : public iRedisClient { +/** Redis operations used by the NIXL REDIS backend. */ +class iRedisClient { public: - /** - * @brief Construct a client from NIXL custom params and environment variables. - * @param custom_params Optional host/port/password/db overrides. - * @param executor Executor used to complete async promises. - */ - hiredisAsyncClient(nixl_b_params_t *custom_params, - std::shared_ptr executor = nullptr); + virtual ~iRedisClient() = default; - ~hiredisAsyncClient() override; + virtual void + putKeyAsync(std::string_view key, + uintptr_t data_ptr, + size_t data_len, + std::shared_ptr> promise) = 0; - void - setExecutor(std::shared_ptr executor) override; + virtual void + getKeyAsync(std::string_view key, + uintptr_t data_ptr, + size_t data_len, + std::shared_ptr> promise) = 0; + + virtual std::optional + checkKeyExistsSync(std::string_view key) = 0; +}; + +/** Hiredis implementation with async SET/GET and a separate synchronous EXISTS connection. */ +class hiredisAsyncClient : public iRedisClient { +public: + explicit hiredisAsyncClient(nixl_b_params_t *custom_params); + ~hiredisAsyncClient() override; void putKeyAsync(std::string_view key, @@ -88,7 +75,6 @@ class hiredisAsyncClient : public iRedisClient { private: struct CallbackContext { - std::shared_ptr executor; uintptr_t data_ptr; size_t data_len; std::shared_ptr> promise_ptr; @@ -96,43 +82,31 @@ class hiredisAsyncClient : public iRedisClient { static void connectCallback(const redisAsyncContext *c, int status); - static void disconnectCallback(const redisAsyncContext *c, int status); - static void authCallback(redisAsyncContext *c, void *reply, void *privdata); - static void selectCallback(redisAsyncContext *c, void *reply, void *privdata); - static void setCallback(redisAsyncContext *c, void *reply, void *privdata); - static void getCallback(redisAsyncContext *c, void *reply, void *privdata); void processEventLoop(); - void connectSyncContext(); - void startAsyncInitialization(); - void startAsyncSelect(); - void completeAsyncInitialization(bool success); - void freeAsyncContextInEventLoop(); - bool scheduleOnEventLoop(std::function task); - void stopEventLoop(); @@ -143,7 +117,6 @@ class hiredisAsyncClient : public iRedisClient { redisAsyncContext *asyncContext_; redisContext *syncContext_; struct event_base *eventBase_; - std::shared_ptr executor_; std::thread eventLoopThread_; std::atomic connected_; std::atomic initDone_; @@ -151,4 +124,4 @@ class hiredisAsyncClient : public iRedisClient { mutable std::mutex syncMutex_; }; -#endif // NIXL_SRC_PLUGINS_KV_REDIS_CLIENT_H +#endif // NIXL_SRC_PLUGINS_REDIS_REDIS_CLIENT_H diff --git a/src/plugins/kv/kv_plugin.cpp b/src/plugins/redis/redis_plugin.cpp similarity index 66% rename from src/plugins/kv/kv_plugin.cpp rename to src/plugins/redis/redis_plugin.cpp index 3622672a28..31edc719a3 100644 --- a/src/plugins/kv/kv_plugin.cpp +++ b/src/plugins/redis/redis_plugin.cpp @@ -15,10 +15,8 @@ * limitations under the License. */ -#include "nixl_types.h" -#include "redis/redis_engine.h" #include "backend/backend_plugin.h" -#include "common/nixl_log.h" +#include "redis_backend.h" using redis_plugin_t = nixlBackendPluginCreator; @@ -27,14 +25,20 @@ static const nixl_mem_list_t supported_segments = {DRAM_SEG, OBJ_SEG}; #ifdef STATIC_PLUGIN_REDIS nixlBackendPlugin * createStaticREDISPlugin() { - return redis_plugin_t::create( - NIXL_PLUGIN_API_VERSION, "REDIS", "0.1.0", {}, supported_segments); + return redis_plugin_t::create(NIXL_PLUGIN_API_VERSION, + REDIS_PLUGIN_NAME, + REDIS_PLUGIN_VERSION, + {}, + supported_segments); } #else extern "C" NIXL_PLUGIN_EXPORT nixlBackendPlugin * nixl_plugin_init() { - return redis_plugin_t::create( - NIXL_PLUGIN_API_VERSION, "REDIS", "0.1.0", {}, supported_segments); + return redis_plugin_t::create(NIXL_PLUGIN_API_VERSION, + REDIS_PLUGIN_NAME, + REDIS_PLUGIN_VERSION, + {}, + supported_segments); } extern "C" NIXL_PLUGIN_EXPORT void diff --git a/test/gtest/unit/redis/meson.build b/test/gtest/unit/redis/meson.build index 18eabd022e..23c636d458 100644 --- a/test/gtest/unit/redis/meson.build +++ b/test/gtest/unit/redis/meson.build @@ -19,11 +19,9 @@ redis_unit_test_dep = declare_dependency( ], include_directories: [ nixl_inc_dirs, utils_inc_dirs, - '../../../../src/plugins/kv', - '../../../../src/plugins/kv/redis', + '../../../../src/plugins/redis', ], dependencies: [ redis_backend_interface, - dependency('asio', required: true), ], ) diff --git a/test/gtest/unit/redis/redis.cpp b/test/gtest/unit/redis/redis.cpp index bc7da6e07b..494b2aed12 100644 --- a/test/gtest/unit/redis/redis.cpp +++ b/test/gtest/unit/redis/redis.cpp @@ -1,24 +1,13 @@ /* * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. */ #include -#include "engine_impl.h" +#include "redis_backend.h" +#include #include #include #include @@ -31,18 +20,13 @@ namespace gtest::redis { class mockRedisClient : public iRedisClient { public: - void - setExecutor(std::shared_ptr executor) override { - executor_ = std::move(executor); - } - void putKeyAsync(std::string_view key, uintptr_t, size_t, std::shared_ptr> promise) override { putKeys_.emplace_back(key); - promise->set_value(NIXL_SUCCESS); + completeOrQueue(std::move(promise)); } void @@ -51,13 +35,41 @@ class mockRedisClient : public iRedisClient { size_t, std::shared_ptr> promise) override { getKeys_.emplace_back(key); - promise->set_value(NIXL_SUCCESS); + completeOrQueue(std::move(promise)); } std::optional checkKeyExistsSync(std::string_view key) override { checkedKeys_.emplace_back(key); - return true; + if (existsResults_.empty()) { + return true; + } + auto result = existsResults_.front(); + existsResults_.pop_front(); + return result; + } + + void + setCompletionStatus(nixl_status_t status) { + completionStatus_ = status; + } + + void + setCompleteImmediately(bool value) { + completeImmediately_ = value; + } + + void + completePending(nixl_status_t status) { + for (auto &promise : pendingPromises_) { + promise->set_value(status); + } + pendingPromises_.clear(); + } + + void + setExistsResults(std::deque> results) { + existsResults_ = std::move(results); } const std::vector & @@ -65,8 +77,30 @@ class mockRedisClient : public iRedisClient { return putKeys_; } + const std::vector & + getKeys() const { + return getKeys_; + } + + const std::vector & + checkedKeys() const { + return checkedKeys_; + } + private: - std::shared_ptr executor_; + void + completeOrQueue(std::shared_ptr> promise) { + if (completeImmediately_) { + promise->set_value(completionStatus_); + } else { + pendingPromises_.push_back(std::move(promise)); + } + } + + nixl_status_t completionStatus_ = NIXL_SUCCESS; + bool completeImmediately_ = true; + std::deque> existsResults_; + std::vector>> pendingPromises_; std::vector putKeys_; std::vector getKeys_; std::vector checkedKeys_; @@ -84,23 +118,187 @@ class redisEngineTest : public ::testing::Test { initParams_.syncMode = nixl_thread_sync_t::NIXL_THREAD_SYNC_RW; mockClient_ = std::make_shared(); - engine_ = std::make_unique(&initParams_, mockClient_); + engine_ = std::make_unique(&initParams_, mockClient_); + } + + nixlBackendMD * + registerRemote(uint64_t dev_id, std::string key, nixl_mem_t type = OBJ_SEG) { + nixlBackendMD *metadata = nullptr; + EXPECT_EQ(engine_->registerMem(nixlBlobDesc(0, 16, dev_id, key), type, metadata), + NIXL_SUCCESS); + EXPECT_NE(metadata, nullptr); + return metadata; + } + + nixlBackendReqH * + prepareTransfer(nixl_xfer_op_t operation, + nixl_meta_dlist_t &local, + nixl_meta_dlist_t &remote) { + nixlBackendReqH *handle = nullptr; + EXPECT_EQ(engine_->prepXfer( + operation, local, remote, initParams_.localAgent, handle, nullptr), + NIXL_SUCCESS); + EXPECT_NE(handle, nullptr); + return handle; } nixlBackendInitParams initParams_; nixl_b_params_t customParams_; std::shared_ptr mockClient_; - std::unique_ptr engine_; + std::unique_ptr engine_; }; +TEST_F(redisEngineTest, ReportsRedisBackendCapabilities) { + EXPECT_FALSE(engine_->supportsRemote()); + EXPECT_TRUE(engine_->supportsLocal()); + EXPECT_FALSE(engine_->supportsNotif()); + EXPECT_EQ(engine_->getSupportedMems(), (nixl_mem_list_t{OBJ_SEG, DRAM_SEG})); + + nixlBackendMD *output = reinterpret_cast(1); + EXPECT_EQ(engine_->loadLocalMD(nullptr, output), NIXL_ERR_INVALID_PARAM); + EXPECT_EQ(output, nullptr); + EXPECT_EQ(engine_->connect(initParams_.localAgent), NIXL_SUCCESS); + EXPECT_EQ(engine_->disconnect(initParams_.localAgent), NIXL_SUCCESS); +} + +TEST_F(redisEngineTest, RegistersMetadataKeyAndDevIdFallback) { + auto *namedMetadata = registerRemote(11, "registered-key"); + auto *fallbackMetadata = registerRemote(22, ""); + + std::vector firstBuffer(16, 'a'); + std::vector secondBuffer(16, 'b'); + nixl_meta_dlist_t localDescs(DRAM_SEG); + localDescs.addDesc(nixlMetaDesc( + reinterpret_cast(firstBuffer.data()), firstBuffer.size(), 1, nullptr)); + localDescs.addDesc(nixlMetaDesc( + reinterpret_cast(secondBuffer.data()), secondBuffer.size(), 2, nullptr)); + nixl_meta_dlist_t remoteDescs(OBJ_SEG); + remoteDescs.addDesc(nixlMetaDesc(0, firstBuffer.size(), 11, nullptr)); + remoteDescs.addDesc(nixlMetaDesc(0, secondBuffer.size(), 22, nullptr)); + + auto *handle = prepareTransfer(NIXL_WRITE, localDescs, remoteDescs); + EXPECT_EQ(engine_->postXfer( + NIXL_WRITE, localDescs, remoteDescs, initParams_.localAgent, handle, nullptr), + NIXL_IN_PROG); + EXPECT_EQ(mockClient_->putKeys(), (std::vector{"registered-key", "22"})); + EXPECT_EQ(engine_->checkXfer(handle), NIXL_SUCCESS); + + EXPECT_EQ(engine_->releaseReqH(handle), NIXL_SUCCESS); + EXPECT_EQ(engine_->deregisterMem(namedMetadata), NIXL_SUCCESS); + EXPECT_EQ(engine_->deregisterMem(fallbackMetadata), NIXL_SUCCESS); + + nixlBackendMD *unsupported = reinterpret_cast(1); + EXPECT_EQ(engine_->registerMem(nixlBlobDesc(), VRAM_SEG, unsupported), + NIXL_ERR_NOT_SUPPORTED); + EXPECT_EQ(unsupported, nullptr); +} + +TEST_F(redisEngineTest, QueryMemPreservesFoundMissingAndErrorResponses) { + mockClient_->setExistsResults({true, false, std::nullopt}); + nixl_reg_dlist_t descs(OBJ_SEG); + descs.addDesc(nixlBlobDesc(nixlBasicDesc(0, 0, 1), "found")); + descs.addDesc(nixlBlobDesc(nixlBasicDesc(0, 0, 2), "missing")); + descs.addDesc(nixlBlobDesc(nixlBasicDesc(0, 0, 3), "")); + + std::vector responses; + EXPECT_EQ(engine_->queryMem(descs, responses), NIXL_ERR_BACKEND); + ASSERT_EQ(responses.size(), 3); + EXPECT_TRUE(responses[0].has_value()); + EXPECT_FALSE(responses[1].has_value()); + EXPECT_FALSE(responses[2].has_value()); + EXPECT_EQ(mockClient_->checkedKeys(), (std::vector{"found", "missing", "3"})); +} + +TEST_F(redisEngineTest, PrepXferRejectsInvalidRequests) { + nixl_meta_dlist_t emptyLocal(DRAM_SEG); + nixl_meta_dlist_t emptyRemote(OBJ_SEG); + nixlBackendReqH *handle = reinterpret_cast(1); + EXPECT_EQ(engine_->prepXfer( + NIXL_WRITE, emptyLocal, emptyRemote, initParams_.localAgent, handle, nullptr), + NIXL_ERR_INVALID_PARAM); + EXPECT_EQ(handle, reinterpret_cast(1)); + + std::vector buffer(16); + nixl_meta_dlist_t local(DRAM_SEG); + local.addDesc( + nixlMetaDesc(reinterpret_cast(buffer.data()), buffer.size(), 1, nullptr)); + nixl_meta_dlist_t remote(OBJ_SEG); + remote.addDesc(nixlMetaDesc(0, buffer.size(), 2, nullptr)); + + handle = nullptr; + EXPECT_EQ(engine_->prepXfer(static_cast(99), + local, + remote, + initParams_.localAgent, + handle, + nullptr), + NIXL_ERR_INVALID_PARAM); + EXPECT_EQ(handle, nullptr); + + nixl_meta_dlist_t wrongLocal(OBJ_SEG); + wrongLocal.addDesc(nixlMetaDesc(0, buffer.size(), 1, nullptr)); + EXPECT_EQ(engine_->prepXfer( + NIXL_READ, wrongLocal, remote, initParams_.localAgent, handle, nullptr), + NIXL_ERR_INVALID_PARAM); +} + +TEST_F(redisEngineTest, PollsReadUntilClientCompletes) { + auto *metadata = registerRemote(22, "read-key"); + mockClient_->setCompleteImmediately(false); + std::vector buffer(16); + nixl_meta_dlist_t local(DRAM_SEG); + local.addDesc( + nixlMetaDesc(reinterpret_cast(buffer.data()), buffer.size(), 1, nullptr)); + nixl_meta_dlist_t remote(OBJ_SEG); + remote.addDesc(nixlMetaDesc(0, buffer.size(), 22, nullptr)); + + auto *handle = prepareTransfer(NIXL_READ, local, remote); + EXPECT_EQ(engine_->postXfer( + NIXL_READ, local, remote, initParams_.localAgent, handle, nullptr), + NIXL_IN_PROG); + EXPECT_EQ(mockClient_->getKeys(), (std::vector{"read-key"})); + EXPECT_EQ(engine_->checkXfer(handle), NIXL_IN_PROG); + mockClient_->completePending(NIXL_SUCCESS); + EXPECT_EQ(engine_->checkXfer(handle), NIXL_SUCCESS); + + EXPECT_EQ(engine_->releaseReqH(handle), NIXL_SUCCESS); + EXPECT_EQ(engine_->deregisterMem(metadata), NIXL_SUCCESS); +} + +TEST_F(redisEngineTest, PropagatesAsyncClientFailure) { + auto *metadata = registerRemote(22, "failed-key"); + mockClient_->setCompletionStatus(NIXL_ERR_BACKEND); + std::vector buffer(16); + nixl_meta_dlist_t local(DRAM_SEG); + local.addDesc( + nixlMetaDesc(reinterpret_cast(buffer.data()), buffer.size(), 1, nullptr)); + nixl_meta_dlist_t remote(OBJ_SEG); + remote.addDesc(nixlMetaDesc(0, buffer.size(), 22, nullptr)); + + auto *handle = prepareTransfer(NIXL_WRITE, local, remote); + EXPECT_EQ(engine_->postXfer( + NIXL_WRITE, local, remote, initParams_.localAgent, handle, nullptr), + NIXL_IN_PROG); + EXPECT_EQ(engine_->checkXfer(handle), NIXL_ERR_BACKEND); + EXPECT_EQ(engine_->releaseReqH(handle), NIXL_SUCCESS); + EXPECT_EQ(engine_->deregisterMem(metadata), NIXL_SUCCESS); +} + +TEST_F(redisEngineTest, RejectsNullTransferHandles) { + nixl_meta_dlist_t local(DRAM_SEG); + nixl_meta_dlist_t remote(OBJ_SEG); + nixlBackendReqH *handle = nullptr; + EXPECT_EQ(engine_->postXfer( + NIXL_WRITE, local, remote, initParams_.localAgent, handle, nullptr), + NIXL_ERR_INVALID_PARAM); + EXPECT_EQ(engine_->checkXfer(nullptr), NIXL_ERR_INVALID_PARAM); +} + TEST_F(redisEngineTest, PostXferDoesNotDispatchPartialCommandsWhenLaterKeyIsMissing) { std::vector firstBuffer(16, 'a'); std::vector secondBuffer(16, 'b'); - nixlBlobDesc registeredRemote(0, firstBuffer.size(), 11, "registered-key"); - nixlBackendMD *registeredMetadata = nullptr; - ASSERT_EQ(engine_->registerMem(registeredRemote, DRAM_SEG, registeredMetadata), NIXL_SUCCESS); - ASSERT_NE(registeredMetadata, nullptr); + auto *registeredMetadata = registerRemote(11, "registered-key", DRAM_SEG); nixl_meta_dlist_t localDescs(DRAM_SEG); localDescs.addDesc(nixlMetaDesc( @@ -112,16 +310,7 @@ TEST_F(redisEngineTest, PostXferDoesNotDispatchPartialCommandsWhenLaterKeyIsMiss remoteDescs.addDesc(nixlMetaDesc(0, firstBuffer.size(), 11, nullptr)); remoteDescs.addDesc(nixlMetaDesc(0, secondBuffer.size(), 22, nullptr)); - nixlBackendReqH *handle = nullptr; - ASSERT_EQ(engine_->prepXfer(NIXL_WRITE, - localDescs, - remoteDescs, - initParams_.localAgent, - initParams_.localAgent, - handle, - nullptr), - NIXL_SUCCESS); - + auto *handle = prepareTransfer(NIXL_WRITE, localDescs, remoteDescs); EXPECT_EQ(engine_->postXfer( NIXL_WRITE, localDescs, remoteDescs, initParams_.localAgent, handle, nullptr), NIXL_ERR_INVALID_PARAM); From 7c8d8861eb711bb0e2b42730033e24141c6c6e0e Mon Sep 17 00:00:00 2001 From: Harry Xie Date: Sat, 4 Jul 2026 05:05:35 +0300 Subject: [PATCH 19/24] feat(redis): add client configuration and validation --- src/plugins/redis/README.md | 166 ++++++++++++++++-- src/plugins/redis/redis_backend.cpp | 3 +- src/plugins/redis/redis_client.cpp | 132 ++++++++------ src/plugins/redis/redis_client.h | 19 +- test/gtest/unit/descriptors/sec_desc_list.cpp | 2 +- test/gtest/unit/redis/redis.cpp | 96 +++++++++- 6 files changed, 350 insertions(+), 68 deletions(-) diff --git a/src/plugins/redis/README.md b/src/plugins/redis/README.md index 958def4211..7e9a550b0f 100644 --- a/src/plugins/redis/README.md +++ b/src/plugins/redis/README.md @@ -5,12 +5,14 @@ SPDX-License-Identifier: Apache-2.0 # NIXL Redis Plugin -The `REDIS` plugin exposes Redis as a local NIXL storage backend. It supports asynchronous -`NIXL_WRITE`/`NIXL_READ` transfers and synchronous `queryMem()` checks. +The `REDIS` plugin exposes Redis as a local NIXL storage backend. It supports + +- asynchronous `NIXL_WRITE`/`NIXL_READ` transfers and +- synchronous `queryMem()` checks. ## Architecture -The plugin follows the same direct-backend layout as `src/plugins/infinia`: +The plugin follows the direct-backend layout similar to existing backends: ```text src/plugins/redis/ @@ -43,19 +45,54 @@ skipped with a warning. ## Configuration -Backend parameters take precedence over environment variables. +`RedisConfig` is the Redis client's resolved, internal configuration value. The backend calls +`RedisConfig::fromBackendParams()` once during construction and passes the result to +`hiredisAsyncClient`; the client does not repeatedly read backend parameters or environment +variables. + +```cpp +struct RedisConfig { + std::string host = "localhost"; + int port = 6379; + std::string username; + std::string password; + int db = 0; +}; +``` + +Each setting is resolved in this precedence order: + +1. A valid value in the NIXL backend parameter map. +2. The corresponding `REDIS_*` environment variable, when one exists. +3. The built-in default. + +An explicitly provided string value, including an empty username or password, takes precedence +over its environment fallback. Port values that fail validation fall through to `REDIS_PORT` and +then to `6379`; database parse failures fall back to `0`. The logical database currently has no +environment fallback. | Parameter | Environment fallback | Default | Description | |-----------|----------------------|---------|-------------| | `host` | `REDIS_HOST` | `localhost` | Redis hostname or IP address | | `port` | `REDIS_PORT` | `6379` | Redis TCP port | +| `username` | `REDIS_USERNAME` | empty | Redis ACL username | | `password` | `REDIS_PASSWORD` | empty | Redis AUTH password | | `db` | none | `0` | Redis logical database | +Authentication behavior is determined by the resolved credentials: + +- When `password` is empty, the client does not send `AUTH`. This is correct for a Redis server + that allows unauthenticated access. A password-protected server will reject subsequent commands. +- When only `password` is set, the client sends legacy/default-user `AUTH password`. +- When both `username` and `password` are set, the client sends ACL-style + `AUTH username password`. A username without a password is rejected during backend creation. + ```cpp nixl_b_params_t params = { {"host", "127.0.0.1"}, {"port", "6379"}, + {"username", "nixl"}, + {"password", "example-password"}, {"db", "2"}, }; agent.createBackend("REDIS", params); @@ -79,12 +116,34 @@ the first backend error once all completed work has been observed. ## Build and test +Build the plugin: + ```bash meson setup build -Denable_plugins=REDIS meson compile -C build REDIS -meson test -C build unit --print-errorlogs ``` +NIXL does not add tests to a `release` build, even when `build_tests` is enabled. To build and run +the Redis unit tests, install GoogleTest and GoogleMock and configure a separate non-release build: + +```bash +# Debian/Ubuntu +sudo apt-get install libgtest-dev libgmock-dev + +meson setup build-redis-tests \ + --buildtype=debug \ + -Dbuild_tests=true \ + -Denable_plugins=REDIS +meson compile -C build-redis-tests unit +meson devenv -C build-redis-tests \ + ./test/gtest/unit/unit \ + '--gtest_filter=redisConfigTest.*:redisEngineTest.*' +``` + +The filter runs only the Redis configuration and backend suites. A successful run currently +reports 11 tests from 2 test suites. These tests inject `mockRedisClient` and do not require a +running Redis server; use the live smoke test below to exercise hiredis and a real server. + The dynamic plugin is produced at: ```text @@ -98,12 +157,97 @@ meson setup build -Denable_plugins=REDIS -Dstatic_plugins=REDIS meson compile -C build ``` -For a live smoke test, start Redis and run one write and one read: +## Live nixlbench smoke test + +The unit tests use an injected Redis client and do not connect to a server. Use this smoke test to +exercise plugin creation, hiredis/libevent, and an actual Redis write/read data path. The example +uses a static REDIS plugin so the standalone nixlbench binary does not depend on dynamic plugin +discovery. + +Start an isolated Redis server on the loopback interface and verify that it is ready: + +```bash +docker run --detach --rm --name nixl-redis-smoke \ + --publish 127.0.0.1:6379:6379 redis:7-alpine +docker exec nixl-redis-smoke redis-cli PING +``` + +The readiness command must print `PONG`. + +Install a static REDIS-enabled NIXL build into a temporary prefix, then build nixlbench against +that installation: + +```bash +export NIXL_PREFIX=/tmp/nixl-redis-smoke-install + +meson setup build-redis \ + --prefix="$NIXL_PREFIX" \ + -Denable_plugins=REDIS \ + -Dstatic_plugins=REDIS +meson compile -C build-redis +meson install -C build-redis + +meson setup build-nixlbench benchmark/nixlbench \ + -Dnixl_path="$NIXL_PREFIX" +meson compile -C build-nixlbench +build-nixlbench/nixlbench --help | grep REDIS +``` + +The NIXL build requires the plugin dependencies listed above. The standalone nixlbench build also +requires the hiredis development package because it seeds Redis before a READ and independently +checks transferred data when consistency checking is enabled. + +Run fixed-size 4 KiB WRITE and READ benchmarks. These commands use one thread, one descriptor per +batch, one in-flight request, 32 warm-up iterations, and 208 measured iterations: + +```bash +export REDIS_HOST=127.0.0.1 +export REDIS_PORT=6379 +export LD_LIBRARY_PATH="$NIXL_PREFIX/lib/$(uname -m)-linux-gnu${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + +build-nixlbench/nixlbench \ + --backend=REDIS \ + --runtime_type=ASIO \ + --op_type=WRITE \ + --start_block_size=4096 \ + --max_block_size=4096 \ + --start_batch_size=1 \ + --max_batch_size=1 \ + --pipeline_depth=1 \ + --num_threads=1 \ + --total_buffer_size=67108864 \ + --warmup_iter=32 \ + --num_iter=208 \ + --check_consistency=true + +build-nixlbench/nixlbench \ + --backend=REDIS \ + --runtime_type=ASIO \ + --op_type=READ \ + --start_block_size=4096 \ + --max_block_size=4096 \ + --start_batch_size=1 \ + --max_batch_size=1 \ + --pipeline_depth=1 \ + --num_threads=1 \ + --total_buffer_size=67108864 \ + --warmup_iter=32 \ + --num_iter=208 \ + --check_consistency=true +``` + +Each command must exit successfully and print a result row for block size `4096`. The READ run +must also print `Seeded Redis key for READ`; with `--check_consistency=true`, nixlbench reports an +error and exits unsuccessfully if the transferred data differs. The reported bandwidth and latency +are loopback smoke-test measurements, not production performance results. + +The example uses Redis database 0 without authentication. `REDIS_PASSWORD` may be set for a +password-protected default user. nixlbench's direct seed/consistency helper does not currently +support `REDIS_USERNAME` or selecting a nonzero `db`, even though the plugin itself supports both. + +Inspect the created keys if desired, then stop the server: ```bash -export REDIS_HOST=127.0.0.1 REDIS_PORT=6379 -nixlbench --backend REDIS --start_block_size 4096 --max_block_size 4096 \ - --num_iter 10 --warmup_iter 1 --op_type WRITE -nixlbench --backend REDIS --start_block_size 4096 --max_block_size 4096 \ - --num_iter 10 --warmup_iter 1 --op_type READ +docker exec nixl-redis-smoke redis-cli DBSIZE +docker stop nixl-redis-smoke ``` diff --git a/src/plugins/redis/redis_backend.cpp b/src/plugins/redis/redis_backend.cpp index b63f565a3d..b429ed03df 100644 --- a/src/plugins/redis/redis_backend.cpp +++ b/src/plugins/redis/redis_backend.cpp @@ -114,7 +114,8 @@ class nixlRedisMetadata : public nixlBackendMD { nixlRedisKVEngine::nixlRedisKVEngine(const nixlBackendInitParams *init_params) : nixlBackendEngine(init_params), - redisClient_(std::make_shared(getInitCustomParams(init_params))) { + redisClient_(std::make_shared( + RedisConfig::fromBackendParams(getInitCustomParams(init_params)))) { NIXL_INFO << "Redis backend initialized"; } diff --git a/src/plugins/redis/redis_client.cpp b/src/plugins/redis/redis_client.cpp index 8319993d2f..1384d6a7e7 100644 --- a/src/plugins/redis/redis_client.cpp +++ b/src/plugins/redis/redis_client.cpp @@ -19,31 +19,28 @@ #include "common/nixl_log.h" #include #include +#include #include #include #include #include #include -#ifdef HAVE_HIREDIS_ASYNC - namespace { -using redis_event_task_t = std::function; - -void -runEventTask(evutil_socket_t, short, void *arg) { - std::unique_ptr task(static_cast(arg)); - (*task)(); -} - std::string -getRedisHost(nixl_b_params_t *custom_params) { - if (custom_params && custom_params->count("host") > 0) { - return custom_params->at("host"); +getStringSetting(const nixl_b_params_t *custom_params, + const char *param_name, + const char *env_name, + const char *default_value = "") { + if (custom_params) { + const auto it = custom_params->find(param_name); + if (it != custom_params->end()) { + return it->second; + } } - const char *env_host = getenv("REDIS_HOST"); - return env_host ? std::string(env_host) : "localhost"; + const char *env_value = std::getenv(env_name); + return env_value ? std::string(env_value) : default_value; } std::optional @@ -65,14 +62,14 @@ parseRedisPort(const std::string &value, const char *source) { } int -getRedisPort(nixl_b_params_t *custom_params) { +getRedisPort(const nixl_b_params_t *custom_params) { if (custom_params && custom_params->count("port") > 0) { auto port = parseRedisPort(custom_params->at("port"), "Redis port"); if (port) { return *port; } } - const char *env_port = getenv("REDIS_PORT"); + const char *env_port = std::getenv("REDIS_PORT"); if (env_port) { auto port = parseRedisPort(env_port, "REDIS_PORT"); if (port) { @@ -82,17 +79,8 @@ getRedisPort(nixl_b_params_t *custom_params) { return 6379; } -std::string -getRedisPassword(nixl_b_params_t *custom_params) { - if (custom_params && custom_params->count("password") > 0) { - return custom_params->at("password"); - } - const char *env_password = getenv("REDIS_PASSWORD"); - return env_password ? std::string(env_password) : ""; -} - int -getRedisDB(nixl_b_params_t *custom_params) { +getRedisDB(const nixl_b_params_t *custom_params) { if (custom_params && custom_params->count("db") > 0) { try { return std::stoi(custom_params->at("db")); @@ -104,6 +92,34 @@ getRedisDB(nixl_b_params_t *custom_params) { return 0; } +} // namespace + +RedisConfig +RedisConfig::fromBackendParams(const nixl_b_params_t *custom_params) { + RedisConfig config; + config.host = getStringSetting(custom_params, "host", "REDIS_HOST", "localhost"); + config.port = getRedisPort(custom_params); + config.username = getStringSetting(custom_params, "username", "REDIS_USERNAME"); + config.password = getStringSetting(custom_params, "password", "REDIS_PASSWORD"); + config.db = getRedisDB(custom_params); + if (!config.username.empty() && config.password.empty()) { + throw std::invalid_argument("Redis username requires a password"); + } + return config; +} + +#ifdef HAVE_HIREDIS_ASYNC + +namespace { + +using redis_event_task_t = std::function; + +void +runEventTask(evutil_socket_t, short, void *arg) { + std::unique_ptr task(static_cast(arg)); + (*task)(); +} + bool checkRedisReplyOk(redisReply *reply, const char *command) { if (!reply) { @@ -131,11 +147,8 @@ setPromiseStatus(const std::shared_ptr> &promise, bo } // namespace -hiredisAsyncClient::hiredisAsyncClient(nixl_b_params_t *custom_params) - : host_(getRedisHost(custom_params)), - port_(getRedisPort(custom_params)), - password_(getRedisPassword(custom_params)), - db_(getRedisDB(custom_params)), +hiredisAsyncClient::hiredisAsyncClient(RedisConfig config) + : config_(std::move(config)), asyncContext_(nullptr), syncContext_(nullptr), eventBase_(nullptr), @@ -149,7 +162,7 @@ hiredisAsyncClient::hiredisAsyncClient(nixl_b_params_t *custom_params) throw std::runtime_error("Failed to create event base"); } - asyncContext_ = redisAsyncConnect(host_.c_str(), port_); + asyncContext_ = redisAsyncConnect(config_.host.c_str(), config_.port); if (!asyncContext_ || asyncContext_->err) { if (asyncContext_) { std::string err_msg = @@ -189,13 +202,14 @@ hiredisAsyncClient::hiredisAsyncClient(nixl_b_params_t *custom_params) connectSyncContext(); - NIXL_INFO << absl::StrFormat("Connected to Redis at %s:%d (db=%d)", host_, port_, db_); + NIXL_INFO << absl::StrFormat( + "Connected to Redis at %s:%d (db=%d)", config_.host, config_.port, config_.db); } void hiredisAsyncClient::connectSyncContext() { struct timeval timeout = {5, 0}; - syncContext_ = redisConnectWithTimeout(host_.c_str(), port_, timeout); + syncContext_ = redisConnectWithTimeout(config_.host.c_str(), config_.port, timeout); if (!syncContext_ || syncContext_->err) { std::string err_msg = syncContext_ ? syncContext_->errstr : "allocation failed"; if (syncContext_) { @@ -204,15 +218,21 @@ hiredisAsyncClient::connectSyncContext() { } NIXL_WARN << absl::StrFormat( "Sync Redis connection for EXISTS failed (%s:%d): %s; queryMem will return errors", - host_, - port_, + config_.host, + config_.port, err_msg); return; } - if (!password_.empty()) { - redisReply *reply = - static_cast(redisCommand(syncContext_, "AUTH %s", password_.c_str())); + if (!config_.password.empty()) { + redisReply *reply = config_.username.empty() + ? static_cast(redisCommand( + syncContext_, "AUTH %s", config_.password.c_str())) + : static_cast( + redisCommand(syncContext_, + "AUTH %s %s", + config_.username.c_str(), + config_.password.c_str())); if (!checkRedisReplyOk(reply, "AUTH")) { freeReplyObject(reply); redisFree(syncContext_); @@ -223,9 +243,9 @@ hiredisAsyncClient::connectSyncContext() { freeReplyObject(reply); } - if (db_ != 0) { + if (config_.db != 0) { redisReply *reply = - static_cast(redisCommand(syncContext_, "SELECT %d", db_)); + static_cast(redisCommand(syncContext_, "SELECT %d", config_.db)); if (!checkRedisReplyOk(reply, "SELECT")) { freeReplyObject(reply); redisFree(syncContext_); @@ -237,7 +257,10 @@ hiredisAsyncClient::connectSyncContext() { } NIXL_INFO << absl::StrFormat( - "Sync Redis connection ready for EXISTS at %s:%d (db=%d)", host_, port_, db_); + "Sync Redis connection ready for EXISTS at %s:%d (db=%d)", + config_.host, + config_.port, + config_.db); } hiredisAsyncClient::~hiredisAsyncClient() { @@ -308,12 +331,12 @@ hiredisAsyncClient::completeAsyncInitialization(bool success) { void hiredisAsyncClient::startAsyncSelect() { - if (db_ == 0) { + if (config_.db == 0) { completeAsyncInitialization(true); return; } - int ret = redisAsyncCommand(asyncContext_, selectCallback, this, "SELECT %d", db_); + int ret = redisAsyncCommand(asyncContext_, selectCallback, this, "SELECT %d", config_.db); if (ret != REDIS_OK) { NIXL_ERROR << "Failed to queue Redis SELECT command"; completeAsyncInitialization(false); @@ -324,9 +347,19 @@ hiredisAsyncClient::startAsyncSelect() { void hiredisAsyncClient::startAsyncInitialization() { - if (!password_.empty()) { - int ret = - redisAsyncCommand(asyncContext_, authCallback, this, "AUTH %s", password_.c_str()); + if (!config_.password.empty()) { + const int ret = config_.username.empty() + ? redisAsyncCommand(asyncContext_, + authCallback, + this, + "AUTH %s", + config_.password.c_str()) + : redisAsyncCommand(asyncContext_, + authCallback, + this, + "AUTH %s %s", + config_.username.c_str(), + config_.password.c_str()); if (ret != REDIS_OK) { NIXL_ERROR << "Failed to queue Redis AUTH command"; completeAsyncInitialization(false); @@ -336,6 +369,7 @@ hiredisAsyncClient::startAsyncInitialization() { return; } + // An empty password means the Redis deployment permits unauthenticated access. startAsyncSelect(); } @@ -560,7 +594,7 @@ hiredisAsyncClient::checkKeyExistsSync(std::string_view key) { #else // HAVE_HIREDIS_ASYNC -hiredisAsyncClient::hiredisAsyncClient(nixl_b_params_t *custom_params) { +hiredisAsyncClient::hiredisAsyncClient(RedisConfig config) { throw std::runtime_error("hiredis-async not available"); } diff --git a/src/plugins/redis/redis_client.h b/src/plugins/redis/redis_client.h index a494a920e5..6a0f012e8b 100644 --- a/src/plugins/redis/redis_client.h +++ b/src/plugins/redis/redis_client.h @@ -31,6 +31,18 @@ struct redisAsyncContext; struct redisContext; #endif +/** Resolved Redis connection settings. Explicit backend parameters override environment values. */ +struct RedisConfig { + std::string host = "localhost"; + int port = 6379; + std::string username; + std::string password; + int db = 0; + + static RedisConfig + fromBackendParams(const nixl_b_params_t *custom_params); +}; + /** Redis operations used by the NIXL REDIS backend. */ class iRedisClient { public: @@ -55,7 +67,7 @@ class iRedisClient { /** Hiredis implementation with async SET/GET and a separate synchronous EXISTS connection. */ class hiredisAsyncClient : public iRedisClient { public: - explicit hiredisAsyncClient(nixl_b_params_t *custom_params); + explicit hiredisAsyncClient(RedisConfig config); ~hiredisAsyncClient() override; void @@ -110,10 +122,7 @@ class hiredisAsyncClient : public iRedisClient { void stopEventLoop(); - std::string host_; - int port_; - std::string password_; - int db_; + RedisConfig config_; redisAsyncContext *asyncContext_; redisContext *syncContext_; struct event_base *eventBase_; diff --git a/test/gtest/unit/descriptors/sec_desc_list.cpp b/test/gtest/unit/descriptors/sec_desc_list.cpp index 3ce1029f5e..c1acd8c5f2 100644 --- a/test/gtest/unit/descriptors/sec_desc_list.cpp +++ b/test/gtest/unit/descriptors/sec_desc_list.cpp @@ -207,7 +207,7 @@ TEST_F(secDescListTest, AddRandomBatches) { list.addDescs(std::move(batch)); num_added += batch_size; - ASSERT_EQ(list.descCount(), num_added); + ASSERT_EQ(static_cast(list.descCount()), num_added); assertSorted(list); } } diff --git a/test/gtest/unit/redis/redis.cpp b/test/gtest/unit/redis/redis.cpp index 494b2aed12..1db816b1bf 100644 --- a/test/gtest/unit/redis/redis.cpp +++ b/test/gtest/unit/redis/redis.cpp @@ -7,6 +7,7 @@ #include "redis_backend.h" +#include #include #include #include @@ -18,6 +19,99 @@ namespace gtest::redis { +class scopedRedisEnvironment { +public: + scopedRedisEnvironment() { + save("REDIS_HOST", host_); + save("REDIS_PORT", port_); + save("REDIS_USERNAME", username_); + save("REDIS_PASSWORD", password_); + } + + ~scopedRedisEnvironment() { + restore("REDIS_HOST", host_); + restore("REDIS_PORT", port_); + restore("REDIS_USERNAME", username_); + restore("REDIS_PASSWORD", password_); + } + + void + clear() { + unsetenv("REDIS_HOST"); + unsetenv("REDIS_PORT"); + unsetenv("REDIS_USERNAME"); + unsetenv("REDIS_PASSWORD"); + } + +private: + static void + save(const char *name, std::optional &value) { + if (const char *current = std::getenv(name)) { + value = current; + } + } + + static void + restore(const char *name, const std::optional &value) { + if (value) { + setenv(name, value->c_str(), 1); + } else { + unsetenv(name); + } + } + + std::optional host_; + std::optional port_; + std::optional username_; + std::optional password_; +}; + +TEST(redisConfigTest, UsesUnauthenticatedDefaultsWhenCredentialsAreAbsent) { + scopedRedisEnvironment environment; + environment.clear(); + nixl_b_params_t params; + + const auto config = RedisConfig::fromBackendParams(¶ms); + + EXPECT_EQ(config.host, "localhost"); + EXPECT_EQ(config.port, 6379); + EXPECT_TRUE(config.username.empty()); + EXPECT_TRUE(config.password.empty()); + EXPECT_EQ(config.db, 0); +} + +TEST(redisConfigTest, BackendParametersOverrideEnvironmentFallbacks) { + scopedRedisEnvironment environment; + environment.clear(); + setenv("REDIS_HOST", "environment-host", 1); + setenv("REDIS_PORT", "6380", 1); + setenv("REDIS_USERNAME", "environment-user", 1); + setenv("REDIS_PASSWORD", "environment-password", 1); + nixl_b_params_t params = { + {"host", "parameter-host"}, + {"port", "6381"}, + {"username", "parameter-user"}, + {"password", "parameter-password"}, + {"db", "2"}, + }; + + const auto config = RedisConfig::fromBackendParams(¶ms); + + EXPECT_EQ(config.host, "parameter-host"); + EXPECT_EQ(config.port, 6381); + EXPECT_EQ(config.username, "parameter-user"); + EXPECT_EQ(config.password, "parameter-password"); + EXPECT_EQ(config.db, 2); +} + +TEST(redisConfigTest, RejectsAclUsernameWithoutPassword) { + scopedRedisEnvironment environment; + environment.clear(); + nixl_b_params_t params = {{"username", "acl-user"}}; + + EXPECT_THROW(RedisConfig::fromBackendParams(¶ms), std::invalid_argument); +} + class mockRedisClient : public iRedisClient { public: void @@ -202,7 +296,7 @@ TEST_F(redisEngineTest, QueryMemPreservesFoundMissingAndErrorResponses) { std::vector responses; EXPECT_EQ(engine_->queryMem(descs, responses), NIXL_ERR_BACKEND); - ASSERT_EQ(responses.size(), 3); + ASSERT_EQ(responses.size(), 3U); EXPECT_TRUE(responses[0].has_value()); EXPECT_FALSE(responses[1].has_value()); EXPECT_FALSE(responses[2].has_value()); From c1b6fc6ad299b1940474c7fc8bee1c7b17e956d2 Mon Sep 17 00:00:00 2001 From: Harry Xie Date: Sat, 4 Jul 2026 05:49:58 +0300 Subject: [PATCH 20/24] fix: clang-format and copyright year for CI checks - Apply clang-format-18 to redis_backend.cpp, redis_client.cpp, and redis unit tests to fix ternary operator placement and line wrapping - Bump nixlbench/contrib/build.sh copyright year to 2025-2026 Co-Authored-By: Claude Sonnet 4.6 --- benchmark/nixlbench/contrib/build.sh | 2 +- src/plugins/redis/redis_backend.cpp | 3 +- src/plugins/redis/redis_client.cpp | 145 ++++++++++++--------------- test/gtest/unit/redis/redis.cpp | 28 ++---- 4 files changed, 79 insertions(+), 99 deletions(-) diff --git a/benchmark/nixlbench/contrib/build.sh b/benchmark/nixlbench/contrib/build.sh index cec9d4b682..bedc2ece94 100755 --- a/benchmark/nixlbench/contrib/build.sh +++ b/benchmark/nixlbench/contrib/build.sh @@ -1,6 +1,6 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/src/plugins/redis/redis_backend.cpp b/src/plugins/redis/redis_backend.cpp index b429ed03df..77891d932d 100644 --- a/src/plugins/redis/redis_backend.cpp +++ b/src/plugins/redis/redis_backend.cpp @@ -129,8 +129,7 @@ nixlRedisKVEngine::registerMem(const nixlBlobDesc &mem, const nixl_mem_t &nixl_mem, nixlBackendMD *&out) { const auto supported_mems = getSupportedMems(); - if (std::find(supported_mems.begin(), supported_mems.end(), nixl_mem) == - supported_mems.end()) { + if (std::find(supported_mems.begin(), supported_mems.end(), nixl_mem) == supported_mems.end()) { out = nullptr; return NIXL_ERR_NOT_SUPPORTED; } diff --git a/src/plugins/redis/redis_client.cpp b/src/plugins/redis/redis_client.cpp index 1384d6a7e7..ad87ecf4cd 100644 --- a/src/plugins/redis/redis_client.cpp +++ b/src/plugins/redis/redis_client.cpp @@ -225,14 +225,11 @@ hiredisAsyncClient::connectSyncContext() { } if (!config_.password.empty()) { - redisReply *reply = config_.username.empty() - ? static_cast(redisCommand( - syncContext_, "AUTH %s", config_.password.c_str())) - : static_cast( - redisCommand(syncContext_, - "AUTH %s %s", - config_.username.c_str(), - config_.password.c_str())); + redisReply *reply = config_.username.empty() ? + static_cast( + redisCommand(syncContext_, "AUTH %s", config_.password.c_str())) : + static_cast(redisCommand( + syncContext_, "AUTH %s %s", config_.username.c_str(), config_.password.c_str())); if (!checkRedisReplyOk(reply, "AUTH")) { freeReplyObject(reply); redisFree(syncContext_); @@ -256,11 +253,10 @@ hiredisAsyncClient::connectSyncContext() { freeReplyObject(reply); } - NIXL_INFO << absl::StrFormat( - "Sync Redis connection ready for EXISTS at %s:%d (db=%d)", - config_.host, - config_.port, - config_.db); + NIXL_INFO << absl::StrFormat("Sync Redis connection ready for EXISTS at %s:%d (db=%d)", + config_.host, + config_.port, + config_.db); } hiredisAsyncClient::~hiredisAsyncClient() { @@ -348,18 +344,15 @@ hiredisAsyncClient::startAsyncSelect() { void hiredisAsyncClient::startAsyncInitialization() { if (!config_.password.empty()) { - const int ret = config_.username.empty() - ? redisAsyncCommand(asyncContext_, - authCallback, - this, - "AUTH %s", - config_.password.c_str()) - : redisAsyncCommand(asyncContext_, - authCallback, - this, - "AUTH %s %s", - config_.username.c_str(), - config_.password.c_str()); + const int ret = config_.username.empty() ? + redisAsyncCommand( + asyncContext_, authCallback, this, "AUTH %s", config_.password.c_str()) : + redisAsyncCommand(asyncContext_, + authCallback, + this, + "AUTH %s %s", + config_.username.c_str(), + config_.password.c_str()); if (ret != REDIS_OK) { NIXL_ERROR << "Failed to queue Redis AUTH command"; completeAsyncInitialization(false); @@ -483,34 +476,31 @@ hiredisAsyncClient::putKeyAsync(std::string_view key, } std::string key_copy(key); - bool scheduled = scheduleOnEventLoop([this, - key = std::move(key_copy), - data_ptr, - data_len, - promise]() mutable { - if (!connected_.load() || !asyncContext_) { - setPromiseStatus(promise, false); - return; - } - - auto *ctx = new CallbackContext; - ctx->promise_ptr = promise; - - int ret = redisAsyncCommand(asyncContext_, - setCallback, - ctx, - "SET %b %b", - key.data(), - key.size(), - reinterpret_cast(data_ptr), - data_len); - - if (ret != REDIS_OK) { - auto promise_ptr = ctx->promise_ptr; - delete ctx; - setPromiseStatus(promise_ptr, false); - } - }); + bool scheduled = scheduleOnEventLoop( + [this, key = std::move(key_copy), data_ptr, data_len, promise]() mutable { + if (!connected_.load() || !asyncContext_) { + setPromiseStatus(promise, false); + return; + } + + auto *ctx = new CallbackContext; + ctx->promise_ptr = promise; + + int ret = redisAsyncCommand(asyncContext_, + setCallback, + ctx, + "SET %b %b", + key.data(), + key.size(), + reinterpret_cast(data_ptr), + data_len); + + if (ret != REDIS_OK) { + auto promise_ptr = ctx->promise_ptr; + delete ctx; + setPromiseStatus(promise_ptr, false); + } + }); if (!scheduled) { setPromiseStatus(promise, false); @@ -528,30 +518,27 @@ hiredisAsyncClient::getKeyAsync(std::string_view key, } std::string key_copy(key); - bool scheduled = scheduleOnEventLoop([this, - key = std::move(key_copy), - data_ptr, - data_len, - promise]() mutable { - if (!connected_.load() || !asyncContext_) { - setPromiseStatus(promise, false); - return; - } - - auto *ctx = new CallbackContext; - ctx->data_ptr = data_ptr; - ctx->data_len = data_len; - ctx->promise_ptr = promise; - - int ret = - redisAsyncCommand(asyncContext_, getCallback, ctx, "GET %b", key.data(), key.size()); - - if (ret != REDIS_OK) { - auto promise_ptr = ctx->promise_ptr; - delete ctx; - setPromiseStatus(promise_ptr, false); - } - }); + bool scheduled = scheduleOnEventLoop( + [this, key = std::move(key_copy), data_ptr, data_len, promise]() mutable { + if (!connected_.load() || !asyncContext_) { + setPromiseStatus(promise, false); + return; + } + + auto *ctx = new CallbackContext; + ctx->data_ptr = data_ptr; + ctx->data_len = data_len; + ctx->promise_ptr = promise; + + int ret = redisAsyncCommand( + asyncContext_, getCallback, ctx, "GET %b", key.data(), key.size()); + + if (ret != REDIS_OK) { + auto promise_ptr = ctx->promise_ptr; + delete ctx; + setPromiseStatus(promise_ptr, false); + } + }); if (!scheduled) { setPromiseStatus(promise, false); @@ -567,8 +554,8 @@ hiredisAsyncClient::checkKeyExistsSync(std::string_view key) { return std::nullopt; } - redisReply *reply = static_cast( - redisCommand(syncContext_, "EXISTS %b", key.data(), key.size())); + redisReply *reply = + static_cast(redisCommand(syncContext_, "EXISTS %b", key.data(), key.size())); if (!reply) { NIXL_ERROR << "Redis EXISTS: no reply"; diff --git a/test/gtest/unit/redis/redis.cpp b/test/gtest/unit/redis/redis.cpp index 1db816b1bf..c242b4953a 100644 --- a/test/gtest/unit/redis/redis.cpp +++ b/test/gtest/unit/redis/redis.cpp @@ -225,13 +225,11 @@ class redisEngineTest : public ::testing::Test { } nixlBackendReqH * - prepareTransfer(nixl_xfer_op_t operation, - nixl_meta_dlist_t &local, - nixl_meta_dlist_t &remote) { + prepareTransfer(nixl_xfer_op_t operation, nixl_meta_dlist_t &local, nixl_meta_dlist_t &remote) { nixlBackendReqH *handle = nullptr; - EXPECT_EQ(engine_->prepXfer( - operation, local, remote, initParams_.localAgent, handle, nullptr), - NIXL_SUCCESS); + EXPECT_EQ( + engine_->prepXfer(operation, local, remote, initParams_.localAgent, handle, nullptr), + NIXL_SUCCESS); EXPECT_NE(handle, nullptr); return handle; } @@ -282,8 +280,7 @@ TEST_F(redisEngineTest, RegistersMetadataKeyAndDevIdFallback) { EXPECT_EQ(engine_->deregisterMem(fallbackMetadata), NIXL_SUCCESS); nixlBackendMD *unsupported = reinterpret_cast(1); - EXPECT_EQ(engine_->registerMem(nixlBlobDesc(), VRAM_SEG, unsupported), - NIXL_ERR_NOT_SUPPORTED); + EXPECT_EQ(engine_->registerMem(nixlBlobDesc(), VRAM_SEG, unsupported), NIXL_ERR_NOT_SUPPORTED); EXPECT_EQ(unsupported, nullptr); } @@ -331,9 +328,9 @@ TEST_F(redisEngineTest, PrepXferRejectsInvalidRequests) { nixl_meta_dlist_t wrongLocal(OBJ_SEG); wrongLocal.addDesc(nixlMetaDesc(0, buffer.size(), 1, nullptr)); - EXPECT_EQ(engine_->prepXfer( - NIXL_READ, wrongLocal, remote, initParams_.localAgent, handle, nullptr), - NIXL_ERR_INVALID_PARAM); + EXPECT_EQ( + engine_->prepXfer(NIXL_READ, wrongLocal, remote, initParams_.localAgent, handle, nullptr), + NIXL_ERR_INVALID_PARAM); } TEST_F(redisEngineTest, PollsReadUntilClientCompletes) { @@ -347,8 +344,7 @@ TEST_F(redisEngineTest, PollsReadUntilClientCompletes) { remote.addDesc(nixlMetaDesc(0, buffer.size(), 22, nullptr)); auto *handle = prepareTransfer(NIXL_READ, local, remote); - EXPECT_EQ(engine_->postXfer( - NIXL_READ, local, remote, initParams_.localAgent, handle, nullptr), + EXPECT_EQ(engine_->postXfer(NIXL_READ, local, remote, initParams_.localAgent, handle, nullptr), NIXL_IN_PROG); EXPECT_EQ(mockClient_->getKeys(), (std::vector{"read-key"})); EXPECT_EQ(engine_->checkXfer(handle), NIXL_IN_PROG); @@ -370,8 +366,7 @@ TEST_F(redisEngineTest, PropagatesAsyncClientFailure) { remote.addDesc(nixlMetaDesc(0, buffer.size(), 22, nullptr)); auto *handle = prepareTransfer(NIXL_WRITE, local, remote); - EXPECT_EQ(engine_->postXfer( - NIXL_WRITE, local, remote, initParams_.localAgent, handle, nullptr), + EXPECT_EQ(engine_->postXfer(NIXL_WRITE, local, remote, initParams_.localAgent, handle, nullptr), NIXL_IN_PROG); EXPECT_EQ(engine_->checkXfer(handle), NIXL_ERR_BACKEND); EXPECT_EQ(engine_->releaseReqH(handle), NIXL_SUCCESS); @@ -382,8 +377,7 @@ TEST_F(redisEngineTest, RejectsNullTransferHandles) { nixl_meta_dlist_t local(DRAM_SEG); nixl_meta_dlist_t remote(OBJ_SEG); nixlBackendReqH *handle = nullptr; - EXPECT_EQ(engine_->postXfer( - NIXL_WRITE, local, remote, initParams_.localAgent, handle, nullptr), + EXPECT_EQ(engine_->postXfer(NIXL_WRITE, local, remote, initParams_.localAgent, handle, nullptr), NIXL_ERR_INVALID_PARAM); EXPECT_EQ(engine_->checkXfer(nullptr), NIXL_ERR_INVALID_PARAM); } From 0657e6a176434a10b471e499a64f81eb29f8c515 Mon Sep 17 00:00:00 2001 From: Harry Xie Date: Sat, 4 Jul 2026 08:56:16 +0300 Subject: [PATCH 21/24] Fix clang-format violations in utils.cpp and redis_plugin.cpp --- benchmark/nixlbench/src/utils/utils.cpp | 15 +++++---------- src/plugins/redis/redis_plugin.cpp | 14 ++++---------- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/benchmark/nixlbench/src/utils/utils.cpp b/benchmark/nixlbench/src/utils/utils.cpp index 2301286172..da01ee0aff 100644 --- a/benchmark/nixlbench/src/utils/utils.cpp +++ b/benchmark/nixlbench/src/utils/utils.cpp @@ -129,8 +129,7 @@ getRedisBench(size_t buffer_size, const std::string &key, void *buffer) { return false; } - redisReply *reply = - (redisReply *)redisCommand(ctx, "GET %b", key.data(), key.size()); + redisReply *reply = (redisReply *)redisCommand(ctx, "GET %b", key.data(), key.size()); bool success = false; if (reply && reply->type == REDIS_REPLY_STRING) { const size_t reply_len = static_cast(reply->len); @@ -140,8 +139,8 @@ getRedisBench(size_t buffer_size, const std::string &key, void *buffer) { } success = true; } else { - std::cerr << "Redis GET size mismatch for key " << key << ": expected " - << buffer_size << " bytes, got " << reply_len << " bytes" << std::endl; + std::cerr << "Redis GET size mismatch for key " << key << ": expected " << buffer_size + << " bytes, got " << reply_len << " bytes" << std::endl; } } else if (reply && reply->type == REDIS_REPLY_NIL) { std::cerr << "Redis GET failed, key not found: " << key << std::endl; @@ -1457,12 +1456,8 @@ xferBenchUtils::putRedis(size_t buffer_size, const std::string &key) { return false; } - redisReply *reply = (redisReply *)redisCommand(ctx, - "SET %b %b", - key.data(), - (size_t)key.size(), - buf, - (size_t)buffer_size); + redisReply *reply = (redisReply *)redisCommand( + ctx, "SET %b %b", key.data(), (size_t)key.size(), buf, (size_t)buffer_size); free(buf); bool ok = false; diff --git a/src/plugins/redis/redis_plugin.cpp b/src/plugins/redis/redis_plugin.cpp index 31edc719a3..d4090abfdf 100644 --- a/src/plugins/redis/redis_plugin.cpp +++ b/src/plugins/redis/redis_plugin.cpp @@ -25,20 +25,14 @@ static const nixl_mem_list_t supported_segments = {DRAM_SEG, OBJ_SEG}; #ifdef STATIC_PLUGIN_REDIS nixlBackendPlugin * createStaticREDISPlugin() { - return redis_plugin_t::create(NIXL_PLUGIN_API_VERSION, - REDIS_PLUGIN_NAME, - REDIS_PLUGIN_VERSION, - {}, - supported_segments); + return redis_plugin_t::create( + NIXL_PLUGIN_API_VERSION, REDIS_PLUGIN_NAME, REDIS_PLUGIN_VERSION, {}, supported_segments); } #else extern "C" NIXL_PLUGIN_EXPORT nixlBackendPlugin * nixl_plugin_init() { - return redis_plugin_t::create(NIXL_PLUGIN_API_VERSION, - REDIS_PLUGIN_NAME, - REDIS_PLUGIN_VERSION, - {}, - supported_segments); + return redis_plugin_t::create( + NIXL_PLUGIN_API_VERSION, REDIS_PLUGIN_NAME, REDIS_PLUGIN_VERSION, {}, supported_segments); } extern "C" NIXL_PLUGIN_EXPORT void From c5d5ca13c0f1ba501c1d04a25bf9d904f344289d Mon Sep 17 00:00:00 2001 From: Harry Xie Date: Tue, 14 Jul 2026 23:03:14 +0300 Subject: [PATCH 22/24] feat(redis): add RedisConnectionPool with configurable pool size Adds a RedisConnectionPool class that wraps N hiredisAsyncClient instances with round-robin dispatch. Pool size defaults to 8 and is configurable via backend param pool_size or env var REDIS_POOL_SIZE. Wires the pool into the nixlRedisKVEngine production constructor via the existing iRedisClient shared_ptr member. Adds 5 unit tests covering pool_size config parsing. --- src/plugins/redis/redis_backend.cpp | 6 +-- src/plugins/redis/redis_client.cpp | 74 +++++++++++++++++++++++++++++ src/plugins/redis/redis_client.h | 36 ++++++++++++++ test/gtest/unit/redis/redis.cpp | 46 ++++++++++++++++++ 4 files changed, 159 insertions(+), 3 deletions(-) diff --git a/src/plugins/redis/redis_backend.cpp b/src/plugins/redis/redis_backend.cpp index 77891d932d..9955a0b218 100644 --- a/src/plugins/redis/redis_backend.cpp +++ b/src/plugins/redis/redis_backend.cpp @@ -113,9 +113,9 @@ class nixlRedisMetadata : public nixlBackendMD { } // namespace nixlRedisKVEngine::nixlRedisKVEngine(const nixlBackendInitParams *init_params) - : nixlBackendEngine(init_params), - redisClient_(std::make_shared( - RedisConfig::fromBackendParams(getInitCustomParams(init_params)))) { + : nixlBackendEngine(init_params) { + auto config = RedisConfig::fromBackendParams(getInitCustomParams(init_params)); + redisClient_ = std::make_shared(std::move(config)); NIXL_INFO << "Redis backend initialized"; } diff --git a/src/plugins/redis/redis_client.cpp b/src/plugins/redis/redis_client.cpp index ad87ecf4cd..c25140273e 100644 --- a/src/plugins/redis/redis_client.cpp +++ b/src/plugins/redis/redis_client.cpp @@ -92,6 +92,41 @@ getRedisDB(const nixl_b_params_t *custom_params) { return 0; } +std::optional +parseRedisPoolSize(const std::string &value, const char *source) { + try { + size_t parsed = 0; + int val = std::stoi(value, &parsed); + if (parsed != value.size() || val <= 0) { + NIXL_WARN << absl::StrFormat("Invalid %s value '%s', using default 8", source, value); + return std::nullopt; + } + return val; + } + catch (const std::exception &) { + NIXL_WARN << absl::StrFormat("Invalid %s value '%s', using default 8", source, value); + return std::nullopt; + } +} + +int +getRedisPoolSize(const nixl_b_params_t *custom_params) { + if (custom_params && custom_params->count("pool_size") > 0) { + auto val = parseRedisPoolSize(custom_params->at("pool_size"), "pool_size"); + if (val) { + return *val; + } + } + const char *env_val = std::getenv("REDIS_POOL_SIZE"); + if (env_val) { + auto val = parseRedisPoolSize(env_val, "REDIS_POOL_SIZE"); + if (val) { + return *val; + } + } + return 8; +} + } // namespace RedisConfig @@ -102,6 +137,7 @@ RedisConfig::fromBackendParams(const nixl_b_params_t *custom_params) { config.username = getStringSetting(custom_params, "username", "REDIS_USERNAME"); config.password = getStringSetting(custom_params, "password", "REDIS_PASSWORD"); config.db = getRedisDB(custom_params); + config.pool_size = getRedisPoolSize(custom_params); if (!config.username.empty() && config.password.empty()) { throw std::invalid_argument("Redis username requires a password"); } @@ -613,3 +649,41 @@ hiredisAsyncClient::checkKeyExistsSync(std::string_view key) { } #endif // HAVE_HIREDIS_ASYNC + +RedisConnectionPool::RedisConnectionPool(RedisConfig config) { + clients_.reserve(static_cast(config.pool_size)); + for (int i = 0; i < config.pool_size; ++i) { + clients_.push_back(std::make_unique(config)); + } +} + +iRedisClient & +RedisConnectionPool::nextSlot() { + return *clients_[nextSlot_.fetch_add(1, std::memory_order_relaxed) % clients_.size()]; +} + +void +RedisConnectionPool::putKeyAsync(std::string_view key, + uintptr_t data_ptr, + size_t data_len, + std::shared_ptr> promise) { + nextSlot().putKeyAsync(key, data_ptr, data_len, std::move(promise)); +} + +void +RedisConnectionPool::getKeyAsync(std::string_view key, + uintptr_t data_ptr, + size_t data_len, + std::shared_ptr> promise) { + nextSlot().getKeyAsync(key, data_ptr, data_len, std::move(promise)); +} + +std::optional +RedisConnectionPool::checkKeyExistsSync(std::string_view key) { + return nextSlot().checkKeyExistsSync(key); +} + +size_t +RedisConnectionPool::size() const { + return clients_.size(); +} diff --git a/src/plugins/redis/redis_client.h b/src/plugins/redis/redis_client.h index 6a0f012e8b..86b4846719 100644 --- a/src/plugins/redis/redis_client.h +++ b/src/plugins/redis/redis_client.h @@ -38,6 +38,7 @@ struct RedisConfig { std::string username; std::string password; int db = 0; + int pool_size = 8; static RedisConfig fromBackendParams(const nixl_b_params_t *custom_params); @@ -133,4 +134,39 @@ class hiredisAsyncClient : public iRedisClient { mutable std::mutex syncMutex_; }; +/** + * Round-robin connection pool over multiple hiredisAsyncClient instances. + * Each slot has its own event loop thread and TCP connections. + */ +class RedisConnectionPool : public iRedisClient { +public: + explicit RedisConnectionPool(RedisConfig config); + ~RedisConnectionPool() override = default; + + void + putKeyAsync(std::string_view key, + uintptr_t data_ptr, + size_t data_len, + std::shared_ptr> promise) override; + + void + getKeyAsync(std::string_view key, + uintptr_t data_ptr, + size_t data_len, + std::shared_ptr> promise) override; + + std::optional + checkKeyExistsSync(std::string_view key) override; + + size_t + size() const; + +private: + iRedisClient & + nextSlot(); + + std::vector> clients_; + std::atomic nextSlot_{0}; +}; + #endif // NIXL_SRC_PLUGINS_REDIS_REDIS_CLIENT_H diff --git a/test/gtest/unit/redis/redis.cpp b/test/gtest/unit/redis/redis.cpp index c242b4953a..81e73f3578 100644 --- a/test/gtest/unit/redis/redis.cpp +++ b/test/gtest/unit/redis/redis.cpp @@ -26,6 +26,7 @@ class scopedRedisEnvironment { save("REDIS_PORT", port_); save("REDIS_USERNAME", username_); save("REDIS_PASSWORD", password_); + save("REDIS_POOL_SIZE", pool_size_); } ~scopedRedisEnvironment() { @@ -33,6 +34,7 @@ class scopedRedisEnvironment { restore("REDIS_PORT", port_); restore("REDIS_USERNAME", username_); restore("REDIS_PASSWORD", password_); + restore("REDIS_POOL_SIZE", pool_size_); } void @@ -41,6 +43,7 @@ class scopedRedisEnvironment { unsetenv("REDIS_PORT"); unsetenv("REDIS_USERNAME"); unsetenv("REDIS_PASSWORD"); + unsetenv("REDIS_POOL_SIZE"); } private: @@ -64,6 +67,7 @@ class scopedRedisEnvironment { std::optional port_; std::optional username_; std::optional password_; + std::optional pool_size_; }; TEST(redisConfigTest, UsesUnauthenticatedDefaultsWhenCredentialsAreAbsent) { @@ -112,6 +116,48 @@ TEST(redisConfigTest, RejectsAclUsernameWithoutPassword) { EXPECT_THROW(RedisConfig::fromBackendParams(¶ms), std::invalid_argument); } +TEST(redisConfigTest, UsesDefaultPoolSizeWhenAbsent) { + scopedRedisEnvironment environment; + environment.clear(); + nixl_b_params_t params; + const auto config = RedisConfig::fromBackendParams(¶ms); + EXPECT_EQ(config.pool_size, 8); +} + +TEST(redisConfigTest, BackendParamOverridesPoolSize) { + scopedRedisEnvironment environment; + environment.clear(); + nixl_b_params_t params = {{"pool_size", "4"}}; + const auto config = RedisConfig::fromBackendParams(¶ms); + EXPECT_EQ(config.pool_size, 4); +} + +TEST(redisConfigTest, EnvVarSetsPoolSize) { + scopedRedisEnvironment environment; + environment.clear(); + setenv("REDIS_POOL_SIZE", "16", 1); + nixl_b_params_t params; + const auto config = RedisConfig::fromBackendParams(¶ms); + EXPECT_EQ(config.pool_size, 16); +} + +TEST(redisConfigTest, BackendParamOverridesPoolSizeEnvVar) { + scopedRedisEnvironment environment; + environment.clear(); + setenv("REDIS_POOL_SIZE", "16", 1); + nixl_b_params_t params = {{"pool_size", "2"}}; + const auto config = RedisConfig::fromBackendParams(¶ms); + EXPECT_EQ(config.pool_size, 2); +} + +TEST(redisConfigTest, InvalidPoolSizeFallsBackToDefault) { + scopedRedisEnvironment environment; + environment.clear(); + nixl_b_params_t params = {{"pool_size", "bad"}}; + const auto config = RedisConfig::fromBackendParams(¶ms); + EXPECT_EQ(config.pool_size, 8); +} + class mockRedisClient : public iRedisClient { public: void From 936d97b0942050af76614ddb262bf9ad85cd29e5 Mon Sep 17 00:00:00 2001 From: Harry Xie Date: Tue, 14 Jul 2026 23:07:51 +0300 Subject: [PATCH 23/24] docs(redis): document pool_size configuration in README --- src/plugins/redis/README.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/plugins/redis/README.md b/src/plugins/redis/README.md index 7e9a550b0f..6117e01eb2 100644 --- a/src/plugins/redis/README.md +++ b/src/plugins/redis/README.md @@ -27,12 +27,15 @@ src/plugins/redis/ interface isolates hiredis/libevent and permits Redis-free unit tests; it is not a generic KV extension API. -The production client uses: +The production client uses a `RedisConnectionPool` (default: 8 slots). Each slot holds: - One hiredis/libevent async connection for `SET` and `GET`. - One blocking hiredis connection for `EXISTS`. - One libevent thread. Hiredis callbacks complete NIXL request promises directly. +Operations are dispatched round-robin across pool slots to saturate multiple Redis server +threads and reduce per-slot head-of-line blocking. + ## Dependencies - `hiredis` with async API support @@ -47,7 +50,7 @@ skipped with a warning. `RedisConfig` is the Redis client's resolved, internal configuration value. The backend calls `RedisConfig::fromBackendParams()` once during construction and passes the result to -`hiredisAsyncClient`; the client does not repeatedly read backend parameters or environment +`RedisConnectionPool`; the client does not repeatedly read backend parameters or environment variables. ```cpp @@ -57,6 +60,7 @@ struct RedisConfig { std::string username; std::string password; int db = 0; + int pool_size = 8; }; ``` @@ -69,7 +73,7 @@ Each setting is resolved in this precedence order: An explicitly provided string value, including an empty username or password, takes precedence over its environment fallback. Port values that fail validation fall through to `REDIS_PORT` and then to `6379`; database parse failures fall back to `0`. The logical database currently has no -environment fallback. +environment fallback. Invalid `pool_size` values fall back to `8`. | Parameter | Environment fallback | Default | Description | |-----------|----------------------|---------|-------------| @@ -78,6 +82,7 @@ environment fallback. | `username` | `REDIS_USERNAME` | empty | Redis ACL username | | `password` | `REDIS_PASSWORD` | empty | Redis AUTH password | | `db` | none | `0` | Redis logical database | +| `pool_size` | `REDIS_POOL_SIZE` | `8` | Number of concurrent Redis connections | Authentication behavior is determined by the resolved credentials: @@ -94,6 +99,7 @@ nixl_b_params_t params = { {"username", "nixl"}, {"password", "example-password"}, {"db", "2"}, + {"pool_size", "8"}, }; agent.createBackend("REDIS", params); ``` From f59892bcfb19439ed69ec01a8f71006aa5907ec6 Mon Sep 17 00:00:00 2001 From: Harry Xie Date: Tue, 14 Jul 2026 23:16:50 +0300 Subject: [PATCH 24/24] fix(redis): remove unused size() method and add zero/negative pool_size tests --- src/plugins/redis/redis_client.cpp | 5 ----- src/plugins/redis/redis_client.h | 3 --- test/gtest/unit/redis/redis.cpp | 16 ++++++++++++++++ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/plugins/redis/redis_client.cpp b/src/plugins/redis/redis_client.cpp index c25140273e..fb88e0d31a 100644 --- a/src/plugins/redis/redis_client.cpp +++ b/src/plugins/redis/redis_client.cpp @@ -682,8 +682,3 @@ std::optional RedisConnectionPool::checkKeyExistsSync(std::string_view key) { return nextSlot().checkKeyExistsSync(key); } - -size_t -RedisConnectionPool::size() const { - return clients_.size(); -} diff --git a/src/plugins/redis/redis_client.h b/src/plugins/redis/redis_client.h index 86b4846719..c78f31c5cb 100644 --- a/src/plugins/redis/redis_client.h +++ b/src/plugins/redis/redis_client.h @@ -158,9 +158,6 @@ class RedisConnectionPool : public iRedisClient { std::optional checkKeyExistsSync(std::string_view key) override; - size_t - size() const; - private: iRedisClient & nextSlot(); diff --git a/test/gtest/unit/redis/redis.cpp b/test/gtest/unit/redis/redis.cpp index 81e73f3578..29bd6b1de2 100644 --- a/test/gtest/unit/redis/redis.cpp +++ b/test/gtest/unit/redis/redis.cpp @@ -158,6 +158,22 @@ TEST(redisConfigTest, InvalidPoolSizeFallsBackToDefault) { EXPECT_EQ(config.pool_size, 8); } +TEST(redisConfigTest, ZeroPoolSizeFallsBackToDefault) { + scopedRedisEnvironment environment; + environment.clear(); + nixl_b_params_t params = {{"pool_size", "0"}}; + const auto config = RedisConfig::fromBackendParams(¶ms); + EXPECT_EQ(config.pool_size, 8); +} + +TEST(redisConfigTest, NegativePoolSizeFallsBackToDefault) { + scopedRedisEnvironment environment; + environment.clear(); + nixl_b_params_t params = {{"pool_size", "-1"}}; + const auto config = RedisConfig::fromBackendParams(¶ms); + EXPECT_EQ(config.pool_size, 8); +} + class mockRedisClient : public iRedisClient { public: void