diff --git a/CMakeLists.txt b/CMakeLists.txt index 6878a11c..3df15d24 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -473,3 +473,39 @@ ExternalProject_Add(aws-sdk-cpp -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/aws-sdk-cpp PATCH_COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/tools/install_src.py --src ${INSTALL_SRC_DEST_ARG} ) + +# +# Build librdkafka +# +ExternalProject_Add(librdkafka + PREFIX librdkafka + GIT_REPOSITORY "https://github.com/edenhill/librdkafka.git" + GIT_TAG "v1.9.2" + SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/librdkafka" + BUILD_IN_SOURCE ON + CONFIGURE_COMMAND "./configure" + EXCLUDE_FROM_ALL ON + CMAKE_CACHE_ARGS + ${_CMAKE_ARGS_CMAKE_TOOLCHAIN_FILE} + ${_CMAKE_ARGS_VCPKG_TARGET_TRIPLET} + -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON + -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/librdkafka + PATCH_COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/tools/install_src.py --src ${INSTALL_SRC_DEST_ARG} +) + +# +# Build modern-cpp-kafka +# +ExternalProject_Add(modern-cpp-kafka + PREFIX modern-cpp-kafka + SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/modern-cpp-kafka" + EXCLUDE_FROM_ALL ON + BUILD_IN_SOURCE ON + DEPENDS librdkafka + CMAKE_CACHE_ARGS + ${_CMAKE_ARGS_CMAKE_TOOLCHAIN_FILE} + ${_CMAKE_ARGS_VCPKG_TARGET_TRIPLET} + -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON + PATCH_COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/tools/install_src.py --src ${INSTALL_SRC_DEST_ARG} +) + diff --git a/modern-cpp-kafka/.bazelrc b/modern-cpp-kafka/.bazelrc new file mode 100644 index 00000000..e1ea66ff --- /dev/null +++ b/modern-cpp-kafka/.bazelrc @@ -0,0 +1,2 @@ +build --copt='-std=c++17' --linkopt='-L/usr/local/lib' + diff --git a/modern-cpp-kafka/.clang-tidy b/modern-cpp-kafka/.clang-tidy new file mode 100644 index 00000000..eda43cde --- /dev/null +++ b/modern-cpp-kafka/.clang-tidy @@ -0,0 +1,46 @@ +Checks: "*,\ + -llvm-header-guard,\ + -llvm-namespace-comment,\ + -llvmlibc-restrict-system-libc-headers,\ + -llvmlibc-callee-namespace,\ + -llvmlibc-implementation-in-namespace,\ + -altera-*,\ + -fuchsia-*,\ + -google-readability-namespace-comments,\ + -google-build-using-namespace,\ + -google-runtime-references,\ + -google-readability-avoid-underscore-in-googletest-name,\ + -modernize-use-nodiscard,\ + -modernize-deprecated-headers,\ + -modernize-use-trailing-return-type,\ + -modernize-concat-nested-namespaces,\ + -hicpp-special-member-functions,\ + -hicpp-vararg,\ + -hicpp-no-malloc,\ + -hicpp-no-array-decay,\ + -hicpp-deprecated-headers,\ + -hicpp-braces-around-statements,\ + -cppcoreguidelines-special-member-function,\ + -cppcoreguidelines-macro-usage,\ + -cppcoreguidelines-avoid-magic-numbers,\ + -cppcoreguidelines-avoid-non-const-global-variables,\ + -cppcoreguidelines-pro-type-vararg,\ + -cppcoreguidelines-pro-bounds-array-to-pointer-decay,\ + -cppcoreguidelines-pro-bounds-pointer-arithmetic,\ + -cppcoreguidelines-special-member-functions,\ + -cppcoreguidelines-owning-memory,\ + -cppcoreguidelines-non-private-member-variables-in-classes,\ + -cppcoreguidelines-pro-type-union-access,\ + -misc-non-private-member-variables-in-classes,\ + -misc-no-recursion,\ + -readability-magic-numbers,\ + -readability-implicit-bool-conversion,\ + -readability-braces-around-statements,\ + -readability-isolate-declaration,\ + -readability-identifier-length,\ + -readability-function-cognitive-complexity,\ + -bugprone-unused-return-value,\ + -bugprone-easily-swappable-parameters,\ + -cert-err58-cpp,\ + -cert-err60-cpp" + diff --git a/modern-cpp-kafka/.gitignore b/modern-cpp-kafka/.gitignore new file mode 100755 index 00000000..0727dffc --- /dev/null +++ b/modern-cpp-kafka/.gitignore @@ -0,0 +1,6 @@ +*.swp +*.pyc +*.*~ + +build/ +install/ \ No newline at end of file diff --git a/modern-cpp-kafka/BUILD.bazel b/modern-cpp-kafka/BUILD.bazel new file mode 100644 index 00000000..5e4db202 --- /dev/null +++ b/modern-cpp-kafka/BUILD.bazel @@ -0,0 +1,12 @@ +cc_library( + name = "modern-cpp-kafka-api", + + hdrs = glob(["include/kafka/*.h", "include/kafka/addons/*.h"]), + + includes = ["include"], + + linkopts = ["-lpthread"], + + visibility = ["//visibility:public"], +) + diff --git a/modern-cpp-kafka/CMakeLists.txt b/modern-cpp-kafka/CMakeLists.txt new file mode 100644 index 00000000..188feec3 --- /dev/null +++ b/modern-cpp-kafka/CMakeLists.txt @@ -0,0 +1,70 @@ +cmake_minimum_required(VERSION "3.8") + +project("Modern C++ Kafka API" VERSION 1.0.0) + +get_property(parent_directory DIRECTORY PROPERTY PARENT_DIRECTORY) +if (NOT parent_directory) + set(cppkafka_master_project ON) + # Use Strict Options + if ((CMAKE_CXX_COMPILER_ID STREQUAL "Clang") OR (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")) + add_compile_options("-Wall" "-Werror" "-Wextra" "-Wshadow" "-Wno-unused-result") + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + add_definitions(-D_CRT_SECURE_NO_WARNINGS) + endif () + if (CMAKE_CXX_STANDARD EQUAL 14) + add_compile_options("-Wno-maybe-uninitialized") + endif () +endif () + +include(CheckCXXCompilerFlag) +include(CMakePushCheckState) + + +#--------------------------- +# C++17 (by default) +#--------------------------- +if (NOT CMAKE_CXX_STANDARD) + set(CMAKE_CXX_STANDARD 17) +endif () +set(CMAKE_CXX_STANDARD_REQUIRED False) + +#--------------------------- +# librdkafka library +#--------------------------- +if (DEFINED ENV{LIBRDKAFKA_ROOT}) + set(LIBRDKAFKA_INCLUDE_DIR $ENV{LIBRDKAFKA_ROOT}/include) + set(LIBRDKAFKA_LIBRARY_DIR $ENV{LIBRDKAFKA_ROOT}/lib) +else () + set(LIBRDKAFKA_INCLUDE_DIR /usr/local/include) + set(LIBRDKAFKA_LIBRARY_DIR /usr/local/lib) +endif () + +if (EXISTS "${LIBRDKAFKA_INCLUDE_DIR}/librdkafka/rdkafka.h") + message(STATUS "librdkafka include directory: ${LIBRDKAFKA_INCLUDE_DIR}") +else () + message(FATAL_ERROR "Could not find headers: librdkafka!") +endif () + +if (EXISTS "${LIBRDKAFKA_LIBRARY_DIR}/librdkafka.a" OR EXISTS "${LIBRDKAFKA_LIBRARY_DIR}/librdkafka.so" OR EXISTS "${LIBRDKAFKA_LIBRARY_DIR}/rdkafka.lib" ) + message(STATUS "librdkafka library directory: ${LIBRDKAFKA_LIBRARY_DIR}") +else () + message(FATAL_ERROR "Could not find library: librdkafka!") +endif () + + +#--------------------------- +# pthread library (for linux only) +#--------------------------- +if (${CMAKE_SYSTEM_NAME} STREQUAL "Linux") + find_library(PTHREAD_LIB pthread) + if (PTHREAD_LIB) + message(STATUS "pthread library: ${PTHREAD_LIB}") + else () + message(FATAL_ERROR "Could not find library: pthread!") + endif () +endif () + +#--------------------------- +# Build Sub-directories +#--------------------------- +add_subdirectory("include") diff --git a/modern-cpp-kafka/LICENSE b/modern-cpp-kafka/LICENSE new file mode 100644 index 00000000..fbf43513 --- /dev/null +++ b/modern-cpp-kafka/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [2019] [Morgan Stanley] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/modern-cpp-kafka/NOTICE b/modern-cpp-kafka/NOTICE new file mode 100644 index 00000000..b80d1bb5 --- /dev/null +++ b/modern-cpp-kafka/NOTICE @@ -0,0 +1,3 @@ +Modern C++ Kafka API +Copyright 2020 Morgan Stanley +This project includes software developed at Morgan Stanley. diff --git a/modern-cpp-kafka/README.md b/modern-cpp-kafka/README.md new file mode 100644 index 00000000..51d59177 --- /dev/null +++ b/modern-cpp-kafka/README.md @@ -0,0 +1,228 @@ +# About the *Modern C++ Kafka API* + +![Lifecycle Active](https://badgen.net/badge/Lifecycle/Active/green) + +## Introduction + +The [Modern C++ Kafka API](http://opensource.morganstanley.com/modern-cpp-kafka/doxygen/annotated.html) is a layer of C++ wrapper based on [librdkafka](https://github.com/edenhill/librdkafka) (the C part), with high quality, but more friendly to users. + +- By now, [modern-cpp-kafka](https://github.com/morganstanley/modern-cpp-kafka) is compatible with [librdkafka v1.9.0](https://github.com/edenhill/librdkafka/releases/tag/v1.9.0). + +``` +KAFKA is a registered trademark of The Apache Software Foundation and +has been licensed for use by modern-cpp-kafka. modern-cpp-kafka has no +affiliation with and is not endorsed by The Apache Software Foundation. +``` + +## Why it's here + +The ***librdkafka*** is a robust high performance C/C++ library, widely used and well maintained. + +Unfortunately, to maintain C++98 compatibility, the C++ interface of ***librdkafka*** is not quite object-oriented or user-friendly. + +Since C++ is evolving quickly, we want to take advantage of new C++ features, thus make the life easier for developers. And this led us to create a new C++ API for Kafka clients. + +Eventually, we worked out the ***modern-cpp-kafka***, -- a header-only library that uses idiomatic C++ features to provide a safe, efficient and easy to use way of producing and consuming Kafka messages. + +## Features + +* Header-only + + * Easy to deploy, and no extra library required to link + +* Ease of Use + + * Interface/Naming matches the Java API + + * Object-oriented + + * RAII is used for lifetime management + + * ***librdkafka***'s polling and queue management is now hidden + +* Robust + + * Verified with kinds of test cases, which cover many abnormal scenarios (edge cases) + + * Stability test with unstable brokers + + * Memory leak check for failed client with in-flight messages + + * Client failure and taking over, etc. + +* Efficient + + * No extra performance cost (No deep copy introduced internally) + + * Much better (2~4 times throughput) performance result than those native language (Java/Scala) implementation, in most commonly used cases (message size: 256 B ~ 2 KB) + + +## Build + +* No need to build for installation + +* To build its `tools`/`tests`/`examples`, you should + + * Specify library locations with environment variables + + * `LIBRDKAFKA_ROOT` -- ***librdkafka*** headers and libraries + + * `GTEST_ROOT` -- ***googletest*** headers and libraries + + * `BOOST_ROOT` -- ***boost*** headers and libraries + + * `SASL_LIBRARYDIR`/`SASL_LIBRARY` -- if SASL connection support is wanted + + * `RAPIDJSON_INCLUDE_DIRS` -- `addons/KafkaMetrics` requires **rapidjson** headers + + * Create an empty directory for the build, and `cd` to it + + * Build commands + + * Type `cmake path-to-project-root` + + * Type `make` (could follow build options with `-D`) + + * `BUILD_OPTION_USE_ASAN=ON` -- Use Address Sanitizer + + * `BUILD_OPTION_USE_TSAN=ON` -- Use Thread Sanitizer + + * `BUILD_OPTION_USE_UBSAN=ON` -- Use Undefined Behavior Sanitizer + + * `BUILD_OPTION_CLANG_TIDY=ON` -- Enable clang-tidy checking + + * `BUILD_OPTION_GEN_DOC=ON` -- Generate documentation as well + + * `BUILD_OPTION_DOC_ONLY=ON` -- Only generate documentation + + * `BUILD_OPTION_GEN_COVERAGE=ON` -- Generate test coverage, only support by clang currently + + * Type `make install` + +## Install + +* Include the `include/kafka` directory in your project + +* To work together with ***modern-cpp-kafka*** API, the compiler should support + + * Option 1: C++17 + + * Option 2: C++14 (with pre-requirements) + + * Need ***boost*** headers (for `boost::optional`) + + * GCC only (with optimization, e.g. -O2) + +## How to Run Tests + +* Unit test (`tests/unit`) + + * The test could be run with no Kafka cluster depolyed + +* Integration test (`tests/integration`) + + * The test should be run with Kafka cluster depolyed + + * The environment variable `KAFKA_BROKER_LIST` should be set + + * E.g. `export KAFKA_BROKER_LIST=127.0.0.1:29091,127.0.0.1:29092,127.0.0.1:29093` + +* Robustness test (`tests/robustness`) + + * The test should be run with Kafka cluster depolyed locally + + * The environment variable `KAFKA_BROKER_LIST` should be set + + * The environment variable `KAFKA_BROKER_PIDS` should be set + + * Make sure the test runner gets the privilege to stop/resume the pids + + * E.g. `export KAFKA_BROKER_PIDS=61567,61569,61571` + +* Additional settings for clients + + * The environment variable `KAFKA_CLIENT_ADDITIONAL_SETTINGS` could be used for customized test environment + + * Especially for Kafka cluster with SASL(or SSL) connections + + * E.g. `export KAFKA_CLIENT_ADDITIONAL_SETTINGS="security.protocol=SASL_PLAINTEXT;sasl.kerberos.service.name=...;sasl.kerberos.keytab=...;sasl.kerberos.principal=..."` + +## To Start + +* Tutorial + + * Confluent Blog [Debuting a Modern C++ API for Apache Kafka](https://www.confluent.io/blog/modern-cpp-kafka-api-for-safe-easy-messaging) + + * [KafkaProducer Quick Start](doc/KafkaProducerQuickStart.md) + + * [KafkaConsumer Quick Start](doc/KafkaConsumerQuickStart.md) + +* User's Manual + + * [Kafka Client API](http://opensource.morganstanley.com/modern-cpp-kafka/doxygen/annotated.html) + + + * Kafka Client Properties + + * In most cases, the `Properties` settings for ***modern-cpp-kafka*** are identical with [librdkafka configuration](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md) + + * With following exceptions + + * KafkaConsumer + + * Properties with random string as default + + * `client.id` + + * `group.id` + + * More properties than ***librdkafka*** + + * `max.poll.records` (default: `500`): The maxmum number of records that a single call to `poll()` would return + + * Property which overrides the one from ***librdkafka*** + + * `enable.auto.commit` (default: `false`): To automatically commit the previously polled offsets on each `poll` operation + + * Properties not supposed to be used (internally shadowed by ***modern-cpp-kafka***) + + * `enable.auto.offset.store` + + * `auto.commit.interval.ms` + + * KafkaProducer + + * Properties with random string as default + + * `client.id` + + * Log level + + * The default `log_level` is `NOTICE` (`5`) for all these clients + +* Test Environment (ZooKeeper/Kafka cluster) Setup + + * [Start the servers](https://kafka.apache.org/documentation/#quickstart_startserver) + + +## How to Achieve High Availability & Performance + +* [Kafka Broker Configuration](doc/KafkaBrokerConfiguration.md) + +* [Good Practices to Use KafkaProducer](doc/GoodPracticesToUseKafkaProducer.md) + +* [Good Practices to Use KafkaConsumer](doc/GoodPracticesToUseKafkaConsumer.md) + +* [How to Make KafkaProducer Reliable](doc/HowToMakeKafkaProducerReliable.md) + + +## Other References + +* Java API for Kafka clients + + * [org.apache.kafka.clients.producer](https://kafka.apache.org/22/javadoc/org/apache/kafka/clients/producer/package-summary.html) + + * [org.apache.kafka.clients.consumer](https://kafka.apache.org/22/javadoc/org/apache/kafka/clients/consumer/package-summary.html) + + * [org.apache.kafka.clients.admin](https://kafka.apache.org/22/javadoc/org/apache/kafka/clients/admin/package-summary.html) + diff --git a/modern-cpp-kafka/WORKSPACE b/modern-cpp-kafka/WORKSPACE new file mode 100644 index 00000000..c71a3290 --- /dev/null +++ b/modern-cpp-kafka/WORKSPACE @@ -0,0 +1,15 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "gtest", + urls = ["https://github.com/google/googletest/archive/refs/tags/release-1.11.0.zip"], + strip_prefix = "googletest-release-1.11.0", +) + +http_archive( + name = "rapidjson", + build_file = "//customrules:rapidjson.BUILD", + urls = ["https://github.com/Tencent/rapidjson/archive/refs/heads/master.zip"], + strip_prefix="rapidjson-master", +) + diff --git a/modern-cpp-kafka/_config.yml b/modern-cpp-kafka/_config.yml new file mode 100644 index 00000000..c7418817 --- /dev/null +++ b/modern-cpp-kafka/_config.yml @@ -0,0 +1 @@ +theme: jekyll-theme-slate \ No newline at end of file diff --git a/modern-cpp-kafka/include/CMakeLists.txt b/modern-cpp-kafka/include/CMakeLists.txt new file mode 100644 index 00000000..0ab2783e --- /dev/null +++ b/modern-cpp-kafka/include/CMakeLists.txt @@ -0,0 +1,30 @@ +project(modern-cpp-kafka-api) + +add_library(${PROJECT_NAME} INTERFACE) + +target_include_directories(${PROJECT_NAME} INTERFACE ${CMAKE_CURRENT_LIST_DIR}) + +#--------------------------- +# librdkafka +#--------------------------- +target_include_directories(${PROJECT_NAME} SYSTEM INTERFACE ${LIBRDKAFKA_INCLUDE_DIR}) +target_link_directories(${PROJECT_NAME} INTERFACE ${LIBRDKAFKA_LIBRARY_DIR}) +target_link_libraries(${PROJECT_NAME} INTERFACE rdkafka) + +#--------------------------- +# pthread +#--------------------------- +if (${CMAKE_SYSTEM_NAME} STREQUAL "Linux") + target_link_libraries(${PROJECT_NAME} INTERFACE pthread) +endif () + +#--------------------------- +# sasl (if required) +#--------------------------- +if (SASL_LIBRARY) + target_link_directories(${PROJECT_NAME} INTERFACE ${SASL_LIBRARYDIR}) + target_link_libraries(${PROJECT_NAME} INTERFACE ${SASL_LIBRARY}) +endif () + +# Header-only +install(DIRECTORY kafka DESTINATION include) diff --git a/modern-cpp-kafka/include/kafka/AdminClient.h b/modern-cpp-kafka/include/kafka/AdminClient.h new file mode 100644 index 00000000..d3ef9f3c --- /dev/null +++ b/modern-cpp-kafka/include/kafka/AdminClient.h @@ -0,0 +1,345 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + + +namespace KAFKA_API { namespace clients { + +/** + * The administrative client for Kafka, which supports managing and inspecting topics, etc. + */ +class AdminClient: public KafkaClient +{ +public: + explicit AdminClient(const Properties& properties) + : KafkaClient(ClientType::AdminClient, + KafkaClient::validateAndReformProperties(properties)) + { + } + + /** + * Create a batch of new topics. + */ + admin::CreateTopicsResult createTopics(const Topics& topics, + int numPartitions, + int replicationFactor, + const Properties& topicConfig = Properties(), + std::chrono::milliseconds timeout = std::chrono::milliseconds(DEFAULT_COMMAND_TIMEOUT_MS)); + /** + * Delete a batch of topics. + */ + admin::DeleteTopicsResult deleteTopics(const Topics& topics, + std::chrono::milliseconds timeout = std::chrono::milliseconds(DEFAULT_COMMAND_TIMEOUT_MS)); + /** + * List the topics available in the cluster. + */ + admin::ListTopicsResult listTopics(std::chrono::milliseconds timeout = std::chrono::milliseconds(DEFAULT_COMMAND_TIMEOUT_MS)); + + /** + * Delete records whose offset is smaller than the given offset of the corresponding partition. + * @param topicPartitionOffsets a batch of offsets for partitions + * @param timeout + * @return + */ + admin::DeleteRecordsResult deleteRecords(const TopicPartitionOffsets& topicPartitionOffsets, + std::chrono::milliseconds timeout = std::chrono::milliseconds(DEFAULT_COMMAND_TIMEOUT_MS)); + +private: + static std::list getPerTopicResults(const rd_kafka_topic_result_t** topicResults, std::size_t topicCount); + static std::list getPerTopicPartitionResults(const rd_kafka_topic_partition_list_t* partitionResults); + static Error combineErrors(const std::list& errors); + +#if COMPILER_SUPPORTS_CPP_17 + static constexpr int DEFAULT_COMMAND_TIMEOUT_MS = 30000; +#else + enum { DEFAULT_COMMAND_TIMEOUT_MS = 30000 }; +#endif +}; + + +inline std::list +AdminClient::getPerTopicResults(const rd_kafka_topic_result_t** topicResults, std::size_t topicCount) +{ + std::list errors; + + for (std::size_t i = 0; i < topicCount; ++i) + { + const rd_kafka_topic_result_t* topicResult = topicResults[i]; + if (rd_kafka_resp_err_t topicError = rd_kafka_topic_result_error(topicResult)) + { + std::string detailedMsg = "topic[" + std::string(rd_kafka_topic_result_name(topicResult)) + "] with error[" + rd_kafka_topic_result_error_string(topicResult) + "]"; + errors.emplace_back(topicError, detailedMsg); + } + } + return errors; +} + +inline std::list +AdminClient::getPerTopicPartitionResults(const rd_kafka_topic_partition_list_t* partitionResults) +{ + std::list errors; + + for (int i = 0; i < (partitionResults ? partitionResults->cnt : 0); ++i) + { + if (rd_kafka_resp_err_t partitionError = partitionResults->elems[i].err) + { + std::string detailedMsg = "topic-partition[" + std::string(partitionResults->elems[i].topic) + "-" + std::to_string(partitionResults->elems[i].partition) + "] with error[" + rd_kafka_err2str(partitionError) + "]"; + errors.emplace_back(partitionError, detailedMsg); + } + } + return errors; +} + +inline Error +AdminClient::combineErrors(const std::list& errors) +{ + if (!errors.empty()) + { + std::string detailedMsg; + std::for_each(errors.cbegin(), errors.cend(), + [&detailedMsg](const auto& error) { + if (!detailedMsg.empty()) detailedMsg += "; "; + + detailedMsg += error.message(); + }); + + return Error{static_cast(errors.front().value()), detailedMsg}; + } + + return Error{RD_KAFKA_RESP_ERR_NO_ERROR, "Success"}; +} + +inline admin::CreateTopicsResult +AdminClient::createTopics(const Topics& topics, + int numPartitions, + int replicationFactor, + const Properties& topicConfig, + std::chrono::milliseconds timeout) +{ + LogBuffer<500> errInfo; + + std::vector rkNewTopics; + + for (const auto& topic: topics) + { + rkNewTopics.emplace_back(rd_kafka_NewTopic_new(topic.c_str(), numPartitions, replicationFactor, errInfo.str(), errInfo.capacity())); + if (!rkNewTopics.back()) + { + return admin::CreateTopicsResult(Error{RD_KAFKA_RESP_ERR__INVALID_ARG, rd_kafka_err2str(RD_KAFKA_RESP_ERR__INVALID_ARG)}); + } + + for (const auto& conf: topicConfig.map()) + { + rd_kafka_resp_err_t err = rd_kafka_NewTopic_set_config(rkNewTopics.back().get(), conf.first.c_str(), conf.second.c_str()); + if (err != RD_KAFKA_RESP_ERR_NO_ERROR) + { + std::string errMsg = "Invalid config[" + conf.first + "=" + conf.second + "]"; + KAFKA_API_DO_LOG(Log::Level::Err, errMsg.c_str()); + return admin::CreateTopicsResult(Error{RD_KAFKA_RESP_ERR__INVALID_ARG, errMsg}); + } + } + } + + std::vector rk_topics; + rk_topics.reserve(rkNewTopics.size()); + for (const auto& topic : rkNewTopics) { rk_topics.emplace_back(topic.get()); } + + auto rk_queue = rd_kafka_queue_unique_ptr(rd_kafka_queue_new(getClientHandle())); + + rd_kafka_CreateTopics(getClientHandle(), rk_topics.data(), rk_topics.size(), nullptr, rk_queue.get()); + + auto rk_ev = rd_kafka_event_unique_ptr(); + + const auto end = std::chrono::steady_clock::now() + timeout; + do + { + rk_ev.reset(rd_kafka_queue_poll(rk_queue.get(), EVENT_POLLING_INTERVAL_MS)); + + if (rd_kafka_event_type(rk_ev.get()) == RD_KAFKA_EVENT_CREATETOPICS_RESULT) break; + + if (rk_ev) + { + KAFKA_API_DO_LOG(Log::Level::Err, "rd_kafka_queue_poll got event[%s], with error[%s]", rd_kafka_event_name(rk_ev.get()), rd_kafka_event_error_string(rk_ev.get())); + rk_ev.reset(); + } + } while (std::chrono::steady_clock::now() < end); + + if (!rk_ev) + { + return admin::CreateTopicsResult(Error{RD_KAFKA_RESP_ERR__TIMED_OUT, "No response within the time limit"}); + } + + std::list errors; + + if (rd_kafka_resp_err_t respErr = rd_kafka_event_error(rk_ev.get())) + { + errors.emplace_back(respErr, rd_kafka_event_error_string(rk_ev.get())); + } + + // Fetch per-topic results + const rd_kafka_CreateTopics_result_t* res = rd_kafka_event_CreateTopics_result(rk_ev.get()); + std::size_t res_topic_cnt{}; + const rd_kafka_topic_result_t** res_topics = rd_kafka_CreateTopics_result_topics(res, &res_topic_cnt); + + errors.splice(errors.end(), getPerTopicResults(res_topics, res_topic_cnt)); + + // Return the error if any + if (!errors.empty()) + { + return admin::CreateTopicsResult{combineErrors(errors)}; + } + + // Update metedata + do + { + auto listResult = listTopics(); + if (!listResult.error) + { + return admin::CreateTopicsResult(Error{RD_KAFKA_RESP_ERR_NO_ERROR, "Success"}); + } + } while (std::chrono::steady_clock::now() < end); + + return admin::CreateTopicsResult(Error{RD_KAFKA_RESP_ERR__TIMED_OUT, "Updating metadata timed out"}); +} + +inline admin::DeleteTopicsResult +AdminClient::deleteTopics(const Topics& topics, std::chrono::milliseconds timeout) +{ + std::vector rkDeleteTopics; + + for (const auto& topic: topics) + { + rkDeleteTopics.emplace_back(rd_kafka_DeleteTopic_new(topic.c_str())); + assert(rkDeleteTopics.back()); + } + + std::vector rk_topics; + rk_topics.reserve(rkDeleteTopics.size()); + for (const auto& topic : rkDeleteTopics) { rk_topics.emplace_back(topic.get()); } + + auto rk_queue = rd_kafka_queue_unique_ptr(rd_kafka_queue_new(getClientHandle())); + + rd_kafka_DeleteTopics(getClientHandle(), rk_topics.data(), rk_topics.size(), nullptr, rk_queue.get()); + + auto rk_ev = rd_kafka_event_unique_ptr(); + + const auto end = std::chrono::steady_clock::now() + timeout; + do + { + rk_ev.reset(rd_kafka_queue_poll(rk_queue.get(), EVENT_POLLING_INTERVAL_MS)); + + if (rd_kafka_event_type(rk_ev.get()) == RD_KAFKA_EVENT_DELETETOPICS_RESULT) break; + + if (rk_ev) + { + KAFKA_API_DO_LOG(Log::Level::Err, "rd_kafka_queue_poll got event[%s], with error[%s]", rd_kafka_event_name(rk_ev.get()), rd_kafka_event_error_string(rk_ev.get())); + rk_ev.reset(); + } + } while (std::chrono::steady_clock::now() < end); + + if (!rk_ev) + { + return admin::DeleteTopicsResult(Error{RD_KAFKA_RESP_ERR__TIMED_OUT, "No response within the time limit"}); + } + + std::list errors; + + if (rd_kafka_resp_err_t respErr = rd_kafka_event_error(rk_ev.get())) + { + errors.emplace_back(respErr, rd_kafka_event_error_string(rk_ev.get())); + } + + // Fetch per-topic results + const rd_kafka_DeleteTopics_result_t* res = rd_kafka_event_DeleteTopics_result(rk_ev.get()); + std::size_t res_topic_cnt{}; + const rd_kafka_topic_result_t** res_topics = rd_kafka_DeleteTopics_result_topics(res, &res_topic_cnt); + + errors.splice(errors.end(), getPerTopicResults(res_topics, res_topic_cnt)); + + return admin::DeleteTopicsResult(combineErrors(errors)); +} + +inline admin::ListTopicsResult +AdminClient::listTopics(std::chrono::milliseconds timeout) +{ + const rd_kafka_metadata_t* rk_metadata = nullptr; + rd_kafka_resp_err_t err = rd_kafka_metadata(getClientHandle(), true, nullptr, &rk_metadata, convertMsDurationToInt(timeout)); + auto guard = rd_kafka_metadata_unique_ptr(rk_metadata); + + if (err != RD_KAFKA_RESP_ERR_NO_ERROR) + { + return admin::ListTopicsResult(Error{err, rd_kafka_err2str(err)}); + } + + Topics names; + for (int i = 0; i < rk_metadata->topic_cnt; ++i) + { + names.insert(rk_metadata->topics[i].topic); + } + return admin::ListTopicsResult(names); +} + +inline admin::DeleteRecordsResult +AdminClient::deleteRecords(const TopicPartitionOffsets& topicPartitionOffsets, + std::chrono::milliseconds timeout) +{ + auto rk_queue = rd_kafka_queue_unique_ptr(rd_kafka_queue_new(getClientHandle())); + + rd_kafka_DeleteRecords_unique_ptr rkDeleteRecords(rd_kafka_DeleteRecords_new(createRkTopicPartitionList(topicPartitionOffsets))); + std::array rk_del_records{rkDeleteRecords.get()}; + + rd_kafka_DeleteRecords(getClientHandle(), rk_del_records.data(), rk_del_records.size(), nullptr, rk_queue.get()); + + auto rk_ev = rd_kafka_event_unique_ptr(); + + const auto end = std::chrono::steady_clock::now() + timeout; + do + { + rk_ev.reset(rd_kafka_queue_poll(rk_queue.get(), EVENT_POLLING_INTERVAL_MS)); + + if (rd_kafka_event_type(rk_ev.get()) == RD_KAFKA_EVENT_DELETERECORDS_RESULT) break; + + if (rk_ev) + { + KAFKA_API_DO_LOG(Log::Level::Err, "rd_kafka_queue_poll got event[%s], with error[%s]", rd_kafka_event_name(rk_ev.get()), rd_kafka_event_error_string(rk_ev.get())); + rk_ev.reset(); + } + } while (std::chrono::steady_clock::now() < end); + + if (!rk_ev) + { + return admin::DeleteRecordsResult(Error{RD_KAFKA_RESP_ERR__TIMED_OUT, "No response within the time limit"}); + } + + std::list errors; + + if (rd_kafka_resp_err_t respErr = rd_kafka_event_error(rk_ev.get())) + { + errors.emplace_back(respErr, rd_kafka_event_error_string(rk_ev.get())); + } + + const rd_kafka_DeleteRecords_result_t* res = rd_kafka_event_DeleteRecords_result(rk_ev.get()); + const rd_kafka_topic_partition_list_t* res_offsets = rd_kafka_DeleteRecords_result_offsets(res); + + errors.splice(errors.end(), getPerTopicPartitionResults(res_offsets)); + + return admin::DeleteRecordsResult(combineErrors(errors)); +} + +} } // end of KAFKA_API::clients + diff --git a/modern-cpp-kafka/include/kafka/AdminClientConfig.h b/modern-cpp-kafka/include/kafka/AdminClientConfig.h new file mode 100644 index 00000000..2bd6977e --- /dev/null +++ b/modern-cpp-kafka/include/kafka/AdminClientConfig.h @@ -0,0 +1,44 @@ +#pragma once + +#include + +#include + + +namespace KAFKA_API { namespace clients { namespace admin { + +/** + * Configuration for the Kafka Consumer. + */ +class Config: public Properties +{ +public: + Config() = default; + Config(const Config&) = default; + explicit Config(const PropertiesMap& kvMap): Properties(kvMap) {} + + /** + * The string contains host:port pairs of brokers (splitted by ",") that the administrative client will use to establish initial connection to the Kafka cluster. + * Note: It's mandatory. + */ + static const constexpr char* BOOTSTRAP_SERVERS = "bootstrap.servers"; + + /** + * Protocol used to communicate with brokers. + * Default value: plaintext + */ + static const constexpr char* SECURITY_PROTOCOL = "security.protocol"; + + /** + * Shell command to refresh or acquire the client's Kerberos ticket. + */ + static const constexpr char* SASL_KERBEROS_KINIT_CMD = "sasl.kerberos.kinit.cmd"; + + /** + * The client's Kerberos principal name. + */ + static const constexpr char* SASL_KERBEROS_SERVICE_NAME = "sasl.kerberos.service.name"; +}; + +} } } // end of KAFKA_API::clients::admin + diff --git a/modern-cpp-kafka/include/kafka/AdminCommon.h b/modern-cpp-kafka/include/kafka/AdminCommon.h new file mode 100644 index 00000000..f821d904 --- /dev/null +++ b/modern-cpp-kafka/include/kafka/AdminCommon.h @@ -0,0 +1,70 @@ +#pragma once + +#include + +#include +#include + + +namespace KAFKA_API { namespace clients { namespace admin { + +/** + * The result of AdminClient::createTopics(). + */ +struct CreateTopicsResult +{ + explicit CreateTopicsResult(const Error& err): error(err) {} + + /** + * The result error. + */ + Error error; +}; + +/** + * The result of AdminClient::deleteTopics(). + */ +struct DeleteTopicsResult +{ + explicit DeleteTopicsResult(const Error& err): error(err) {} + + /** + * The result error. + */ + Error error; +}; + +/** + * The result of AdminClient::deleteRecords(). + */ +struct DeleteRecordsResult +{ + explicit DeleteRecordsResult(const Error& err): error(err) {} + + /** + * The result error. + */ + Error error; +}; + +/** + * The result of AdminClient::listTopics(). + */ +struct ListTopicsResult +{ + explicit ListTopicsResult(const Error& err): error(err) {} + explicit ListTopicsResult(Topics names): topics(std::move(names)) {} + + /** + * The result error. + */ + Error error; + + /** + * The topics fetched. + */ + Topics topics; +}; + +} } } // end of KAFKA_API::clients::admin + diff --git a/modern-cpp-kafka/include/kafka/BrokerMetadata.h b/modern-cpp-kafka/include/kafka/BrokerMetadata.h new file mode 100644 index 00000000..e0f86bce --- /dev/null +++ b/modern-cpp-kafka/include/kafka/BrokerMetadata.h @@ -0,0 +1,187 @@ +#pragma once + +#include + +#include +#include + +#include + +#include +#include + + +namespace KAFKA_API { + +/** + * The metadata info for a topic. + */ +struct BrokerMetadata { + /** + * Information for a Kafka node. + */ + struct Node + { + public: + using Id = int; + using Host = std::string; + using Port = int; + + Node(Id i, Host h, Port p): id(i), host(std::move(h)), port(p) {} + + /** + * The node id. + */ + Node::Id id; + + /** + * The host name. + */ + Node::Host host; + + /** + * The port. + */ + Node::Port port; + + /** + * Obtains explanatory string. + */ + std::string toString() const { return host + ":" + std::to_string(port) + "/" + std::to_string(id); } + }; + + /** + * It is used to describe per-partition state in the MetadataResponse. + */ + struct PartitionInfo + { + explicit PartitionInfo(Node::Id leaderId): leader(leaderId) {} + + void addReplica(Node::Id id) { replicas.emplace_back(id); } + void addInSyncReplica(Node::Id id) { inSyncReplicas.emplace_back(id); } + + /** + * The node id currently acting as a leader for this partition or null if there is no leader. + */ + Node::Id leader; + + /** + * The complete set of replicas id for this partition regardless of whether they are alive or up-to-date. + */ + std::vector replicas; + + /** + * The subset of the replicas id that are in sync, that is caught-up to the leader and ready to take over as leader if the leader should fail. + */ + std::vector inSyncReplicas; + + }; + + /** + * Obtains explanatory string from Node::Id. + */ + std::string getNodeDescription(Node::Id id) const; + + /** + * Obtains explanatory string for PartitionInfo. + */ + std::string toString(const PartitionInfo& partitionInfo) const; + + /** + * The BrokerMetadata is per-topic constructed. + */ + explicit BrokerMetadata(Topic topic): _topic(std::move(topic)) {} + + /** + * The topic name. + */ + const std::string& topic() const { return _topic; } + + /** + * The nodes info in the MetadataResponse. + */ + std::vector> nodes() const; + + /** + * The partitions' state in the MetadataResponse. + */ + const std::map& partitions() const { return _partitions; } + + /** + * Obtains explanatory string. + */ + std::string toString() const; + + void setOrigNodeName(const std::string& origNodeName) { _origNodeName = origNodeName; } + void addNode(Node::Id nodeId, const Node::Host& host, Node::Port port) { _nodes[nodeId] = std::make_shared(nodeId, host, port); } + void addPartitionInfo(Partition partition, const PartitionInfo& partitionInfo) { _partitions.emplace(partition, partitionInfo); } + +private: + Topic _topic; + std::string _origNodeName; + std::map> _nodes; + std::map _partitions; +}; + +inline std::vector> +BrokerMetadata::nodes() const +{ + std::vector> ret; + for (const auto& nodeInfo: _nodes) + { + ret.emplace_back(nodeInfo.second); + } + return ret; +} + +inline std::string +BrokerMetadata::getNodeDescription(Node::Id id) const +{ + const auto& found = _nodes.find(id); + if (found == _nodes.cend()) return "-:-/" + std::to_string(id); + + auto node = found->second; + return node->host + ":" + std::to_string(node->port) + "/" + std::to_string(id); +} + +inline std::string +BrokerMetadata::toString(const PartitionInfo& partitionInfo) const +{ + std::ostringstream oss; + + auto streamNodes = [this](std::ostringstream& ss, const std::vector& nodeIds) -> std::ostringstream& { + bool isTheFirst = true; + for (const auto id: nodeIds) + { + ss << (isTheFirst ? (isTheFirst = false, "") : ", ") << getNodeDescription(id); + } + return ss; + }; + + oss << "leader[" << getNodeDescription(partitionInfo.leader) << "], replicas["; + streamNodes(oss, partitionInfo.replicas) << "], inSyncReplicas["; + streamNodes(oss, partitionInfo.inSyncReplicas) << "]"; + + return oss.str(); +} + +inline std::string +BrokerMetadata::toString() const +{ + std::ostringstream oss; + + oss << "originatingNode[" << _origNodeName << "], topic[" << _topic << "], partitions{"; + bool isTheFirst = true; + for (const auto& partitionInfoPair: _partitions) + { + const Partition partition = partitionInfoPair.first; + const PartitionInfo& partitionInfo = partitionInfoPair.second; + oss << (isTheFirst ? (isTheFirst = false, "") : "; ") << partition << ": " << toString(partitionInfo); + } + oss << "}"; + + return oss.str(); +} + +} // end of KAFKA_API + diff --git a/modern-cpp-kafka/include/kafka/ConsumerCommon.h b/modern-cpp-kafka/include/kafka/ConsumerCommon.h new file mode 100644 index 00000000..a1150c3a --- /dev/null +++ b/modern-cpp-kafka/include/kafka/ConsumerCommon.h @@ -0,0 +1,65 @@ +#pragma once + +#include + +#include +#include +#include + +#include + +#include + + +namespace KAFKA_API { namespace clients { namespace consumer { + + /** + * To identify which kind of re-balance event is handling, when the set of partitions assigned to the consumer changes. + * It's guaranteed that rebalance callback will be called twice (first with PartitionsRevoked, and then with PartitionsAssigned). + */ + enum class RebalanceEventType { PartitionsAssigned, PartitionsRevoked }; + + /** + * A callback interface that the user can implement to trigger custom actions when the set of partitions assigned to the consumer changes. + */ + using RebalanceCallback = std::function; + + /** + * Null RebalanceCallback + */ +#if COMPILER_SUPPORTS_CPP_17 + const inline RebalanceCallback NullRebalanceCallback = RebalanceCallback{}; +#else + const static RebalanceCallback NullRebalanceCallback = RebalanceCallback{}; +#endif + + /** + * A callback interface that the user can implement to trigger custom actions when a commit request completes. + */ + using OffsetCommitCallback = std::function; + + /** + * Null OffsetCommitCallback + */ +#if COMPILER_SUPPORTS_CPP_17 + const inline OffsetCommitCallback NullOffsetCommitCallback = OffsetCommitCallback{}; +#else + const static OffsetCommitCallback NullOffsetCommitCallback = OffsetCommitCallback{}; +#endif + + /** + * A metadata struct containing the consumer group information. + */ + class ConsumerGroupMetadata + { + public: + explicit ConsumerGroupMetadata(rd_kafka_consumer_group_metadata_t* p): _rkConsumerGroupMetadata(p) {} + + const rd_kafka_consumer_group_metadata_t* rawHandle() const { return _rkConsumerGroupMetadata.get(); } + + private: + rd_kafka_consumer_group_metadata_unique_ptr _rkConsumerGroupMetadata; + }; + +} } } // end of KAFKA_API::clients::consumer + diff --git a/modern-cpp-kafka/include/kafka/ConsumerConfig.h b/modern-cpp-kafka/include/kafka/ConsumerConfig.h new file mode 100644 index 00000000..95a7db1f --- /dev/null +++ b/modern-cpp-kafka/include/kafka/ConsumerConfig.h @@ -0,0 +1,115 @@ +#pragma once + +#include + +#include + + +namespace KAFKA_API { namespace clients { namespace consumer { + +/** + * Configuration for the Kafka Consumer. + */ +class Config: public Properties +{ +public: + Config() = default; + Config(const Config&) = default; + explicit Config(const PropertiesMap& kvMap): Properties(kvMap) {} + + /** + * The string contains host:port pairs of brokers (splitted by ",") that the consumer will use to establish initial connection to the Kafka cluster. + * Note: It's mandatory. + */ + static const constexpr char* BOOTSTRAP_SERVERS = "bootstrap.servers"; + + /** + * Group identifier. + * Note: It's better to configure it manually, otherwise a random one would be used for it. + * + */ + static const constexpr char* GROUP_ID = "group.id"; + + /** + * Client identifier. + */ + static const constexpr char* CLIENT_ID = "client.id"; + + /** + * Automatically commits previously polled offsets on each `poll` operation. + */ + static const constexpr char* ENABLE_AUTO_COMMIT = "enable.auto.commit"; + + /** + * This property controls the behavior of the consumer when it starts reading a partition for which it doesn't have a valid committed offset. + * The "latest" means the consumer will begin reading the newest records written after the consumer started. While "earliest" means that the consumer will read from the very beginning. + * Available options: latest, earliest + * Default value: latest + */ + static const constexpr char* AUTO_OFFSET_RESET = "auto.offset.reset"; + + /** + * Emit RD_KAFKA_RESP_ERR_PARTITION_EOF event whenever the consumer reaches the end of a partition. + * Default value: false + */ + static const constexpr char* ENABLE_PARTITION_EOF = "enable.partition.eof"; + + /** + * This controls the maximum number of records that a single call to poll() will return. + * Default value: 500 + */ + static const constexpr char* MAX_POLL_RECORDS = "max.poll.records"; + + /** + * Minimum number of messages per topic/partition tries to maintain in the local consumer queue. + * Note: With a larger value configured, the consumer would send FetchRequest towards brokers more frequently. + * Defalut value: 100000 + */ + static const constexpr char* QUEUED_MIN_MESSAGES = "queued.min.messages"; + + /** + * Client group session and failure detection timeout. + * If no heartbeat received by the broker within this timeout, the broker will remove the consumer and trigger a rebalance. + * Default value: 10000 + */ + static const constexpr char* SESSION_TIMEOUT_MS = "session.timeout.ms"; + + /** + * Timeout for network requests. + * Default value: 60000 + */ + static const constexpr char* SOCKET_TIMEOUT_MS = "socket.timeout.ms"; + + /** + * Control how to read messages written transactionally. + * Available options: read_uncommitted, read_committed + * Default value: read_committed + */ + static const constexpr char* ISOLATION_LEVEL = "isolation.level"; + + /* + * The name of one or more partition assignment strategies. + * The elected group leader will use a strategy supported by all members of the group to assign partitions to group members. + * Available options: range, roundrobin, cooperative-sticky + * Default value: range,roundrobin + */ + static const constexpr char* PARTITION_ASSIGNMENT_STRATEGY = "partition.assignment.strategy"; + /** + * Protocol used to communicate with brokers. + * Default value: plaintext + */ + static const constexpr char* SECURITY_PROTOCOL = "security.protocol"; + + /** + * Shell command to refresh or acquire the client's Kerberos ticket. + */ + static const constexpr char* SASL_KERBEROS_KINIT_CMD = "sasl.kerberos.kinit.cmd"; + + /** + * The client's Kerberos principal name. + */ + static const constexpr char* SASL_KERBEROS_SERVICE_NAME = "sasl.kerberos.service.name"; +}; + +} } } // end of KAFKA_API::clients::consumer + diff --git a/modern-cpp-kafka/include/kafka/ConsumerRecord.h b/modern-cpp-kafka/include/kafka/ConsumerRecord.h new file mode 100644 index 00000000..e78f6030 --- /dev/null +++ b/modern-cpp-kafka/include/kafka/ConsumerRecord.h @@ -0,0 +1,154 @@ +#pragma once + +#include + +#include +#include +#include +#include + +#include + +#include + + +namespace KAFKA_API { namespace clients { namespace consumer { + +/** + * A key/value pair to be received from Kafka. + * This also consists of a topic name and a partition number from which the record is being received, an offset that points to the record in a Kafka partition + */ +class ConsumerRecord +{ +public: + // ConsumerRecord will take the ownership of msg (rd_kafka_message_t*) + explicit ConsumerRecord(rd_kafka_message_t* msg): _rk_msg(msg, rd_kafka_message_destroy) {} + + /** + * The topic this record is received from. + */ + Topic topic() const { return _rk_msg->rkt ? rd_kafka_topic_name(_rk_msg->rkt): ""; } + + /** + * The partition from which this record is received. + */ + Partition partition() const { return _rk_msg->partition; } + + /** + * The position of this record in the corresponding Kafka partition. + */ + Offset offset() const { return _rk_msg->offset; } + + /** + * The key (or null if no key is specified). + */ + Key key() const { return Key(_rk_msg->key, _rk_msg->key_len); } + + /** + * The value. + */ + Value value() const { return Value(_rk_msg->payload, _rk_msg->len); } + + /** + * The timestamp of the record. + */ + Timestamp timestamp() const + { + rd_kafka_timestamp_type_t tstype{}; + Timestamp::Value tsValue = rd_kafka_message_timestamp(_rk_msg.get(), &tstype); + return {tsValue, tstype}; + } + + /** + * The headers of the record. + */ + Headers headers() const; + + /** + * Return just one (the very last) header's value for the given key. + */ + Header::Value lastHeaderValue(const Header::Key& key); + + /** + * The error. + * + * Possible cases: + * 1. Success + * - RD_KAFKA_RESP_ERR_NO_ERROR (0), -- got a message successfully + * - RD_KAFKA_RESP_ERR__PARTITION_EOF, -- reached the end of a partition (got no message) + * 2. Failure + * - [Error Codes] (https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-ErrorCodes) + */ + Error error() const { return Error{_rk_msg->err}; } + + /** + * Obtains explanatory string. + */ + std::string toString() const; + +private: + using rd_kafka_message_shared_ptr = std::shared_ptr; + rd_kafka_message_shared_ptr _rk_msg; +}; + +inline Headers +ConsumerRecord::headers() const +{ + Headers headers; + + rd_kafka_headers_t* hdrs = nullptr; + if (rd_kafka_message_headers(_rk_msg.get(), &hdrs) != RD_KAFKA_RESP_ERR_NO_ERROR) + { + return headers; + } + + headers.reserve(rd_kafka_header_cnt(hdrs)); + + const char* name = nullptr; + const void* valuePtr = nullptr; + std::size_t valueSize = 0; + for (std::size_t i = 0; !rd_kafka_header_get_all(hdrs, i, &name, &valuePtr, &valueSize); i++) + { + headers.emplace_back(name, Header::Value(valuePtr, valueSize)); + } + + return headers; +} + +inline Header::Value +ConsumerRecord::lastHeaderValue(const Header::Key& key) +{ + rd_kafka_headers_t* hdrs = nullptr; + if (rd_kafka_message_headers(_rk_msg.get(), &hdrs) != RD_KAFKA_RESP_ERR_NO_ERROR) + { + return Header::Value(); + } + + const void* valuePtr = nullptr; + std::size_t valueSize = 0; + return (rd_kafka_header_get_last(hdrs, key.c_str(), &valuePtr, &valueSize) == RD_KAFKA_RESP_ERR_NO_ERROR) ? + Header::Value(valuePtr, valueSize) : Header::Value(); +} + +inline std::string +ConsumerRecord::toString() const +{ + std::ostringstream oss; + if (!error()) + { + oss << topic() << "-" << partition() << ":" << offset() << ", " << timestamp().toString() << ", " + << (key().size() ? (key().toString() + "/") : "") << value().toString(); + } + else if (error().value() == RD_KAFKA_RESP_ERR__PARTITION_EOF) + { + oss << "EOF[" << topic() << "-" << partition() << ":" << offset() << "]"; + } + else + { + oss << "ERROR[" << error().message() << ", " << topic() << "-" << partition() << ":" << offset() << "]"; + } + return oss.str(); +} + +} } } // end of KAFKA_API::clients::consumer + diff --git a/modern-cpp-kafka/include/kafka/Error.h b/modern-cpp-kafka/include/kafka/Error.h new file mode 100644 index 00000000..4212d1b1 --- /dev/null +++ b/modern-cpp-kafka/include/kafka/Error.h @@ -0,0 +1,148 @@ +#pragma once + +#include + +#include + +#include + +#include +#include + + +namespace KAFKA_API { + +struct ErrorCategory: public std::error_category +{ + const char* name() const noexcept override { return "KafkaError"; } + std::string message(int ev) const override { return rd_kafka_err2str(static_cast(ev)); } + + template + struct Global { static ErrorCategory category; }; +}; + +template +ErrorCategory ErrorCategory::Global::category; + + +/** + * Unified error type. + */ +class Error +{ +public: + // The error with rich info + explicit Error(rd_kafka_error_t* error = nullptr): _rkError(error, RkErrorDeleter) {} + // The error with brief info + explicit Error(rd_kafka_resp_err_t respErr): _respErr(respErr) {} + // The error with detailed message + Error(rd_kafka_resp_err_t respErr, std::string message, bool fatal = false) + : _respErr(respErr), _message(std::move(message)), _isFatal(fatal) {} + // Copy constructor + Error(const Error& error) { *this = error; } + + // Assignment operator + Error& operator=(const Error& error) + { + if (this == &error) return *this; + + _rkError.reset(); + + _respErr = static_cast(error.value()); + _message = error._message; + _isFatal = error.isFatal(); + _txnRequiresAbort = error.transactionRequiresAbort(); + _isRetriable = error.isRetriable(); + + return *this; + } + + /** + * Check if the error is valid. + */ + explicit operator bool() const { return static_cast(value()); } + + /** + * Conversion to `std::error_code` + */ + explicit operator std::error_code() const + { + return {value(), ErrorCategory::Global<>::category}; + } + + /** + * Obtains the underlying error code value. + * + * Actually, it's the same as 'rd_kafka_resp_err_t', which is defined by librdkafka. + * 1. The negative values are for internal errors. + * 2. Non-negative values are for external errors. See the defination at, + * - [Error Codes] (https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-ErrorCodes) + */ + int value() const + { + return static_cast(_rkError ? rd_kafka_error_code(_rkError.get()) : _respErr); + } + + /** + * Readable error string. + */ + std::string message() const + { + return _message ? *_message : + (_rkError ? rd_kafka_error_string(_rkError.get()) : rd_kafka_err2str(_respErr)); + } + + /** + * Detailed error string. + */ + std::string toString() const + { + std::ostringstream oss; + + oss << rd_kafka_err2str(static_cast(value())) << " [" << value() << "]" << (isFatal() ? " fatal" : ""); + if (transactionRequiresAbort()) oss << " | transaction-requires-abort"; + if (auto retriable = isRetriable()) oss << " | " << (*retriable ? "retriable" : "non-retriable"); + if (_message) oss << " | " << *_message; + + return oss.str(); + } + + /** + * Fatal error indicates that the client instance is no longer usable. + */ + bool isFatal() const + { + return _rkError ? rd_kafka_error_is_fatal(_rkError.get()) : _isFatal; + } + + /** + * Show whether the operation may be retried. + */ + Optional isRetriable() const + { + return _rkError ? rd_kafka_error_is_retriable(_rkError.get()) : _isRetriable; + } + + /** + * Show whether the error is an abortable transaction error. + * + * Note: + * 1. Only valid for transactional API. + * 2. If `true`, the producer must call `abortTransaction` and start a new transaction with `beginTransaction` to proceed with transactions. + */ + bool transactionRequiresAbort() const + { + return _rkError ? rd_kafka_error_txn_requires_abort(_rkError.get()) : false; + } + +private: + rd_kafka_error_shared_ptr _rkError; // For error with rich info + rd_kafka_resp_err_t _respErr{}; // For error with a simple response code + Optional _message; // Additional detailed message (if any) + bool _isFatal = false; + bool _txnRequiresAbort = false; + Optional _isRetriable; // Retriable flag (if any) +}; + +} // end of KAFKA_API + diff --git a/modern-cpp-kafka/include/kafka/Header.h b/modern-cpp-kafka/include/kafka/Header.h new file mode 100644 index 00000000..58cc6767 --- /dev/null +++ b/modern-cpp-kafka/include/kafka/Header.h @@ -0,0 +1,65 @@ +#pragma once + +#include + +#include + +#include +#include +#include + + +namespace KAFKA_API { + +/** + * Message Header (with a key value pair) + */ +struct Header +{ + using Key = std::string; + using Value = ConstBuffer; + + Header() = default; + Header(Key k, Value v): key(std::move(k)), value(v) {} + + /** + * Obtains explanatory string. + */ + std::string toString() const + { + return key + ":" + value.toString(); + } + + Key key; + Value value; +}; + +/** + * Message Headers. + */ +using Headers = std::vector
; + +/** + * Null Headers. + */ +#if COMPILER_SUPPORTS_CPP_17 +const inline Headers NullHeaders = Headers{}; +#else +const static Headers NullHeaders = Headers{}; +#endif + +/** + * Obtains explanatory string for Headers. + */ +inline std::string toString(const Headers& headers) +{ + std::string ret; + std::for_each(headers.cbegin(), headers.cend(), + [&ret](const auto& header) { + ret.append(ret.empty() ? "" : ",").append(header.toString()); + }); + return ret; +} + +} // end of KAFKA_API + diff --git a/modern-cpp-kafka/include/kafka/KafkaClient.h b/modern-cpp-kafka/include/kafka/KafkaClient.h new file mode 100644 index 00000000..f6b07297 --- /dev/null +++ b/modern-cpp-kafka/include/kafka/KafkaClient.h @@ -0,0 +1,626 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace KAFKA_API { namespace clients { + +/** + * The base class for Kafka clients. + */ +class KafkaClient +{ +public: + /** + * The option shows whether user wants to call `pollEvents()` manually to trigger internal callbacks. + */ + enum class EventsPollingOption { Manual, Auto }; + + virtual ~KafkaClient() = default; + + /** + * Get the client id. + */ + const std::string& clientId() const { return _clientId; } + + /** + * Get the client name (i.e. client type + id). + */ + const std::string& name() const { return _clientName; } + + /** + * Set a log callback for kafka clients, which do not have a client specific logging callback configured (see `setLogger`). + */ + static void setGlobalLogger(Logger logger = NullLogger) + { + std::call_once(Global<>::initOnce, [](){}); // Then no need to init within KafkaClient constructor + Global<>::logger = std::move(logger); + } + + /** + * Set the log callback for the kafka client (it's a per-client setting). + */ + void setLogger(Logger logger) { _logger = std::move(logger); } + + /** + * Set log level for the kafka client (the default value: 5). + */ + void setLogLevel(int level); + + /** + * Callback type for statistics info dumping. + */ + using StatsCallback = std::function; + + /** + * Set callback to receive the periodic statistics info. + * Note: 1) It only works while the "statistics.interval.ms" property is configured with a non-0 value. + * 2) The callback would be triggered periodically, receiving the internal statistics info (with JSON format) emited from librdkafka. + */ + void setStatsCallback(StatsCallback cb) { _statsCb = std::move(cb); } + + /** + * Callback type for error notification. + */ + using ErrorCallback = std::function; + + /** + * Set callback for error notification. + */ + void setErrorCallback(ErrorCallback cb) { _errorCb = std::move(cb); } + + /** + * Return the properties which took effect. + */ + const Properties& properties() const { return _properties; } + + /** + * Fetch the effected property (including the property internally set by librdkafka). + */ + Optional getProperty(const std::string& name) const; + + /** + * Call the OffsetCommit callbacks (if any) + * Note: The Kafka client should be constructed with option `EventsPollingOption::Manual`. + */ + void pollEvents(std::chrono::milliseconds timeout) + { + _pollable->poll(convertMsDurationToInt(timeout)); + } + + /** + * Fetch matadata from a available broker. + * Note: the Metadata response information may trigger a re-join if any subscribed topic has changed partition count or existence state. + */ + Optional fetchBrokerMetadata(const std::string& topic, + std::chrono::milliseconds timeout = std::chrono::milliseconds(DEFAULT_METADATA_TIMEOUT_MS), + bool disableErrorLogging = false); + + template + void doLog(int level, const char* filename, int lineno, const char* format, Args... args) const + { + const auto& logger = _logger ? _logger : Global<>::logger; + if (level >= 0 && level <= _logLevel && logger) + { + LogBuffer logBuffer; + logBuffer.print("%s ", name().c_str()).print(format, args...); + logger(level, filename, lineno, logBuffer.c_str()); + } + } + + void doLog(int level, const char* filename, int lineno, const char* msg) const + { + doLog(level, filename, lineno, "%s", msg); + } + +#define KAFKA_API_DO_LOG(lvl, ...) doLog(lvl, __FILE__, __LINE__, ##__VA_ARGS__) + + template + static void doGlobalLog(int level, const char* filename, int lineno, const char* format, Args... args) + { + if (!Global<>::logger) return; + + LogBuffer logBuffer; + logBuffer.print(format, args...); + Global<>::logger(level, filename, lineno, logBuffer.c_str()); + } + static void doGlobalLog(int level, const char* filename, int lineno, const char* msg) + { + doGlobalLog(level, filename, lineno, "%s", msg); + } + +/** + * Log for kafka clients, with the callback which `setGlobalLogger` assigned. + * + * E.g, + * KAFKA_API_LOG(Log::Level::Err, "something wrong happened! %s", detailedInfo.c_str()); + */ +#define KAFKA_API_LOG(lvl, ...) KafkaClient::doGlobalLog(lvl, __FILE__, __LINE__, ##__VA_ARGS__) + +#if COMPILER_SUPPORTS_CPP_17 + static constexpr int DEFAULT_METADATA_TIMEOUT_MS = 10000; +#else + enum { DEFAULT_METADATA_TIMEOUT_MS = 10000 }; +#endif + +protected: + // There're 3 derived classes: KafkaConsumer, KafkaProducer, AdminClient + enum class ClientType { KafkaConsumer, KafkaProducer, AdminClient }; + + using ConfigCallbacksRegister = std::function; + + KafkaClient(ClientType clientType, + const Properties& properties, + const ConfigCallbacksRegister& extraConfigRegister = ConfigCallbacksRegister{}, + EventsPollingOption eventsPollingOption = EventsPollingOption::Auto); + + rd_kafka_t* getClientHandle() const { return _rk.get(); } + + static const KafkaClient& kafkaClient(const rd_kafka_t* rk) { return *static_cast(rd_kafka_opaque(rk)); } + static KafkaClient& kafkaClient(rd_kafka_t* rk) { return *static_cast(rd_kafka_opaque(rk)); } + + static constexpr int TIMEOUT_INFINITE = -1; + + static int convertMsDurationToInt(std::chrono::milliseconds ms) + { + return ms > std::chrono::milliseconds(INT_MAX) ? TIMEOUT_INFINITE : static_cast(ms.count()); + } + + // Show whether it's using automatical events polling + bool isWithAutoEventsPolling() const { return _eventsPollingOption == EventsPollingOption::Auto; } + + // Buffer size for single line logging + static const constexpr int LOG_BUFFER_SIZE = 1024; + + // Global logger + template + struct Global + { + static Logger logger; + static std::once_flag initOnce; + }; + + // Validate properties (and fix it if necesary) + static Properties validateAndReformProperties(const Properties& properties); + + // To avoid double-close + bool _opened = false; + + // Accepted properties + Properties _properties; + +#if COMPILER_SUPPORTS_CPP_17 + static constexpr int EVENT_POLLING_INTERVAL_MS = 100; +#else + enum { EVENT_POLLING_INTERVAL_MS = 100 }; +#endif + +private: + std::string _clientId; + std::string _clientName; + std::atomic _logLevel = {Log::Level::Notice}; + Logger _logger; + StatsCallback _statsCb; + ErrorCallback _errorCb; + rd_kafka_unique_ptr _rk; + EventsPollingOption _eventsPollingOption; + + static std::string getClientTypeString(ClientType type) + { + return (type == ClientType::KafkaConsumer ? "KafkaConsumer" + : (type == ClientType::KafkaProducer ? "KafkaProducer" : "AdminClient")); + } + + // Log callback (for librdkafka) + static void logCallback(const rd_kafka_t* rk, int level, const char* fac, const char* buf); + + // Statistics callback (for librdkafka) + static int statsCallback(rd_kafka_t* rk, char* jsonStrBuf, size_t jsonStrLen, void* opaque); + + // Error callback (for librdkafka) + static void errorCallback(rd_kafka_t* rk, int err, const char* reason, void* opaque); + + // Log callback (for class instance) + void onLog(int level, const char* fac, const char* buf) const; + + // Stats callback (for class instance) + void onStats(const std::string& jsonString); + + // Error callback (for class instance) + void onError(const Error& error); + + static const constexpr char* BOOTSTRAP_SERVERS = "bootstrap.servers"; + static const constexpr char* CLIENT_ID = "client.id"; + static const constexpr char* LOG_LEVEL = "log_level"; + +protected: + struct Pollable + { + virtual ~Pollable() = default; + virtual void poll(int timeoutMs) = 0; + }; + + class PollableCallback: public Pollable + { + public: + using Callback = std::function; + + explicit PollableCallback(Callback cb): _cb(std::move(cb)) {} + + void poll(int timeoutMs) override { _cb(timeoutMs); } + + private: + const Callback _cb; + }; + + class PollThread + { + public: + explicit PollThread(Pollable& pollable) + : _running(true), _thread(keepPolling, std::ref(_running), std::ref(pollable)) + { + } + + ~PollThread() + { + _running = false; + + if (_thread.joinable()) _thread.join(); + } + + private: + static void keepPolling(std::atomic_bool& running, Pollable& pollable) + { + while (running.load()) + { + pollable.poll(CALLBACK_POLLING_INTERVAL_MS); + } + } + + static constexpr int CALLBACK_POLLING_INTERVAL_MS = 10; + + std::atomic_bool _running; + std::thread _thread; + }; + + void startBackgroundPollingIfNecessary(const PollableCallback::Callback& pollableCallback) + { + _pollable = std::make_unique(pollableCallback); + + if (isWithAutoEventsPolling()) _pollThread = std::make_unique(*_pollable); + } + + void stopBackgroundPollingIfNecessary() + { + _pollThread.reset(); // Join the polling thread (in case it's running) + + _pollable.reset(); + } + +private: + std::unique_ptr _pollable; + std::unique_ptr _pollThread; +}; + +template +Logger KafkaClient::Global::logger; + +template +std::once_flag KafkaClient::Global::initOnce; + +inline +KafkaClient::KafkaClient(ClientType clientType, + const Properties& properties, + const ConfigCallbacksRegister& extraConfigRegister, + EventsPollingOption eventsPollingOption) + : _eventsPollingOption(eventsPollingOption) +{ + static const std::set PRIVATE_PROPERTY_KEYS = { "max.poll.records" }; + + // Save clientID + if (auto clientId = properties.getProperty(CLIENT_ID)) + { + _clientId = *clientId; + _clientName = getClientTypeString(clientType) + "[" + _clientId + "]"; + } + + // Init global logger + std::call_once(Global<>::initOnce, [](){ Global<>::logger = DefaultLogger; }); + + // Save LogLevel + if (auto logLevel = properties.getProperty(LOG_LEVEL)) + { + try + { + _logLevel = std::stoi(*logLevel); + } + catch (const std::exception& e) + { + KAFKA_THROW_ERROR(Error(RD_KAFKA_RESP_ERR__INVALID_ARG, std::string("Invalid log_level[").append(*logLevel).append("], which must be an number!").append(e.what()))); + } + + if (_logLevel < Log::Level::Emerg || _logLevel > Log::Level::Debug) + { + KAFKA_THROW_ERROR(Error(RD_KAFKA_RESP_ERR__INVALID_ARG, std::string("Invalid log_level[").append(*logLevel).append("], which must be a value between 0 and 7!"))); + } + } + + LogBuffer errInfo; + + auto rk_conf = rd_kafka_conf_unique_ptr(rd_kafka_conf_new()); + + for (const auto& prop: properties.map()) + { + // Those private properties are only available for `C++ wrapper`, not for librdkafka + if (PRIVATE_PROPERTY_KEYS.count(prop.first)) + { + _properties.put(prop.first, prop.second); + continue; + } + + rd_kafka_conf_res_t result = rd_kafka_conf_set(rk_conf.get(), prop.first.c_str(), prop.second.c_str(), errInfo.str(), errInfo.capacity()); + if (result == RD_KAFKA_CONF_OK) + { + _properties.put(prop.first, prop.second); + } + else + { + KAFKA_API_DO_LOG(Log::Level::Err, "failed to be initialized with property[%s:%s], result[%d]", prop.first.c_str(), prop.second.c_str(), result); + } + } + + // Save KafkaClient's raw pointer to the "opaque" field, thus we could fetch it later (for kinds of callbacks) + rd_kafka_conf_set_opaque(rk_conf.get(), this); + + // Log Callback + rd_kafka_conf_set_log_cb(rk_conf.get(), KafkaClient::logCallback); + + // Statistics Callback + rd_kafka_conf_set_stats_cb(rk_conf.get(), KafkaClient::statsCallback); + + // Error Callback + rd_kafka_conf_set_error_cb(rk_conf.get(), KafkaClient::errorCallback); + + // Other Callbacks + if (extraConfigRegister) extraConfigRegister(rk_conf.get()); + + // Set client handler + _rk.reset(rd_kafka_new((clientType == ClientType::KafkaConsumer ? RD_KAFKA_CONSUMER : RD_KAFKA_PRODUCER), + rk_conf.release(), // rk_conf's ownship would be transferred to rk, after the "rd_kafka_new()" call + errInfo.clear().str(), + errInfo.capacity())); + KAFKA_THROW_IF_WITH_ERROR(Error(rd_kafka_last_error())); + + // Add brokers + auto brokers = properties.getProperty(BOOTSTRAP_SERVERS); + if (rd_kafka_brokers_add(getClientHandle(), brokers->c_str()) == 0) + { + KAFKA_THROW_ERROR(Error(RD_KAFKA_RESP_ERR__INVALID_ARG,\ + "No broker could be added successfully, BOOTSTRAP_SERVERS=[" + *brokers + "]")); + } + + _opened = true; +} + +inline Properties +KafkaClient::validateAndReformProperties(const Properties& properties) +{ + auto newProperties = properties; + + // BOOTSTRAP_SERVERS property is mandatory + if (!newProperties.getProperty(BOOTSTRAP_SERVERS)) + { + KAFKA_THROW_ERROR(Error(RD_KAFKA_RESP_ERR__INVALID_ARG,\ + "Validation failed! With no property [" + std::string(BOOTSTRAP_SERVERS) + "]")); + } + + // If no "client.id" configured, generate a random one for user + if (!newProperties.getProperty(CLIENT_ID)) + { + newProperties.put(CLIENT_ID, utility::getRandomString()); + } + + // If no "log_level" configured, use Log::Level::Notice as default + if (!newProperties.getProperty(LOG_LEVEL)) + { + newProperties.put(LOG_LEVEL, std::to_string(static_cast(Log::Level::Notice))); + } + + return newProperties; +} + +inline Optional +KafkaClient::getProperty(const std::string& name) const +{ + // Find it in pre-saved properties + if (auto property = _properties.getProperty(name)) return *property; + + constexpr int DEFAULT_BUF_SIZE = 512; + + const rd_kafka_conf_t* conf = rd_kafka_conf(getClientHandle()); + + std::vector valueBuf(DEFAULT_BUF_SIZE); + std::size_t valueSize = valueBuf.size(); + + // Try with a default buf size. If could not find the property, return immediately. + if (rd_kafka_conf_get(conf, name.c_str(), valueBuf.data(), &valueSize) != RD_KAFKA_CONF_OK) return Optional{}; + + // If the default buf size is not big enough, retry with a larger one + if (valueSize > valueBuf.size()) + { + valueBuf.resize(valueSize); + [[maybe_unused]] rd_kafka_conf_res_t result = rd_kafka_conf_get(conf, name.c_str(), valueBuf.data(), &valueSize); + assert(result == RD_KAFKA_CONF_OK); + } + + return std::string(valueBuf.data()); +} + +inline void +KafkaClient::setLogLevel(int level) +{ + _logLevel = level < Log::Level::Emerg ? Log::Level::Emerg : (level > Log::Level::Debug ? Log::Level::Debug : level); + rd_kafka_set_log_level(getClientHandle(), _logLevel); +} + +inline void +KafkaClient::onLog(int level, const char* fac, const char* buf) const +{ + doLog(level, "LIBRDKAFKA", 0, "%s | %s", fac, buf); // The log is coming from librdkafka +} + +inline void +KafkaClient::logCallback(const rd_kafka_t* rk, int level, const char* fac, const char* buf) +{ + kafkaClient(rk).onLog(level, fac, buf); +} + +inline void +KafkaClient::onStats(const std::string& jsonString) +{ + if (_statsCb) _statsCb(jsonString); +} + +inline int +KafkaClient::statsCallback(rd_kafka_t* rk, char* jsonStrBuf, size_t jsonStrLen, void* /*opaque*/) +{ + std::string stats(jsonStrBuf, jsonStrBuf+jsonStrLen); + kafkaClient(rk).onStats(stats); + return 0; +} + +inline void +KafkaClient::onError(const Error& error) +{ + if (_errorCb) _errorCb(error); +} + +inline void +KafkaClient::errorCallback(rd_kafka_t* rk, int err, const char* reason, void* /*opaque*/) +{ + auto respErr = static_cast(err); + + Error error; + if (respErr != RD_KAFKA_RESP_ERR__FATAL) + { + error = Error{respErr, reason}; + } + else + { + LogBuffer errInfo; + respErr = rd_kafka_fatal_error(rk, errInfo.str(), errInfo.capacity()); + error = Error{respErr, errInfo.c_str(), true}; + } + + kafkaClient(rk).onError(error); +} + +inline Optional +KafkaClient::fetchBrokerMetadata(const std::string& topic, std::chrono::milliseconds timeout, bool disableErrorLogging) +{ + const rd_kafka_metadata_t* rk_metadata = nullptr; + // Here the input parameter for `all_topics` is `true`, since we want the `cgrp_update` + rd_kafka_resp_err_t err = rd_kafka_metadata(getClientHandle(), true, nullptr, &rk_metadata, convertMsDurationToInt(timeout)); + + auto guard = rd_kafka_metadata_unique_ptr(rk_metadata); + + if (err != RD_KAFKA_RESP_ERR_NO_ERROR) + { + if (!disableErrorLogging) + { + KAFKA_API_DO_LOG(Log::Level::Err, "failed to get BrokerMetadata! error[%s]", rd_kafka_err2str(err)); + } + return Optional{}; + } + + const rd_kafka_metadata_topic* metadata_topic = nullptr; + for (int i = 0; i < rk_metadata->topic_cnt; ++i) + { + if (rk_metadata->topics[i].topic == topic) + { + metadata_topic = &rk_metadata->topics[i]; + break; + } + } + + if (!metadata_topic || metadata_topic->err) + { + if (!disableErrorLogging) + { + if (!metadata_topic) + { + KAFKA_API_DO_LOG(Log::Level::Err, "failed to find BrokerMetadata for topic[%s]", topic.c_str()); + } + else + { + KAFKA_API_DO_LOG(Log::Level::Err, "failed to get BrokerMetadata for topic[%s]! error[%s]", topic.c_str(), rd_kafka_err2str(metadata_topic->err)); + } + } + return Optional{}; + } + + // Construct the BrokerMetadata + BrokerMetadata metadata(metadata_topic->topic); + metadata.setOrigNodeName(rk_metadata->orig_broker_name ? std::string(rk_metadata->orig_broker_name) : ""); + + for (int i = 0; i < rk_metadata->broker_cnt; ++i) + { + metadata.addNode(rk_metadata->brokers[i].id, rk_metadata->brokers[i].host, rk_metadata->brokers[i].port); + } + + for (int i = 0; i < metadata_topic->partition_cnt; ++i) + { + const rd_kafka_metadata_partition& metadata_partition = metadata_topic->partitions[i]; + + Partition partition = metadata_partition.id; + + if (metadata_partition.err != 0) + { + if (!disableErrorLogging) + { + KAFKA_API_DO_LOG(Log::Level::Err, "got error[%s] while constructing BrokerMetadata for topic[%s]-partition[%d]", rd_kafka_err2str(metadata_partition.err), topic.c_str(), partition); + } + + continue; + } + + BrokerMetadata::PartitionInfo partitionInfo(metadata_partition.leader); + + for (int j = 0; j < metadata_partition.replica_cnt; ++j) + { + partitionInfo.addReplica(metadata_partition.replicas[j]); + } + + for (int j = 0; j < metadata_partition.isr_cnt; ++j) + { + partitionInfo.addInSyncReplica(metadata_partition.isrs[j]); + } + + metadata.addPartitionInfo(partition, partitionInfo); + } + + return metadata; +} + + +} } // end of KAFKA_API::clients + diff --git a/modern-cpp-kafka/include/kafka/KafkaConsumer.h b/modern-cpp-kafka/include/kafka/KafkaConsumer.h new file mode 100644 index 00000000..9f580d08 --- /dev/null +++ b/modern-cpp-kafka/include/kafka/KafkaConsumer.h @@ -0,0 +1,1051 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + + +namespace KAFKA_API { namespace clients { + +/** + * KafkaConsumer class. + */ +class KafkaConsumer: public KafkaClient +{ +public: + // Default value for property "max.poll.records" (which is same with Java API) + static const constexpr char* DEFAULT_MAX_POLL_RECORDS_VALUE = "500"; + + /** + * The constructor for KafkaConsumer. + * + * Options: + * - EventsPollingOption::Auto (default) : An internal thread would be started for OffsetCommit callbacks handling. + * - EventsPollingOption::Maunal : User have to call the member function `pollEvents()` to trigger OffsetCommit callbacks. + * + * Throws KafkaException with errors: + * - RD_KAFKA_RESP_ERR__INVALID_ARG : Invalid BOOTSTRAP_SERVERS property + * - RD_KAFKA_RESP_ERR__CRIT_SYS_RESOURCE: Fail to create internal threads + */ + explicit KafkaConsumer(const Properties& properties, + EventsPollingOption eventsPollingOption = EventsPollingOption::Auto); + + /** + * The destructor for KafkaConsumer. + */ + ~KafkaConsumer() override { if (_opened) close(); } + + /** + * Close the consumer, waiting for any needed cleanup. + */ + void close(); + + /** + * To get group ID. + */ + std::string getGroupId() const { return _groupId; } + + /** + * To set group ID. The group ID is mandatory for a Consumer. + */ + void setGroupId(const std::string& id) { _groupId = id; } + + /** + * Subscribe to the given list of topics to get dynamically assigned partitions. + * An exception would be thrown if assign is called previously (without a subsequent call to unsubscribe()) + */ + void subscribe(const Topics& topics, + consumer::RebalanceCallback rebalanceCallback = consumer::NullRebalanceCallback, + std::chrono::milliseconds timeout = std::chrono::milliseconds(DEFAULT_SUBSCRIBE_TIMEOUT_MS)); + /** + * Get the current subscription. + */ + Topics subscription() const; + + /** + * Unsubscribe from topics currently subscribed. + */ + void unsubscribe(std::chrono::milliseconds timeout = std::chrono::milliseconds(DEFAULT_UNSUBSCRIBE_TIMEOUT_MS)); + + /** + * Manually assign a list of partitions to this consumer. + * An exception would be thrown if subscribe is called previously (without a subsequent call to unsubscribe()) + */ + void assign(const TopicPartitions& topicPartitions); + + /** + * Get the set of partitions currently assigned to this consumer. + */ + TopicPartitions assignment() const; + + // Seek & Position + /** + * Overrides the fetch offsets that the consumer will use on the next poll(timeout). + * If this API is invoked for the same partition more than once, the latest offset will be used on the next poll(). + * Throws KafkaException with errors: + * - RD_KAFKA_RESP_ERR__TIMED_OUT: Operation timed out + * - RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION: Invalid partition + * - RD_KAFKA_RESP_ERR__STATE: Invalid broker state + */ + void seek(const TopicPartition& topicPartition, Offset offset, std::chrono::milliseconds timeout = std::chrono::milliseconds(DEFAULT_SEEK_TIMEOUT_MS)); + + /** + * Seek to the first offset for each of the given partitions. + * This function evaluates lazily, seeking to the first offset in all partitions only when poll(long) or position(TopicPartition) are called. + * If no partitions are provided, seek to the first offset for all of the currently assigned partitions. + * Throws KafkaException with errors: + * - RD_KAFKA_RESP_ERR__TIMED_OUT: Operation timed out + * - RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION: Invalid partition + * - RD_KAFKA_RESP_ERR__STATE: Invalid broker state + */ + void seekToBeginning(const TopicPartitions& topicPartitions, + std::chrono::milliseconds timeout = std::chrono::milliseconds(DEFAULT_SEEK_TIMEOUT_MS)) { seekToBeginningOrEnd(topicPartitions, true, timeout); } + void seekToBeginning(std::chrono::milliseconds timeout = std::chrono::milliseconds(DEFAULT_SEEK_TIMEOUT_MS)) { seekToBeginningOrEnd(_assignment, true, timeout); } + + /** + * Seek to the last offset for each of the given partitions. + * This function evaluates lazily, seeking to the final offset in all partitions only when poll(long) or position(TopicPartition) are called. + * If no partitions are provided, seek to the first offset for all of the currently assigned partitions. + * Throws KafkaException with errors: + * - RD_KAFKA_RESP_ERR__TIMED_OUT: Operation timed out + * - RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION: Invalid partition + * - RD_KAFKA_RESP_ERR__STATE: Invalid broker state + */ + void seekToEnd(const TopicPartitions& topicPartitions, + std::chrono::milliseconds timeout = std::chrono::milliseconds(DEFAULT_SEEK_TIMEOUT_MS)) { seekToBeginningOrEnd(topicPartitions, false, timeout); } + void seekToEnd(std::chrono::milliseconds timeout = std::chrono::milliseconds(DEFAULT_SEEK_TIMEOUT_MS)) { seekToBeginningOrEnd(_assignment, false, timeout); } + + /** + * Get the offset of the next record that will be fetched (if a record with that offset exists). + */ + Offset position(const TopicPartition& topicPartition) const; + + /** + * Get the first offset for the given partitions. + * This method does not change the current consumer position of the partitions. + * Throws KafkaException with errors: + * - RD_KAFKA_RESP_ERR__FAIL: Generic failure + */ + std::map beginningOffsets(const TopicPartitions& topicPartitions) const { return getOffsets(topicPartitions, true); } + + /** + * Get the last offset for the given partitions. The last offset of a partition is the offset of the upcoming message, i.e. the offset of the last available message + 1. + * This method does not change the current consumer position of the partitions. + * Throws KafkaException with errors: + * - RD_KAFKA_RESP_ERR__FAIL: Generic failure + */ + std::map endOffsets(const TopicPartitions& topicPartitions) const { return getOffsets(topicPartitions, false); } + + /** + * Get the offsets for the given partitions by time-point. + * Throws KafkaException with errors: + * - RD_KAFKA_RESP_ERR__TIMED_OUT: Not all offsets could be fetched in time. + * - RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION: All partitions are unknown. + * - RD_KAFKA_RESP_ERR_LEADER_NOT_AVAILABLE: Unable to query leaders from the given partitions. + */ + std::map offsetsForTime(const TopicPartitions& topicPartitions, + std::chrono::time_point timepoint, + std::chrono::milliseconds timeout = std::chrono::milliseconds(DEFAULT_QUERY_TIMEOUT_MS)) const; + + /** + * Commit offsets returned on the last poll() for all the subscribed list of topics and partitions. + */ + void commitSync(); + + /** + * Commit the specified offsets for the specified records + */ + void commitSync(const consumer::ConsumerRecord& record); + + /** + * Commit the specified offsets for the specified list of topics and partitions. + */ + void commitSync(const TopicPartitionOffsets& topicPartitionOffsets); + + /** + * Commit offsets returned on the last poll() for all the subscribed list of topics and partition. + * Note: If a callback is provided, it's guaranteed to be triggered (before closing the consumer). + */ + void commitAsync(const consumer::OffsetCommitCallback& offsetCommitCallback = consumer::NullOffsetCommitCallback); + + /** + * Commit the specified offsets for the specified records + * Note: If a callback is provided, it's guaranteed to be triggered (before closing the consumer). + */ + void commitAsync(const consumer::ConsumerRecord& record, const consumer::OffsetCommitCallback& offsetCommitCallback = consumer::NullOffsetCommitCallback); + + /** + * Commit the specified offsets for the specified list of topics and partitions to Kafka. + * Note: If a callback is provided, it's guaranteed to be triggered (before closing the consumer). + */ + void commitAsync(const TopicPartitionOffsets& topicPartitionOffsets, const consumer::OffsetCommitCallback& offsetCommitCallback = consumer::NullOffsetCommitCallback); + + /** + * Get the last committed offset for the given partition (whether the commit happened by this process or another).This offset will be used as the position for the consumer in the event of a failure. + * This call will block to do a remote call to get the latest committed offsets from the server. + * Throws KafkaException with errors: + * - RD_KAFKA_RESP_ERR__INVALID_ARG: Invalid partition + */ + Offset committed(const TopicPartition& topicPartition); + + /** + * Fetch data for the topics or partitions specified using one of the subscribe/assign APIs. + * Returns the polled records. + * Note: 1) The result could be fetched through ConsumerRecord (with member function `error`). + * 2) Make sure the `ConsumerRecord` be destructed before the `KafkaConsumer.close()`. + * Throws KafkaException with errors: + * - RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION: Unknow partition + */ + std::vector poll(std::chrono::milliseconds timeout); + + /** + * Fetch data for the topics or partitions specified using one of the subscribe/assign APIs. + * Returns the number of polled records (which have been saved into parameter `output`). + * Note: 1) The result could be fetched through ConsumerRecord (with member function `error`). + * 2) Make sure the `ConsumerRecord` be destructed before the `KafkaConsumer.close()`. + * Throws KafkaException with errors: + * - RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION: Unknow partition + */ + std::size_t poll(std::chrono::milliseconds timeout, std::vector& output); + + /** + * Suspend fetching from the requested partitions. Future calls to poll() will not return any records from these partitions until they have been resumed using resume(). + * Note: 1) After pausing, the application still need to call `poll()` at regular intervals. + * 2) This method does not affect partition subscription/assignment (i.e, pause fetching from partitions would not trigger a rebalance, since the consumer is still alive). + * 3) If none of the provided partitions is assigned to this consumer, an exception would be thrown. + * Throws KafkaException with error: + * - RD_KAFKA_RESP_ERR__INVALID_ARG: Invalid arguments + */ + void pause(const TopicPartitions& topicPartitions); + + /** + * Suspend fetching from all assigned partitions. Future calls to poll() will not return any records until they have been resumed using resume(). + * Note: This method does not affect partition subscription/assignment. + */ + void pause(); + + /** + * Resume specified partitions which have been paused with pause(). New calls to poll() will return records from these partitions if there are any to be fetched. + * Note: If the partitions were not previously paused, this method is a no-op. + */ + void resume(const TopicPartitions& topicPartitions); + + /** + * Resume all partitions which have been paused with pause(). New calls to poll() will return records from these partitions if there are any to be fetched. + */ + void resume(); + + /** + * Return the current group metadata associated with this consumer. + */ + consumer::ConsumerGroupMetadata groupMetadata(); + +private: + static const constexpr char* ENABLE_AUTO_OFFSET_STORE = "enable.auto.offset.store"; + static const constexpr char* AUTO_COMMIT_INTERVAL_MS = "auto.commit.interval.ms"; + +#if COMPILER_SUPPORTS_CPP_17 + static constexpr int DEFAULT_SUBSCRIBE_TIMEOUT_MS = 30000; + static constexpr int DEFAULT_UNSUBSCRIBE_TIMEOUT_MS = 10000; + static constexpr int DEFAULT_QUERY_TIMEOUT_MS = 10000; + static constexpr int DEFAULT_SEEK_TIMEOUT_MS = 10000; + static constexpr int SEEK_RETRY_INTERVAL_MS = 5000; +#else + enum { DEFAULT_SUBSCRIBE_TIMEOUT_MS = 30000 }; + enum { DEFAULT_UNSUBSCRIBE_TIMEOUT_MS = 10000 }; + enum { DEFAULT_QUERY_TIMEOUT_MS = 10000 }; + enum { DEFAULT_SEEK_TIMEOUT_MS = 10000 }; + enum { SEEK_RETRY_INTERVAL_MS = 5000 }; +#endif + + enum class CommitType { Sync, Async }; + void commit(const TopicPartitionOffsets& topicPartitionOffsets, CommitType type); + + // Offset Commit Callback (for librdkafka) + static void offsetCommitCallback(rd_kafka_t* rk, rd_kafka_resp_err_t err, rd_kafka_topic_partition_list_t* rk_tpos, void* opaque); + + // Validate properties (and fix it if necesary) + static Properties validateAndReformProperties(Properties properties); + + void commitStoredOffsetsIfNecessary(CommitType type); + void storeOffsetsIfNecessary(const std::vector& records); + + void seekToBeginningOrEnd(const TopicPartitions& topicPartitions, bool toBeginning, std::chrono::milliseconds timeout); + std::map getOffsets(const TopicPartitions& topicPartitions, bool atBeginning) const; + + enum class PartitionsRebalanceEvent { Assign, Revoke, IncrementalAssign, IncrementalUnassign }; + void changeAssignment(PartitionsRebalanceEvent event, const TopicPartitions& tps); + + std::string _groupId; + + std::size_t _maxPollRecords = 500; // From "max.poll.records" property, and here is the default for batch-poll + bool _enableAutoCommit = false; // From "enable.auto.commit" property + + rd_kafka_queue_unique_ptr _rk_queue; + + // Save assignment info (from "assign()" call or rebalance callback) locally, to accelerate seeking procedure + TopicPartitions _assignment; + // Assignment from user's input, -- by calling "assign()" + TopicPartitions _userAssignment; + // Subscription from user's input, -- by calling "subscribe()" + Topics _userSubscription; + + enum class PendingEvent { PartitionsAssignment, PartitionsRevocation }; + Optional _pendingEvent; + + // Identify whether the "partition.assignment.strategy" is "cooperative-sticky" + Optional _cooperativeEnabled; + bool isCooperativeEnabled() const { return _cooperativeEnabled && *_cooperativeEnabled; } + + // The offsets to store (and commit later) + std::map _offsetsToStore; + + // Register Callbacks for rd_kafka_conf_t + static void registerConfigCallbacks(rd_kafka_conf_t* conf); + + void pollMessages(int timeoutMs, std::vector& output); + + enum class PauseOrResumeOperation { Pause, Resume }; + void pauseOrResumePartitions(const TopicPartitions& topicPartitions, PauseOrResumeOperation op); + + // Rebalance Callback (for librdkafka) + static void rebalanceCallback(rd_kafka_t* rk, rd_kafka_resp_err_t err, rd_kafka_topic_partition_list_t* partitions, void* opaque); + // Rebalance Callback (for class instance) + void onRebalance(rd_kafka_resp_err_t err, rd_kafka_topic_partition_list_t* rk_partitions); + + consumer::RebalanceCallback _rebalanceCb; + + rd_kafka_queue_t* getCommitCbQueue() { return _rk_commit_cb_queue.get(); } + + rd_kafka_queue_unique_ptr _rk_commit_cb_queue; + + void pollCallbacks(int timeoutMs) + { + rd_kafka_queue_t* queue = getCommitCbQueue(); + rd_kafka_queue_poll_callback(queue, timeoutMs); + } +}; + + +// Validate properties (and fix it if necesary) +inline Properties +KafkaConsumer::validateAndReformProperties(Properties properties) +{ + // Don't pass the "max.poll.records" property to librdkafka + properties.remove(consumer::Config::MAX_POLL_RECORDS); + + // Let the base class validate first + auto newProperties = KafkaClient::validateAndReformProperties(properties); + + // If no "group.id" configured, generate a random one for user + if (!newProperties.getProperty(consumer::Config::GROUP_ID)) + { + newProperties.put(consumer::Config::GROUP_ID, utility::getRandomString()); + } + + // Disable the internal auto-commit from librdkafka, since we want to customize the behavior + newProperties.put(consumer::Config::ENABLE_AUTO_COMMIT, "false"); + newProperties.put(AUTO_COMMIT_INTERVAL_MS, "0"); + newProperties.put(ENABLE_AUTO_OFFSET_STORE, "true"); + + return newProperties; +} + +// Register Callbacks for rd_kafka_conf_t +inline void +KafkaConsumer::registerConfigCallbacks(rd_kafka_conf_t* conf) +{ + // Rebalance Callback + // would turn off librdkafka's automatic partition assignment/revocation + rd_kafka_conf_set_rebalance_cb(conf, KafkaConsumer::rebalanceCallback); +} + +inline +KafkaConsumer::KafkaConsumer(const Properties &properties, EventsPollingOption eventsPollingOption) + : KafkaClient(ClientType::KafkaConsumer, + validateAndReformProperties(properties), + registerConfigCallbacks, + eventsPollingOption) +{ + // Pick up the "max.poll.records" property + if (auto maxPollRecordsProperty = properties.getProperty(consumer::Config::MAX_POLL_RECORDS)) + { + const std::string maxPollRecords = *maxPollRecordsProperty; + _maxPollRecords = static_cast(std::stoi(maxPollRecords)); + } + _properties.put(consumer::Config::MAX_POLL_RECORDS, std::to_string(_maxPollRecords)); + + // Pick up the "enable.auto.commit" property + if (auto enableAutoCommitProperty = properties.getProperty(consumer::Config::ENABLE_AUTO_COMMIT)) + { + const std::string enableAutoCommit = *enableAutoCommitProperty; + + auto isTrue = [](const std::string& str) { return str == "1" || str == "true"; }; + auto isFalse = [](const std::string& str) { return str == "0" || str == "false"; }; + + if (!isTrue(enableAutoCommit) && !isFalse(enableAutoCommit)) + { + KAFKA_THROW_ERROR(Error(RD_KAFKA_RESP_ERR__INVALID_ARG, std::string("Invalid property[enable.auto.commit=").append(enableAutoCommit).append("], which MUST be true(1) or false(0)!"))); + } + + _enableAutoCommit = isTrue(enableAutoCommit); + } + _properties.put(consumer::Config::ENABLE_AUTO_COMMIT, (_enableAutoCommit ? "true" : "false")); + + // Fetch groupId from reformed configuration + auto groupId = _properties.getProperty(consumer::Config::GROUP_ID); + assert(groupId); + setGroupId(*groupId); + + // Redirect the reply queue (to the client group queue) + Error result{ rd_kafka_poll_set_consumer(getClientHandle()) }; + KAFKA_THROW_IF_WITH_ERROR(result); + + // Initialize message-fetching queue + _rk_queue.reset(rd_kafka_queue_get_consumer(getClientHandle())); + + // Initialize commit-callback queue + _rk_commit_cb_queue.reset(rd_kafka_queue_new(getClientHandle())); + + // Start background polling (if needed) + startBackgroundPollingIfNecessary([this](int timeoutMs){ pollCallbacks(timeoutMs); }); + + const auto propsStr = KafkaClient::properties().toString(); + KAFKA_API_DO_LOG(Log::Level::Notice, "initialized with properties[%s]", propsStr.c_str()); +} + +inline void +KafkaConsumer::close() +{ + _opened = false; + + stopBackgroundPollingIfNecessary(); + + try + { + // Commit the offsets for these messages which had been polled last time (for `enable.auto.commit=true` case.) + commitStoredOffsetsIfNecessary(CommitType::Sync); + } + catch (const KafkaException& e) + { + KAFKA_API_DO_LOG(Log::Level::Err, "met error[%s] while closing", e.what()); + } + + rd_kafka_consumer_close(getClientHandle()); + + while (rd_kafka_outq_len(getClientHandle())) + { + rd_kafka_poll(getClientHandle(), KafkaClient::TIMEOUT_INFINITE); + } + + rd_kafka_queue_t* queue = getCommitCbQueue(); + while (rd_kafka_queue_length(queue)) + { + rd_kafka_queue_poll_callback(queue, TIMEOUT_INFINITE); + } + + KAFKA_API_DO_LOG(Log::Level::Notice, "closed"); +} + + +// Subscription +inline void +KafkaConsumer::subscribe(const Topics& topics, consumer::RebalanceCallback rebalanceCallback, std::chrono::milliseconds timeout) +{ + if (!_userAssignment.empty()) + { + KAFKA_THROW_ERROR(Error(RD_KAFKA_RESP_ERR__FAIL, "Unexpected Operation! Once assign() was used, subscribe() should not be called any more!")); + } + + if (isCooperativeEnabled() && topics == _userSubscription) + { + KAFKA_API_DO_LOG(Log::Level::Info, "skip subscribe (no change since last time)"); + return; + } + + _userSubscription = topics; + + std::string topicsStr = toString(topics); + KAFKA_API_DO_LOG(Log::Level::Info, "will subscribe, topics[%s]", topicsStr.c_str()); + + _rebalanceCb = std::move(rebalanceCallback); + + auto rk_topics = rd_kafka_topic_partition_list_unique_ptr(createRkTopicPartitionList(topics)); + + Error result{ rd_kafka_subscribe(getClientHandle(), rk_topics.get()) }; + KAFKA_THROW_IF_WITH_ERROR(result); + + _pendingEvent = PendingEvent::PartitionsAssignment; + + // The rebalcance callback would be served during the time (within this thread) + for (const auto end = std::chrono::steady_clock::now() + timeout; std::chrono::steady_clock::now() < end; ) + { + rd_kafka_poll(getClientHandle(), EVENT_POLLING_INTERVAL_MS); + + if (!_pendingEvent) + { + KAFKA_API_DO_LOG(Log::Level::Notice, "subscribed, topics[%s]", topicsStr.c_str()); + return; + } + } + + _pendingEvent.reset(); + KAFKA_THROW_ERROR(Error(RD_KAFKA_RESP_ERR__TIMED_OUT, "subscribe() timed out!")); +} + +inline void +KafkaConsumer::unsubscribe(std::chrono::milliseconds timeout) +{ + if (_userSubscription.empty() && _userAssignment.empty()) + { + KAFKA_API_DO_LOG(Log::Level::Info, "skip unsubscribe (no assignment/subscription yet)"); + return; + } + + KAFKA_API_DO_LOG(Log::Level::Info, "will unsubscribe"); + + // While it's for the previous `assign(...)` + if (!_userAssignment.empty()) + { + changeAssignment(isCooperativeEnabled() ? PartitionsRebalanceEvent::IncrementalUnassign : PartitionsRebalanceEvent::Revoke, + _userAssignment); + _userAssignment.clear(); + + KAFKA_API_DO_LOG(Log::Level::Notice, "unsubscribed (the previously assigned partitions)"); + return; + } + + _userSubscription.clear(); + + Error result{ rd_kafka_unsubscribe(getClientHandle()) }; + KAFKA_THROW_IF_WITH_ERROR(result); + + _pendingEvent = PendingEvent::PartitionsRevocation; + + // The rebalance callback would be served during the time (within this thread) + for (const auto end = std::chrono::steady_clock::now() + timeout; std::chrono::steady_clock::now() < end; ) + { + rd_kafka_poll(getClientHandle(), EVENT_POLLING_INTERVAL_MS); + + if (!_pendingEvent) + { + KAFKA_API_DO_LOG(Log::Level::Notice, "unsubscribed"); + return; + } + } + + _pendingEvent.reset(); + KAFKA_THROW_ERROR(Error(RD_KAFKA_RESP_ERR__TIMED_OUT, "unsubscribe() timed out!")); +} + +inline Topics +KafkaConsumer::subscription() const +{ + rd_kafka_topic_partition_list_t* raw_topics = nullptr; + Error result{ rd_kafka_subscription(getClientHandle(), &raw_topics) }; + auto rk_topics = rd_kafka_topic_partition_list_unique_ptr(raw_topics); + + KAFKA_THROW_IF_WITH_ERROR(result); + + return getTopics(rk_topics.get()); +} + +inline void +KafkaConsumer::changeAssignment(PartitionsRebalanceEvent event, const TopicPartitions& tps) +{ + auto rk_tps = rd_kafka_topic_partition_list_unique_ptr(createRkTopicPartitionList(tps)); + + Error result; + switch (event) + { + case PartitionsRebalanceEvent::Assign: + result = Error{ rd_kafka_assign(getClientHandle(), rk_tps.get()) }; + // Update assignment + _assignment = tps; + break; + + case PartitionsRebalanceEvent::Revoke: + result = Error{ rd_kafka_assign(getClientHandle(), nullptr) }; + // Update assignment + _assignment.clear(); + break; + + case PartitionsRebalanceEvent::IncrementalAssign: + result = Error{ rd_kafka_incremental_assign(getClientHandle(), rk_tps.get()) }; + // Update assignment + for (const auto& tp: tps) + { + auto found = _assignment.find(tp); + if (found != _assignment.end()) + { + std::string tpStr = toString(tp); + KAFKA_API_DO_LOG(Log::Level::Err, "incremental assign partition[%s] has already been assigned", tpStr.c_str()); + continue; + } + _assignment.emplace(tp); + } + break; + + case PartitionsRebalanceEvent::IncrementalUnassign: + result = Error{ rd_kafka_incremental_unassign(getClientHandle(), rk_tps.get()) }; + // Update assignment + for (const auto& tp: tps) + { + auto found = _assignment.find(tp); + if (found == _assignment.end()) + { + std::string tpStr = toString(tp); + KAFKA_API_DO_LOG(Log::Level::Err, "incremental unassign partition[%s] could not be found", tpStr.c_str()); + continue; + } + _assignment.erase(found); + } + break; + } + + KAFKA_THROW_IF_WITH_ERROR(result); +} + +// Assign Topic-Partitions +inline void +KafkaConsumer::assign(const TopicPartitions& topicPartitions) +{ + if (!_userSubscription.empty()) + { + KAFKA_THROW_ERROR(Error(RD_KAFKA_RESP_ERR__FAIL, "Unexpected Operation! Once subscribe() was used, assign() should not be called any more!")); + } + + _userAssignment = topicPartitions; + + changeAssignment(isCooperativeEnabled() ? PartitionsRebalanceEvent::IncrementalAssign : PartitionsRebalanceEvent::Assign, + topicPartitions); +} + +// Assignment +inline TopicPartitions +KafkaConsumer::assignment() const +{ + rd_kafka_topic_partition_list_t* raw_tps = nullptr; + Error result{ rd_kafka_assignment(getClientHandle(), &raw_tps) }; + + auto rk_tps = rd_kafka_topic_partition_list_unique_ptr(raw_tps); + + KAFKA_THROW_IF_WITH_ERROR(result); + + return getTopicPartitions(rk_tps.get()); +} + + +// Seek & Position +inline void +KafkaConsumer::seek(const TopicPartition& topicPartition, Offset offset, std::chrono::milliseconds timeout) +{ + std::string topicPartitionStr = toString(topicPartition); + KAFKA_API_DO_LOG(Log::Level::Info, "will seek with topic-partition[%s], offset[%d]", topicPartitionStr.c_str(), offset); + + auto rkt = rd_kafka_topic_unique_ptr(rd_kafka_topic_new(getClientHandle(), topicPartition.first.c_str(), nullptr)); + if (!rkt) + { + KAFKA_THROW_ERROR(Error(rd_kafka_last_error())); + } + + const auto end = std::chrono::steady_clock::now() + timeout; + + rd_kafka_resp_err_t respErr = RD_KAFKA_RESP_ERR_NO_ERROR; + do + { + respErr = rd_kafka_seek(rkt.get(), topicPartition.second, offset, SEEK_RETRY_INTERVAL_MS); + if (respErr != RD_KAFKA_RESP_ERR__STATE && respErr != RD_KAFKA_RESP_ERR__TIMED_OUT && respErr != RD_KAFKA_RESP_ERR__OUTDATED) + { + break; + } + + // If the "seek" was called just after "assign", there's a chance that the toppar's "fetch_state" (async setted) was not ready yes. + // If that's the case, we would retry again (normally, just after a very short while, the "seek" would succeed) + std::this_thread::yield(); + } while (std::chrono::steady_clock::now() < end); + + KAFKA_THROW_IF_WITH_ERROR(Error(respErr)); + + KAFKA_API_DO_LOG(Log::Level::Info, "seeked with topic-partition[%s], offset[%d]", topicPartitionStr.c_str(), offset); +} + +inline void +KafkaConsumer::seekToBeginningOrEnd(const TopicPartitions& topicPartitions, bool toBeginning, std::chrono::milliseconds timeout) +{ + for (const auto& topicPartition: topicPartitions) + { + seek(topicPartition, (toBeginning ? RD_KAFKA_OFFSET_BEGINNING : RD_KAFKA_OFFSET_END), timeout); + } +} + +inline Offset +KafkaConsumer::position(const TopicPartition& topicPartition) const +{ + auto rk_tp = rd_kafka_topic_partition_list_unique_ptr(createRkTopicPartitionList({topicPartition})); + + Error error{ rd_kafka_position(getClientHandle(), rk_tp.get()) }; + KAFKA_THROW_IF_WITH_ERROR(error); + + return rk_tp->elems[0].offset; +} + +inline std::map +KafkaConsumer::offsetsForTime(const TopicPartitions& topicPartitions, + std::chrono::time_point timepoint, + std::chrono::milliseconds timeout) const +{ + if (topicPartitions.empty()) return {}; + + auto msSinceEpoch = std::chrono::duration_cast(timepoint.time_since_epoch()).count(); + + auto rk_tpos = rd_kafka_topic_partition_list_unique_ptr(createRkTopicPartitionList(topicPartitions)); + + for (int i = 0; i < rk_tpos->cnt; ++i) + { + rd_kafka_topic_partition_t& rk_tp = rk_tpos->elems[i]; + // Here the `msSinceEpoch` would be overridden by the offset result (after called by `rd_kafka_offsets_for_times`) + rk_tp.offset = msSinceEpoch; + } + + Error error{ rd_kafka_offsets_for_times(getClientHandle(), rk_tpos.get(), static_cast(timeout.count())) }; // NOLINT + KAFKA_THROW_IF_WITH_ERROR(error); + + auto results = getTopicPartitionOffsets(rk_tpos.get()); + + // Remove invalid results (which are not updated with an valid offset) + for (auto it = results.begin(); it != results.end(); ) + { + it = ((it->second == msSinceEpoch) ? results.erase(it) : std::next(it)); + } + + return results; +} + +inline std::map +KafkaConsumer::getOffsets(const TopicPartitions& topicPartitions, bool atBeginning) const +{ + std::map result; + + for (const auto& topicPartition: topicPartitions) + { + Offset beginning{}, end{}; + Error error{ rd_kafka_query_watermark_offsets(getClientHandle(), topicPartition.first.c_str(), topicPartition.second, &beginning, &end, 0) }; + KAFKA_THROW_IF_WITH_ERROR(error); + + result[topicPartition] = (atBeginning ? beginning : end); + } + + return result; +} + +// Commit +inline void +KafkaConsumer::commit(const TopicPartitionOffsets& topicPartitionOffsets, CommitType type) +{ + auto rk_tpos = rd_kafka_topic_partition_list_unique_ptr(topicPartitionOffsets.empty() ? nullptr : createRkTopicPartitionList(topicPartitionOffsets)); + + Error error{ rd_kafka_commit(getClientHandle(), rk_tpos.get(), type == CommitType::Async ? 1 : 0) }; + // No stored offset to commit (it might happen and should not be treated as a mistake) + if (topicPartitionOffsets.empty() && error.value() == RD_KAFKA_RESP_ERR__NO_OFFSET) + { + error = Error{}; + } + + KAFKA_THROW_IF_WITH_ERROR(error); +} + +// Fetch committed offset +inline Offset +KafkaConsumer::committed(const TopicPartition& topicPartition) +{ + auto rk_tps = rd_kafka_topic_partition_list_unique_ptr(createRkTopicPartitionList({topicPartition})); + + Error error {rd_kafka_committed(getClientHandle(), rk_tps.get(), TIMEOUT_INFINITE) }; + KAFKA_THROW_IF_WITH_ERROR(error); + + return rk_tps->elems[0].offset; +} + +// Commit stored offsets +inline void +KafkaConsumer::commitStoredOffsetsIfNecessary(CommitType type) +{ + if (_enableAutoCommit && !_offsetsToStore.empty()) + { + for (auto& o: _offsetsToStore) + { + ++o.second; + } + commit(_offsetsToStore, type); + _offsetsToStore.clear(); + } +} + +// Store offsets +inline void +KafkaConsumer::storeOffsetsIfNecessary(const std::vector& records) +{ + if (_enableAutoCommit) + { + for (const auto& record: records) + { + _offsetsToStore[TopicPartition(record.topic(), record.partition())] = record.offset(); + } + } +} + +// Fetch messages (internally used) +inline void +KafkaConsumer::pollMessages(int timeoutMs, std::vector& output) +{ + // Commit the offsets for these messages which had been polled last time (for "enable.auto.commit=true" case) + commitStoredOffsetsIfNecessary(CommitType::Async); + + // Poll messages with librdkafka's API + std::vector msgPtrArray(_maxPollRecords); + auto msgReceived = rd_kafka_consume_batch_queue(_rk_queue.get(), timeoutMs, msgPtrArray.data(), _maxPollRecords); + if (msgReceived < 0) + { + KAFKA_THROW_ERROR(Error(rd_kafka_last_error())); + } + + // Wrap messages with ConsumerRecord + output.clear(); + output.reserve(static_cast(msgReceived)); + std::for_each(msgPtrArray.begin(), msgPtrArray.begin() + msgReceived, [&output](rd_kafka_message_t* rkMsg) { output.emplace_back(rkMsg); }); + + // Store the offsets for all these polled messages (for "enable.auto.commit=true" case) + storeOffsetsIfNecessary(output); +} + +// Fetch messages (return via return value) +inline std::vector +KafkaConsumer::poll(std::chrono::milliseconds timeout) +{ + std::vector result; + poll(timeout, result); + return result; +} + +// Fetch messages (return via input parameter) +inline std::size_t +KafkaConsumer::poll(std::chrono::milliseconds timeout, std::vector& output) +{ + pollMessages(convertMsDurationToInt(timeout), output); + return output.size(); +} + +inline void +KafkaConsumer::pauseOrResumePartitions(const TopicPartitions& topicPartitions, PauseOrResumeOperation op) +{ + auto rk_tpos = rd_kafka_topic_partition_list_unique_ptr(createRkTopicPartitionList(topicPartitions)); + + Error error{ (op == PauseOrResumeOperation::Pause) ? + rd_kafka_pause_partitions(getClientHandle(), rk_tpos.get()) : rd_kafka_resume_partitions(getClientHandle(), rk_tpos.get()) }; + KAFKA_THROW_IF_WITH_ERROR(error); + + const char* opString = (op == PauseOrResumeOperation::Pause) ? "pause" : "resume"; + int cnt = 0; + for (int i = 0; i < rk_tpos->cnt; ++i) + { + const rd_kafka_topic_partition_t& rk_tp = rk_tpos->elems[i]; + if (rk_tp.err != RD_KAFKA_RESP_ERR_NO_ERROR) + { + KAFKA_API_DO_LOG(Log::Level::Err, "%s topic-partition[%s-%d] error[%s]", opString, rk_tp.topic, rk_tp.partition, rd_kafka_err2str(rk_tp.err)); + } + else + { + KAFKA_API_DO_LOG(Log::Level::Notice, "%sd topic-partition[%s-%d]", opString, rk_tp.topic, rk_tp.partition, rd_kafka_err2str(rk_tp.err)); + ++cnt; + } + } + + if (cnt == 0 && op == PauseOrResumeOperation::Pause) + { + std::string errMsg = std::string("No partition could be ") + opString + std::string("d among TopicPartitions[") + toString(topicPartitions) + std::string("]"); + KAFKA_THROW_ERROR(Error(RD_KAFKA_RESP_ERR__INVALID_ARG, errMsg)); + } +} + +inline void +KafkaConsumer::pause(const TopicPartitions& topicPartitions) +{ + pauseOrResumePartitions(topicPartitions, PauseOrResumeOperation::Pause); +} + +inline void +KafkaConsumer::pause() +{ + pause(_assignment); +} + +inline void +KafkaConsumer::resume(const TopicPartitions& topicPartitions) +{ + pauseOrResumePartitions(topicPartitions, PauseOrResumeOperation::Resume); +} + +inline void +KafkaConsumer::resume() +{ + resume(_assignment); +} + +// Rebalance Callback (for class instance) +inline void +KafkaConsumer::onRebalance(rd_kafka_resp_err_t err, rd_kafka_topic_partition_list_t* rk_partitions) +{ + TopicPartitions tps = getTopicPartitions(rk_partitions); + std::string tpsStr = toString(tps); + + if (err != RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS && err != RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS) + { + KAFKA_API_DO_LOG(Log::Level::Err, "unknown re-balance event[%d], topic-partitions[%s]", err, tpsStr.c_str()); + return; + } + + // Initialize attribute for cooperative protocol + if (!_cooperativeEnabled) + { + if (const char* protocol = rd_kafka_rebalance_protocol(getClientHandle())) + { + _cooperativeEnabled = (std::string(protocol) == "COOPERATIVE"); + } + } + + KAFKA_API_DO_LOG(Log::Level::Notice, "re-balance event triggered[%s], cooperative[%s], topic-partitions[%s]", + err == RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS ? "ASSIGN_PARTITIONS" : "REVOKE_PARTITIONS", + isCooperativeEnabled() ? "enabled" : "disabled", + tpsStr.c_str()); + + // Remove the mark for pending event + if (_pendingEvent + && ((err == RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS && *_pendingEvent == PendingEvent::PartitionsAssignment) + || (err == RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS && *_pendingEvent == PendingEvent::PartitionsRevocation))) + { + _pendingEvent.reset(); + } + + PartitionsRebalanceEvent event = (err == RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS ? + (isCooperativeEnabled() ? PartitionsRebalanceEvent::IncrementalAssign : PartitionsRebalanceEvent::Assign) + : (isCooperativeEnabled() ? PartitionsRebalanceEvent::IncrementalUnassign : PartitionsRebalanceEvent::Revoke)); + + if (err == RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS) + { + changeAssignment(event, tps); + } + + if (_rebalanceCb) + { + _rebalanceCb(err == RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS ? consumer::RebalanceEventType::PartitionsAssigned : consumer::RebalanceEventType::PartitionsRevoked, + tps); + } + + if (err == RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS) + { + changeAssignment(event, isCooperativeEnabled() ? tps : TopicPartitions{}); + } +} + +// Rebalance Callback (for librdkafka) +inline void +KafkaConsumer::rebalanceCallback(rd_kafka_t* rk, rd_kafka_resp_err_t err, rd_kafka_topic_partition_list_t* partitions, void* /* opaque */) +{ + KafkaClient& client = kafkaClient(rk); + auto& consumer = dynamic_cast(client); + consumer.onRebalance(err, partitions); +} + +// Offset Commit Callback (for librdkafka) +inline void +KafkaConsumer::offsetCommitCallback(rd_kafka_t* rk, rd_kafka_resp_err_t err, rd_kafka_topic_partition_list_t* rk_tpos, void* opaque) +{ + TopicPartitionOffsets tpos = getTopicPartitionOffsets(rk_tpos); + + if (err != RD_KAFKA_RESP_ERR_NO_ERROR) + { + auto tposStr = toString(tpos); + kafkaClient(rk).KAFKA_API_DO_LOG(Log::Level::Err, "invoked offset-commit callback. offsets[%s], result[%s]", tposStr.c_str(), rd_kafka_err2str(err)); + } + + auto* cb = static_cast(opaque); + if (cb && *cb) + { + (*cb)(tpos, Error(err)); + } + delete cb; +} + +inline consumer::ConsumerGroupMetadata +KafkaConsumer::groupMetadata() +{ + return consumer::ConsumerGroupMetadata{rd_kafka_consumer_group_metadata(getClientHandle())}; +} + + + +inline void +KafkaConsumer::commitSync() +{ + commit(TopicPartitionOffsets(), CommitType::Sync); +} + +inline void +KafkaConsumer::commitSync(const consumer::ConsumerRecord& record) +{ + TopicPartitionOffsets tpos; + // committed offset should be "current-received-offset + 1" + tpos[TopicPartition(record.topic(), record.partition())] = record.offset() + 1; + + commit(tpos, CommitType::Sync); +} + +inline void +KafkaConsumer::commitSync(const TopicPartitionOffsets& topicPartitionOffsets) +{ + commit(topicPartitionOffsets, CommitType::Sync); +} + +inline void +KafkaConsumer::commitAsync(const TopicPartitionOffsets& topicPartitionOffsets, const consumer::OffsetCommitCallback& offsetCommitCallback) +{ + auto rk_tpos = rd_kafka_topic_partition_list_unique_ptr(topicPartitionOffsets.empty() ? nullptr : createRkTopicPartitionList(topicPartitionOffsets)); + + Error error{ rd_kafka_commit_queue(getClientHandle(), + rk_tpos.get(), + getCommitCbQueue(), + &KafkaConsumer::offsetCommitCallback, + new consumer::OffsetCommitCallback(offsetCommitCallback)) }; + KAFKA_THROW_IF_WITH_ERROR(error); +} + +inline void +KafkaConsumer::commitAsync(const consumer::ConsumerRecord& record, const consumer::OffsetCommitCallback& offsetCommitCallback) +{ + TopicPartitionOffsets tpos; + // committed offset should be "current received record's offset" + 1 + tpos[TopicPartition(record.topic(), record.partition())] = record.offset() + 1; + commitAsync(tpos, offsetCommitCallback); +} + +inline void +KafkaConsumer::commitAsync(const consumer::OffsetCommitCallback& offsetCommitCallback) +{ + commitAsync(TopicPartitionOffsets(), offsetCommitCallback); +} + +} } // end of KAFKA_API::clients + diff --git a/modern-cpp-kafka/include/kafka/KafkaException.h b/modern-cpp-kafka/include/kafka/KafkaException.h new file mode 100644 index 00000000..a4fd3d5d --- /dev/null +++ b/modern-cpp-kafka/include/kafka/KafkaException.h @@ -0,0 +1,60 @@ +#pragma once + +#include + +#include +#include +#include + +#include + +#include +#include +#include + + +namespace KAFKA_API { + +/** + * Specific exception for Kafka clients. + */ +class KafkaException: public std::exception +{ +public: + KafkaException(const char* filename, std::size_t lineno, const Error& error) + : _when(std::chrono::system_clock::now()), + _filename(filename), + _lineno(lineno), + _error(std::make_shared(error)) + {} + + /** + * Obtains the underlying error. + */ + const Error& error() const { return *_error; } + + /** + * Obtains explanatory string. + */ + const char* what() const noexcept override + { + _what = utility::getLocalTimeString(_when) + ": " + _error->toString() + " (" + std::string(_filename) + ":" + std::to_string(_lineno) + ")"; + return _what.c_str(); + } + +private: + using TimePoint = std::chrono::system_clock::time_point; + + const TimePoint _when; + const std::string _filename; + const std::size_t _lineno; + const std::shared_ptr _error; + mutable std::string _what; +}; + + +#define KAFKA_THROW_ERROR(error) throw KafkaException(__FILE__, __LINE__, error) +#define KAFKA_THROW_IF_WITH_ERROR(error) if (error) KAFKA_THROW_ERROR(error) + +} // end of KAFKA_API + diff --git a/modern-cpp-kafka/include/kafka/KafkaProducer.h b/modern-cpp-kafka/include/kafka/KafkaProducer.h new file mode 100644 index 00000000..5d6c59f0 --- /dev/null +++ b/modern-cpp-kafka/include/kafka/KafkaProducer.h @@ -0,0 +1,516 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + + +namespace KAFKA_API { namespace clients { + +/** + * KafkaProducer class. + */ +class KafkaProducer: public KafkaClient +{ +public: + /** + * The constructor for KafkaProducer. + * + * Options: + * - EventsPollingOption::Auto (default) : An internal thread would be started for MessageDelivery callbacks handling. + * - EventsPollingOption::Manual : User have to call the member function `pollEvents()` to trigger MessageDelivery callbacks. + * + * Throws KafkaException with errors: + * - RD_KAFKA_RESP_ERR__INVALID_ARG : Invalid BOOTSTRAP_SERVERS property + * - RD_KAFKA_RESP_ERR__CRIT_SYS_RESOURCE: Fail to create internal threads + */ + explicit KafkaProducer(const Properties& properties, + EventsPollingOption eventsPollingOption = EventsPollingOption::Auto); + + /** + * The destructor for KafkaProducer. + */ + ~KafkaProducer() override { if (_opened) close(); } + + /** + * Invoking this method makes all buffered records immediately available to send, and blocks on the completion of the requests associated with these records. + * + * Possible error values: + * - RD_KAFKA_RESP_ERR__TIMED_OUT: The `timeout` was reached before all outstanding requests were completed. + */ + Error flush(std::chrono::milliseconds timeout = std::chrono::milliseconds::max()); + + /** + * Purge messages currently handled by the KafkaProducer. + */ + Error purge(); + + /** + * Close this producer. This method would wait up to timeout for the producer to complete the sending of all incomplete requests (before purging them). + */ + void close(std::chrono::milliseconds timeout = std::chrono::milliseconds::max()); + + /** + * Options for sending messages. + */ + enum class SendOption { NoCopyRecordValue, ToCopyRecordValue }; + + /** + * Choose the action while the sending buffer is full. + */ + enum class ActionWhileQueueIsFull { Block, NoBlock }; + + /** + * Asynchronously send a record to a topic. + * + * Note: + * - If a callback is provided, it's guaranteed to be triggered (before closing the producer). + * - If any error occured, an exception would be thrown. + * - Make sure the memory block (for ProducerRecord's value) is valid until the delivery callback finishes; Otherwise, should be with option `KafkaProducer::SendOption::ToCopyRecordValue`. + * + * Possible errors: + * Local errors, + * - RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC: The topic doesn't exist + * - RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION: The partition doesn't exist + * - RD_KAFKA_RESP_ERR__INVALID_ARG: Invalid topic(topic is null, or the length is too long (> 512) + * - RD_KAFKA_RESP_ERR__MSG_TIMED_OUT: No ack received within the time limit + * - RD_KAFKA_RESP_ERR__QUEUE_FULL: The message buffing queue is full + * Broker errors, + * - [Error Codes] (https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-ErrorCodes) + */ + void send(const producer::ProducerRecord& record, + const producer::Callback& deliveryCb, + SendOption option = SendOption::NoCopyRecordValue, + ActionWhileQueueIsFull action = ActionWhileQueueIsFull::Block); + + /** + * Asynchronously send a record to a topic. + * + * Note: + * - If a callback is provided, it's guaranteed to be triggered (before closing the producer). + * - The input reference parameter `error` will be set if an error occurred. + * - Make sure the memory block (for ProducerRecord's value) is valid until the delivery callback finishes; Otherwise, should be with option `KafkaProducer::SendOption::ToCopyRecordValue`. + * + * Possible errors: + * Local errors, + * - RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC: The topic doesn't exist + * - RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION: The partition doesn't exist + * - RD_KAFKA_RESP_ERR__INVALID_ARG: Invalid topic(topic is null, or the length is too long (> 512) + * - RD_KAFKA_RESP_ERR__MSG_TIMED_OUT: No ack received within the time limit + * - RD_KAFKA_RESP_ERR__QUEUE_FULL: The message buffing queue is full + * Broker errors, + * - [Error Codes] (https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-ErrorCodes) + */ + void send(const producer::ProducerRecord& record, + const producer::Callback& deliveryCb, + Error& error, + SendOption option = SendOption::NoCopyRecordValue, + ActionWhileQueueIsFull action = ActionWhileQueueIsFull::Block) + { + try { send(record, deliveryCb, option, action); } catch (const KafkaException& e) { error = e.error(); } + } + + /** + * Synchronously send a record to a topic. + * Throws KafkaException with errors: + * Local errors, + * - RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC: The topic doesn't exist + * - RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION: The partition doesn't exist + * - RD_KAFKA_RESP_ERR__INVALID_ARG: Invalid topic(topic is null, or the length is too long (> 512) + * - RD_KAFKA_RESP_ERR__MSG_TIMED_OUT: No ack received within the time limit + * Broker errors, + * - [Error Codes] (https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-ErrorCodes) + */ + producer::RecordMetadata syncSend(const producer::ProducerRecord& record); + + /** + * Needs to be called before any other methods when the transactional.id is set in the configuration. + */ + void initTransactions(std::chrono::milliseconds timeout = std::chrono::milliseconds(KafkaProducer::DEFAULT_INIT_TRANSACTIONS_TIMEOUT_MS)); + + /** + * Should be called before the start of each new transaction. + */ + void beginTransaction(); + + /** + * Commit the ongoing transaction. + */ + void commitTransaction(std::chrono::milliseconds timeout = std::chrono::milliseconds(KafkaProducer::DEFAULT_COMMIT_TRANSACTION_TIMEOUT_MS)); + + /** + * Abort the ongoing transaction. + */ + void abortTransaction(std::chrono::milliseconds timeout = std::chrono::milliseconds::max()); + + + /** + * Send a list of specified offsets to the consumer group coodinator, and also marks those offsets as part of the current transaction. + */ + void sendOffsetsToTransaction(const TopicPartitionOffsets& topicPartitionOffsets, + const consumer::ConsumerGroupMetadata& groupMetadata, + std::chrono::milliseconds timeout); + +#if COMPILER_SUPPORTS_CPP_17 + static constexpr int DEFAULT_INIT_TRANSACTIONS_TIMEOUT_MS = 10000; + static constexpr int DEFAULT_COMMIT_TRANSACTION_TIMEOUT_MS = 10000; +#else + enum { DEFAULT_INIT_TRANSACTIONS_TIMEOUT_MS = 10000 }; + enum { DEFAULT_COMMIT_TRANSACTION_TIMEOUT_MS = 10000 }; +#endif + +private: + void pollCallbacks(int timeoutMs) + { + rd_kafka_poll(getClientHandle(), timeoutMs); + } + + // Define datatypes for "opaque" (as an input for rd_kafka_produceva), in order to handle the delivery callback + class DeliveryCbOpaque + { + public: + DeliveryCbOpaque(Optional id, producer::Callback cb): _recordId(id), _deliveryCb(std::move(cb)) {} + + void operator()(rd_kafka_t* /*rk*/, const rd_kafka_message_t* rkmsg) + { + _deliveryCb(producer::RecordMetadata{rkmsg, _recordId}, Error{rkmsg->err}); + } + + private: + const Optional _recordId; + const producer::Callback _deliveryCb; + }; + + // Validate properties (and fix it if necesary) + static Properties validateAndReformProperties(const Properties& properties); + + // Delivery Callback (for librdkafka) + static void deliveryCallback(rd_kafka_t* rk, const rd_kafka_message_t* rkmsg, void* opaque); + + // Register Callbacks for rd_kafka_conf_t + static void registerConfigCallbacks(rd_kafka_conf_t* conf); + +#ifdef KAFKA_API_ENABLE_UNIT_TEST_STUBS +public: + using HandleProduceResponseCb = std::function; + + /** + * Stub for ProduceResponse handing. + * Note: Only for internal unit tests + */ + void stubHandleProduceResponse(HandleProduceResponseCb cb = HandleProduceResponseCb()) { _handleProduceRespCb = std::move(cb); } + +private: + static rd_kafka_resp_err_t handleProduceResponse(rd_kafka_t* rk, int32_t brokerId, uint64_t msgSeq, rd_kafka_resp_err_t err) + { + auto* client = static_cast(rd_kafka_opaque(rk)); + auto* producer = dynamic_cast(client); + auto respCb = producer->_handleProduceRespCb; + return respCb ? respCb(rk, brokerId, msgSeq, err) : err; + } + + HandleProduceResponseCb _handleProduceRespCb; +#endif +}; + +inline +KafkaProducer::KafkaProducer(const Properties& properties, EventsPollingOption eventsPollingOption) + : KafkaClient(ClientType::KafkaProducer, + validateAndReformProperties(properties), + registerConfigCallbacks, + eventsPollingOption) +{ + // Start background polling (if needed) + startBackgroundPollingIfNecessary([this](int timeoutMs){ pollCallbacks(timeoutMs); }); + + const auto propStr = KafkaClient::properties().toString(); + KAFKA_API_DO_LOG(Log::Level::Notice, "initializes with properties[%s]", propStr.c_str()); +} + +inline void +KafkaProducer::registerConfigCallbacks(rd_kafka_conf_t* conf) +{ + // Delivery Callback + rd_kafka_conf_set_dr_msg_cb(conf, deliveryCallback); + +#ifdef KAFKA_API_ENABLE_UNIT_TEST_STUBS + // UT stub for ProduceResponse + LogBuffer errInfo; + if (rd_kafka_conf_set(conf, "ut_handle_ProduceResponse", reinterpret_cast(&handleProduceResponse), errInfo.str(), errInfo.capacity())) // NOLINT + { + KafkaClient* client = nullptr; + size_t clientPtrSize = 0; + if (rd_kafka_conf_get(conf, "opaque", reinterpret_cast(&client), &clientPtrSize)) // NOLINT + { + KAFKA_API_LOG(Log::Level::Crit, "failed to stub ut_handle_ProduceResponse! error[%s]. Meanwhile, failed to get the Kafka client!", errInfo.c_str()); + } + else + { + assert(clientPtrSize == sizeof(client)); // NOLINT + client->KAFKA_API_DO_LOG(Log::Level::Err, "failed to stub ut_handle_ProduceResponse! error[%s]", errInfo.c_str()); + } + } +#endif +} + +inline Properties +KafkaProducer::validateAndReformProperties(const Properties& properties) +{ + // Let the base class validate first + auto newProperties = KafkaClient::validateAndReformProperties(properties); + + // Check whether it's an available partitioner + const std::set availPartitioners = {"murmur2_random", "murmur2", "random", "consistent", "consistent_random", "fnv1a", "fnv1a_random"}; + auto partitioner = newProperties.getProperty(producer::Config::PARTITIONER); + if (partitioner && !availPartitioners.count(*partitioner)) + { + std::string errMsg = "Invalid partitioner [" + *partitioner + "]! Valid options: "; + bool isTheFirst = true; + for (const auto& availPartitioner: availPartitioners) + { + errMsg += (std::string(isTheFirst ? (isTheFirst = false, "") : ", ") + availPartitioner); + } + errMsg += "."; + + KAFKA_THROW_ERROR(Error(RD_KAFKA_RESP_ERR__INVALID_ARG, errMsg)); + } + + // For "idempotence" feature + constexpr int KAFKA_IDEMP_MAX_INFLIGHT = 5; + const auto enableIdempotence = newProperties.getProperty(producer::Config::ENABLE_IDEMPOTENCE); + if (enableIdempotence && *enableIdempotence == "true") + { + if (const auto maxInFlight = newProperties.getProperty(producer::Config::MAX_IN_FLIGHT)) + { + if (std::stoi(*maxInFlight) > KAFKA_IDEMP_MAX_INFLIGHT) + { + KAFKA_THROW_ERROR(Error(RD_KAFKA_RESP_ERR__INVALID_ARG,\ + "`max.in.flight` must be set <= " + std::to_string(KAFKA_IDEMP_MAX_INFLIGHT) + " when `enable.idempotence` is `true`")); + } + } + + if (const auto acks = newProperties.getProperty(producer::Config::ACKS)) + { + if (*acks != "all" && *acks != "-1") + { + KAFKA_THROW_ERROR(Error(RD_KAFKA_RESP_ERR__INVALID_ARG,\ + "`acks` must be set to `all`/`-1` when `enable.idempotence` is `true`")); + } + } + } + + return newProperties; +} + +// Delivery Callback (for librdkafka) +inline void +KafkaProducer::deliveryCallback(rd_kafka_t* rk, const rd_kafka_message_t* rkmsg, void* /*opaque*/) +{ + if (auto* deliveryCbOpaque = static_cast(rkmsg->_private)) + { + (*deliveryCbOpaque)(rk, rkmsg); + delete deliveryCbOpaque; + } +} + +inline void +KafkaProducer::send(const producer::ProducerRecord& record, + const producer::Callback& deliveryCb, + SendOption option, + ActionWhileQueueIsFull action) +{ + auto deliveryCbOpaque = std::make_unique(record.id(), deliveryCb); + auto queueFullAction = (isWithAutoEventsPolling() ? action : ActionWhileQueueIsFull::NoBlock); + + const auto* topic = record.topic().c_str(); + const auto partition = record.partition(); + const auto msgFlags = (static_cast(option == SendOption::ToCopyRecordValue ? RD_KAFKA_MSG_F_COPY : 0) + | static_cast(queueFullAction == ActionWhileQueueIsFull::Block ? RD_KAFKA_MSG_F_BLOCK : 0)); + const auto* keyPtr = record.key().data(); + const auto keyLen = record.key().size(); + const auto* valuePtr = record.value().data(); + const auto valueLen = record.value().size(); + + auto* rk = getClientHandle(); + auto* opaquePtr = deliveryCbOpaque.get(); + + constexpr std::size_t VU_LIST_SIZE_WITH_NO_HEADERS = 6; + std::vector rkVUs(VU_LIST_SIZE_WITH_NO_HEADERS + record.headers().size()); + + std::size_t uvCount = 0; + + { // Topic + auto& vu = rkVUs[uvCount++]; + vu.vtype = RD_KAFKA_VTYPE_TOPIC; + vu.u.cstr = topic; + } + + { // Partition + auto& vu = rkVUs[uvCount++]; + vu.vtype = RD_KAFKA_VTYPE_PARTITION; + vu.u.i32 = partition; + } + + { // Message flags + auto& vu = rkVUs[uvCount++]; + vu.vtype = RD_KAFKA_VTYPE_MSGFLAGS; + vu.u.i = static_cast(msgFlags); + } + + { // Key + auto& vu = rkVUs[uvCount++]; + vu.vtype = RD_KAFKA_VTYPE_KEY; + vu.u.mem.ptr = const_cast(keyPtr); // NOLINT + vu.u.mem.size = keyLen; + } + + { // Value + auto& vu = rkVUs[uvCount++]; + vu.vtype = RD_KAFKA_VTYPE_VALUE; + vu.u.mem.ptr = const_cast(valuePtr); // NOLINT + vu.u.mem.size = valueLen; + } + + { // Opaque + auto& vu = rkVUs[uvCount++]; + vu.vtype = RD_KAFKA_VTYPE_OPAQUE; + vu.u.ptr = opaquePtr; + } + + // Headers + for (const auto& header: record.headers()) + { + auto& vu = rkVUs[uvCount++]; + vu.vtype = RD_KAFKA_VTYPE_HEADER; + vu.u.header.name = header.key.c_str(); + vu.u.header.val = header.value.data(); + vu.u.header.size = static_cast(header.value.size()); + } + + assert(uvCount == rkVUs.size()); + + Error sendResult{ rd_kafka_produceva(rk, rkVUs.data(), rkVUs.size()) }; + KAFKA_THROW_IF_WITH_ERROR(sendResult); + + // KafkaProducer::deliveryCallback would delete the "opaque" + deliveryCbOpaque.release(); +} + +inline producer::RecordMetadata +KafkaProducer::syncSend(const producer::ProducerRecord& record) +{ + Optional deliveryResult; + producer::RecordMetadata recordMetadata; + std::mutex mtx; + std::condition_variable delivered; + + auto deliveryCb = [&deliveryResult, &recordMetadata, &mtx, &delivered] (const producer::RecordMetadata& metadata, const Error& error) { + std::lock_guard guard(mtx); + + deliveryResult = error; + recordMetadata = metadata; + + delivered.notify_one(); + }; + + send(record, deliveryCb); + + std::unique_lock lock(mtx); + delivered.wait(lock, [&deliveryResult]{ return static_cast(deliveryResult); }); + + KAFKA_THROW_IF_WITH_ERROR(*deliveryResult); + + return recordMetadata; +} + +inline Error +KafkaProducer::flush(std::chrono::milliseconds timeout) +{ + return Error{rd_kafka_flush(getClientHandle(), convertMsDurationToInt(timeout))}; +} + +inline Error +KafkaProducer::purge() +{ + return Error{rd_kafka_purge(getClientHandle(), + (static_cast(RD_KAFKA_PURGE_F_QUEUE) | static_cast(RD_KAFKA_PURGE_F_INFLIGHT)))}; +} + +inline void +KafkaProducer::close(std::chrono::milliseconds timeout) +{ + _opened = false; + + stopBackgroundPollingIfNecessary(); + + Error result = flush(timeout); + if (result.value() == RD_KAFKA_RESP_ERR__TIMED_OUT) + { + KAFKA_API_DO_LOG(Log::Level::Notice, "purge messages before close, outQLen[%d]", rd_kafka_outq_len(getClientHandle())); + purge(); + } + + rd_kafka_poll(getClientHandle(), 0); + + KAFKA_API_DO_LOG(Log::Level::Notice, "closed"); + +} + +inline void +KafkaProducer::initTransactions(std::chrono::milliseconds timeout) +{ + Error result{ rd_kafka_init_transactions(getClientHandle(), static_cast(timeout.count())) }; // NOLINT + KAFKA_THROW_IF_WITH_ERROR(result); +} + +inline void +KafkaProducer::beginTransaction() +{ + Error result{ rd_kafka_begin_transaction(getClientHandle()) }; + KAFKA_THROW_IF_WITH_ERROR(result); +} + +inline void +KafkaProducer::commitTransaction(std::chrono::milliseconds timeout) +{ + Error result{ rd_kafka_commit_transaction(getClientHandle(), static_cast(timeout.count())) }; // NOLINT + KAFKA_THROW_IF_WITH_ERROR(result); +} + +inline void +KafkaProducer::abortTransaction(std::chrono::milliseconds timeout) +{ + Error result{ rd_kafka_abort_transaction(getClientHandle(), static_cast(timeout.count())) }; // NOLINT + KAFKA_THROW_IF_WITH_ERROR(result); +} + +inline void +KafkaProducer::sendOffsetsToTransaction(const TopicPartitionOffsets& topicPartitionOffsets, + const consumer::ConsumerGroupMetadata& groupMetadata, + std::chrono::milliseconds timeout) +{ + auto rk_tpos = rd_kafka_topic_partition_list_unique_ptr(createRkTopicPartitionList(topicPartitionOffsets)); + Error result{ rd_kafka_send_offsets_to_transaction(getClientHandle(), + rk_tpos.get(), + groupMetadata.rawHandle(), + static_cast(timeout.count())) }; // NOLINT + KAFKA_THROW_IF_WITH_ERROR(result); +} + +} } // end of KAFKA_API::clients + diff --git a/modern-cpp-kafka/include/kafka/Log.h b/modern-cpp-kafka/include/kafka/Log.h new file mode 100644 index 00000000..d8790568 --- /dev/null +++ b/modern-cpp-kafka/include/kafka/Log.h @@ -0,0 +1,88 @@ +#pragma once + +#include + +#include + +#include +#include +#include +#include +#include + + +namespace KAFKA_API { + +struct Log +{ + enum Level + { + Emerg = 0, + Alert = 1, + Crit = 2, + Err = 3, + Warning = 4, + Notice = 5, + Info = 6, + Debug = 7 + }; + + static const std::string& levelString(std::size_t level) + { + static const std::vector levelNames = {"EMERG", "ALERT", "CRIT", "ERR", "WARNING", "NOTICE", "INFO", "DEBUG", "INVALID"}; + static const std::size_t maxIndex = levelNames.size() - 1; + + return levelNames[std::min(level, maxIndex)]; + } +}; + +template +class LogBuffer +{ +public: + LogBuffer():_wptr(_buf.data()) { _buf[0] = 0; } // NOLINT + + LogBuffer& clear() + { + _wptr = _buf.data(); + _buf[0] = 0; + return *this; + } + + template + LogBuffer& print(const char* format, Args... args) + { + assert(!(_buf[0] != 0 && _wptr == _buf.data())); // means it has already been used as a plain buffer (with `str()`) + + auto cnt = std::snprintf(_wptr, capacity(), format, args...); // returns number of characters written if successful (not including '\0') + if (cnt > 0) + { + _wptr = std::min(_wptr + cnt, _buf.data() + MAX_CAPACITY - 1); + } + return *this; + } + LogBuffer& print(const char* format) { return print("%s", format); } + + std::size_t capacity() const { return static_cast(_buf.data() + MAX_CAPACITY - _wptr); } + char* str() { return _buf.data(); } + const char* c_str() const { return _buf.data(); } + +private: + std::array _buf; + char* _wptr; +}; + +using Logger = std::function; + +inline void DefaultLogger(int level, const char* /*filename*/, int /*lineno*/, const char* msg) +{ + std::cout << "[" << utility::getCurrentTime() << "]" << Log::levelString(static_cast(level)) << " " << msg; + std::cout << std::endl; +} + +inline void NullLogger(int /*level*/, const char* /*filename*/, int /*lineno*/, const char* /*msg*/) +{ +} + +} // end of KAFKA_API + diff --git a/modern-cpp-kafka/include/kafka/ProducerCommon.h b/modern-cpp-kafka/include/kafka/ProducerCommon.h new file mode 100644 index 00000000..ede7fd02 --- /dev/null +++ b/modern-cpp-kafka/include/kafka/ProducerCommon.h @@ -0,0 +1,185 @@ +#pragma once + +#include + +#include +#include +#include +#include + +#include + +#include +#include + + +namespace KAFKA_API { namespace clients { namespace producer { + +/** + * The metadata for a record that has been acknowledged by the server. + */ +class RecordMetadata +{ +public: + enum class PersistedStatus { Not, Possibly, Done }; + + RecordMetadata() = default; + + RecordMetadata(const RecordMetadata& another) { *this = another; } + + // This is only called by the KafkaProducer::deliveryCallback (with a valid rkmsg pointer) + RecordMetadata(const rd_kafka_message_t* rkmsg, Optional recordId) + : _rkmsg(rkmsg), _recordId(recordId) {} + + RecordMetadata& operator=(const RecordMetadata& another) + { + if (this != &another) + { + _cachedInfo = std::make_unique(another.topic(), + another.partition(), + another.offset() ? *another.offset() : RD_KAFKA_OFFSET_INVALID, + another.keySize(), + another.valueSize(), + another.timestamp(), + another.persistedStatus()); + _recordId = another._recordId; + _rkmsg = nullptr; + } + + return *this; + } + + /** + * The topic the record was appended to. + */ + std::string topic() const + { + return _rkmsg ? (_rkmsg->rkt ? rd_kafka_topic_name(_rkmsg->rkt) : "") : _cachedInfo->topic; + } + + /** + * The partition the record was sent to. + */ + Partition partition() const + { + return _rkmsg ? _rkmsg->partition : _cachedInfo->partition; + } + + /** + * The offset of the record in the topic/partition. + */ + Optional offset() const + { + auto offset = _rkmsg ? _rkmsg->offset : _cachedInfo->offset; + return (offset != RD_KAFKA_OFFSET_INVALID) ? Optional(offset) : Optional(); + } + + /** + * The recordId could be used to identify the acknowledged message. + */ + Optional recordId() const + { + return _recordId; + } + + /** + * The size of the key in bytes. + */ + KeySize keySize() const + { + return _rkmsg ? _rkmsg->key_len : _cachedInfo->keySize; + } + + /** + * The size of the value in bytes. + */ + ValueSize valueSize() const + { + return _rkmsg ? _rkmsg->len : _cachedInfo->valueSize; + } + + /** + * The timestamp of the record in the topic/partition. + */ + Timestamp timestamp() const + { + return _rkmsg ? getMsgTimestamp(_rkmsg) : _cachedInfo->timestamp; + } + + /** + * The persisted status of the record. + */ + PersistedStatus persistedStatus() const + { + return _rkmsg ? getMsgPersistedStatus(_rkmsg) : _cachedInfo->persistedStatus; + } + + std::string persistedStatusString() const + { + return getPersistedStatusString(persistedStatus()); + } + + std::string toString() const + { + return topic() + "-" + std::to_string(partition()) + "@" + (offset() ? std::to_string(*offset()) : "NA") + + (recordId() ? (":id[" + std::to_string(*recordId()) + "],") : ",") + + timestamp().toString() + "," + persistedStatusString(); + } + +private: + static Timestamp getMsgTimestamp(const rd_kafka_message_t* rkmsg) + { + rd_kafka_timestamp_type_t tstype{}; + Timestamp::Value tsValue = rd_kafka_message_timestamp(rkmsg, &tstype); + return {tsValue, tstype}; + } + + static PersistedStatus getMsgPersistedStatus(const rd_kafka_message_t* rkmsg) + { + rd_kafka_msg_status_t status = rd_kafka_message_status(rkmsg); + return status == RD_KAFKA_MSG_STATUS_NOT_PERSISTED ? PersistedStatus::Not : (status == RD_KAFKA_MSG_STATUS_PERSISTED ? PersistedStatus::Done : PersistedStatus::Possibly); + } + + static std::string getPersistedStatusString(PersistedStatus status) + { + return status == PersistedStatus::Not ? "NotPersisted" : + (status == PersistedStatus::Done ? "Persisted" : "PossiblyPersisted"); + } + + struct CachedInfo + { + CachedInfo(Topic t, Partition p, Offset o, KeySize ks, ValueSize vs, Timestamp ts, PersistedStatus pst) + : topic(std::move(t)), + partition(p), + offset(o), + keySize(ks), + valueSize(vs), + timestamp(ts), + persistedStatus(pst) + { + } + + CachedInfo(const CachedInfo&) = default; + + std::string topic; + Partition partition; + Offset offset; + KeySize keySize; + ValueSize valueSize; + Timestamp timestamp; + PersistedStatus persistedStatus; + }; + + std::unique_ptr _cachedInfo; + const rd_kafka_message_t* _rkmsg = nullptr; + Optional _recordId; +}; + +/** + * A callback method could be used to provide asynchronous handling of request completion. + * This method will be called when the record sent (by KafkaAsyncProducer) to the server has been acknowledged. + */ +using Callback = std::function; + +} } } // end of KAFKA_API::clients::producer + diff --git a/modern-cpp-kafka/include/kafka/ProducerConfig.h b/modern-cpp-kafka/include/kafka/ProducerConfig.h new file mode 100644 index 00000000..b3a8b44b --- /dev/null +++ b/modern-cpp-kafka/include/kafka/ProducerConfig.h @@ -0,0 +1,150 @@ +#pragma once + +#include + +#include + + +namespace KAFKA_API { namespace clients { namespace producer { + +/** + * Configuration for the Kafka Producer. + */ +class Config: public Properties +{ +public: + Config() = default; + Config(const Config&) = default; + explicit Config(const PropertiesMap& kvMap): Properties(kvMap) {} + + /** + * The string contains host:port pairs of brokers (splitted by ",") that the producer will use to establish initial connection to the Kafka cluster. + * Note: It's mandatory. + */ + static const constexpr char* BOOTSTRAP_SERVERS = "bootstrap.servers"; + + /** + * This can be any string, and will be used by the brokers to identify messages sent from the client. + */ + static const constexpr char* CLIENT_ID = "client.id"; + + /** + * The acks parameter controls how many partition replicas must receive the record before the producer can consider the write successful. + * 1) acks=0, the producer will not wait for a reply from the broker before assuming the message was sent successfully. + * 2) acks=1, the producer will receive a success response from the broker the moment the leader replica received the message. + * 3) acks=all, the producer will receive a success response from the broker once all in-sync replicas received the message. + * Note: if "ack=all", please make sure the topic's replication factor be larger than 1. + * That means, if the topic is automaticly created by producer's `send`, the `default.replication.factor` property for the kafka server should be larger than 1. + * The "ack=all" property is mandatory for reliability requirements, but would increase the ack latency and impact the throughput. + * Default value: all + */ + static const constexpr char* ACKS = "acks"; + + /** + * Maximum number of messages allowed on the producer queue. + * Default value: 100000 + */ + static const constexpr char* QUEUE_BUFFERING_MAX_MESSAGES = "queue.buffering.max.messages"; + + /** + * Maximum total message size sum allowed on the producer queue. + * Default value: 0x100000 (1GB) + */ + static const constexpr char* QUEUE_BUFFERING_MAX_KBYTES = "queue.buffering.max.kbytes"; + + /** + * Delay in milliseconds to wait for messages in the producer queue, to accumulate before constructing messages batches to transmit to brokers. + * Default value: 0 (KafkaSyncProducer); 0.5 (KafkaAsyncProducer) + */ + static const constexpr char* LINGER_MS = "linger.ms"; + + /** + * Maximum number of messages batched in one messageSet. The total MessageSet size is also limited by MESSAGE_MAX_BYTES. + * Default value: 10000 + */ + static const constexpr char* BATCH_NUM_MESSAGES = "batch.num.messages"; + + /** + * Maximum size (in bytes) of all messages batched in one MessageSet (including protocol framing overhead). + * Default value: 1000000 + */ + static const constexpr char* BATCH_SIZE = "batch.size"; + + /** + * Maximum Kafka protocol request message size. + * Note: Should be coordinated with the brokers's configuration. Otherwise, any larger message would be rejected! + * Default value: 1000000 + */ + static const constexpr char* MESSAGE_MAX_BYTES = "message.max.bytes"; + + /** + * This value is enforced locally and limits the time a produced message waits for successful delivery. + * Note: If failed to get the ack within this limit, an exception would be thrown (in `SyncProducer.send()`), or an error code would be passed into the delivery callback (AsyncProducer). + * Default value: 300000 + */ + static const constexpr char* MESSAGE_TIMEOUT_MS = "message.timeout.ms"; + + /** + * This value is only enforced by the brokers and relies on `ACKS` being non-zero. + * Note: The leading broker waits for in-sync replicas to acknowledge the message, and will return an error if the time elapses without the necessary acks. + * Default value: 5000 + */ + static const constexpr char* REQUEST_TIMEOUT_MS = "request.timeout.ms"; + + /** + * The default partitioner for a ProducerRecord (with no partition assigned). + * Note: It's not the same with Java version's "partitioner.class" property + * Available options: + * 1) random -- random distribution + * 2) consistent -- CRC32 hash of key (`ProducerRecord`s with empty/null key are mapped to single partition) + * 3) consistent_random -- CRC32 hash of key (`ProducerRecord`s with empty/null key are randomly partitioned) + * 4) murmur2 -- Java Producer compatible Murmur2 hash of key (`ProducerRecord`s with null key are mapped to single partition) + * 5) murmur2_random -- Java Producer compatible Murmur2 hash of key (`ProducerRecord`s with null key are randomly partitioned. It's equivalent to the Java Producer's default partitioner) + * 6) fnv1a -- FNV-1a hash of key (`ProducerRecord`s with null key are mapped to single partition) + * 7) fnv1a_random -- FNV-1a hash of key (`ProducerRecord`s with null key are randomly partitioned) + * Default value: murmur2_random + */ + static const constexpr char* PARTITIONER = "partitioner"; + + /** + * Maximum number of in-flight requests per broker connection. + * Default value: 1000000 (while `enable.idempotence`=false); 5 (while `enable.idempotence`=true) + */ + static const constexpr char* MAX_IN_FLIGHT = "max.in.flight"; + + /** + * When set to `true`, the producer will ensure that messages are succefully sent exactly once and in the original order. + * Default value: false + */ + static const constexpr char* ENABLE_IDEMPOTENCE = "enable.idempotence"; + + /** + * It's used to identify the same transactional producer instance across process restarts. + */ + static const constexpr char* TRANSACTIONAL_ID = "transactional.id"; + + /** + * Th maximus amount of time in milliseconds that the transaction coordinator will wait for a trnsaction status update from the producer before proactively ablrting the ongoing transaction. + * Default value: 60000 + */ + static const constexpr char* TRANSACTION_TIMEOUT_MS = "transaction.timeout.ms"; + + /** + * Protocol used to communicate with brokers. + * Default value: plaintext + */ + static const constexpr char* SECURITY_PROTOCOL = "security.protocol"; + + /** + * Shell command to refresh or acquire the client's Kerberos ticket. + */ + static const constexpr char* SASL_KERBEROS_KINIT_CMD = "sasl.kerberos.kinit.cmd"; + + /** + * The client's Kerberos principal name. + */ + static const constexpr char* SASL_KERBEROS_SERVICE_NAME = "sasl.kerberos.service.name"; +}; + +} } } // end of KAFKA_API::clients::producer + diff --git a/modern-cpp-kafka/include/kafka/ProducerRecord.h b/modern-cpp-kafka/include/kafka/ProducerRecord.h new file mode 100644 index 00000000..f466bfce --- /dev/null +++ b/modern-cpp-kafka/include/kafka/ProducerRecord.h @@ -0,0 +1,109 @@ +#pragma once + +#include + +#include +#include + +#include + + +namespace KAFKA_API { namespace clients { namespace producer { + +/** + * A key/value pair to be sent to Kafka. + * This consists of a topic name to which the record is being sent, an optional partition number, and an optional key and value. + * Note: `ProducerRecord` would not take the ownership from the memory block of `Value`. + */ +class ProducerRecord +{ +public: + using Id = std::uint64_t; + + ProducerRecord(Topic topic, Partition partition, const Key& key, const Value& value) + : _topic(std::move(topic)), _partition(partition), _key(key), _value(value) {} + + ProducerRecord(const Topic& topic, Partition partition, const Key& key, const Value& value, Id id) + : ProducerRecord(topic, partition, key, value) { _id = id; } + + ProducerRecord(const Topic& topic, const Key& key, const Value& value) + : ProducerRecord(topic, RD_KAFKA_PARTITION_UA, key, value) {} + + ProducerRecord(const Topic& topic, const Key& key, const Value& value, Id id) + : ProducerRecord(topic, key, value) { _id = id; } + + /** + * The topic this record is being sent to. + */ + const Topic& topic() const { return _topic; } + + /** + * The partition to which the record will be sent (or UNKNOWN_PARTITION if no partition was specified). + */ + Partition partition() const { return _partition; } + + /** + * The key (or null if no key is specified). + */ + Key key() const { return _key; } + + /** + * The value. + */ + Value value() const { return _value; } + + /** + * The id to identify the message (consistent with `Producer::Metadata::recordId()`). + */ + Optional id() const { return _id; } + + /** + * The headers. + */ + const Headers& headers() const { return _headers; } + + /** + * The headers. + * Note: Users could set headers with the reference. + */ + Headers& headers() { return _headers; } + + /** + * Set the partition. + */ + void setPartition(Partition partition) { _partition = partition; } + + /** + * Set the key. + */ + void setKey(const Key& key) { _key = key; } + + /** + * Set the value. + */ + void setValue(const Value& value) { _value = value; } + + /** + * Set the record id. + */ + void setId(Id id) { _id = id; } + + std::string toString() const + { + return _topic + "-" + (_partition == RD_KAFKA_PARTITION_UA ? "NA" : std::to_string(_partition)) + std::string(":") + + (_id ? (std::to_string(*_id) + std::string(", ")) : " ") + + (_headers.empty() ? "" : ("headers[" + KAFKA_API::toString(_headers) + "], ")) + + _key.toString() + std::string("/") + _value.toString(); + } + +private: + Topic _topic; + Partition _partition; + Key _key; + Value _value; + Headers _headers; + Optional _id; +}; + +} } } // end of KAFKA_API::clients::producer + diff --git a/modern-cpp-kafka/include/kafka/Project.h b/modern-cpp-kafka/include/kafka/Project.h new file mode 100644 index 00000000..9ab1bf08 --- /dev/null +++ b/modern-cpp-kafka/include/kafka/Project.h @@ -0,0 +1,22 @@ +#pragma once + +// Customize the namespace (default is `kafka`) if necessary +#ifndef KAFKA_API +#define KAFKA_API kafka +#endif + +// Here is the MACRO to enable internal stubs for UT +// #ifndef KAFKA_API_ENABLE_UNIT_TEST_STUBS +// #define KAFKA_API_ENABLE_UNIT_TEST_STUBS +// #endif + +#if defined(WIN32) && !defined(NOMINMAX) +#define NOMINMAX +#endif + +#if ((__cplusplus >= 201703L) || (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L)) +#define COMPILER_SUPPORTS_CPP_17 1 +#else +#define COMPILER_SUPPORTS_CPP_17 0 +#endif + diff --git a/modern-cpp-kafka/include/kafka/Properties.h b/modern-cpp-kafka/include/kafka/Properties.h new file mode 100644 index 00000000..f21b04ff --- /dev/null +++ b/modern-cpp-kafka/include/kafka/Properties.h @@ -0,0 +1,100 @@ +#pragma once + +#include + +#include + +#include +#include +#include +#include + + +namespace KAFKA_API { + +/** + * The properties for Kafka clients. + */ +class Properties +{ +public: + // Just make sure key will printed in order + using PropertiesMap = std::map; + + Properties() = default; + Properties(const Properties&) = default; + explicit Properties(PropertiesMap kvMap): _kvMap(std::move(kvMap)) {} + + virtual ~Properties() = default; + + bool operator==(const Properties& rhs) const { return map() == rhs.map(); } + + /** + * Set a property. + * If the map previously contained a mapping for the key, the old value is replaced by the specified value. + */ + Properties& put(const std::string& key, const std::string& value) + { + _kvMap[key] = value; + return *this; + } + + /** + * Remove the property (if one exists). + */ + void remove(const std::string& key) + { + _kvMap.erase(key); + } + + /** + * Get a property. + * If the map previously contained a mapping for the key, the old value is replaced by the specified value. + */ + Optional getProperty(const std::string& key) const + { + Optional ret; + auto search = _kvMap.find(key); + if (search != _kvMap.end()) + { + ret = search->second; + } + return ret; + } + + /** + * Remove a property. + */ + void eraseProperty(const std::string& key) + { + _kvMap.erase(key); + } + + std::string toString() const + { + + std::string ret; + std::for_each(_kvMap.cbegin(), _kvMap.cend(), + [&ret](const auto& kv) { + const std::string& key = kv.first; + const std::string& value = kv.second; + + static const std::regex reSensitiveKey(R"(.+\.password|.+\.username)"); + bool isSensitive = std::regex_match(key, reSensitiveKey); + + ret.append(ret.empty() ? "" : "|").append(key).append("=").append(isSensitive ? "*" : value); + }); + return ret; + } + + /** + * Get all properties with a map. + */ + const PropertiesMap& map() const { return _kvMap; } + +private: + PropertiesMap _kvMap; +}; + +} // end of KAFKA_API + diff --git a/modern-cpp-kafka/include/kafka/RdKafkaHelper.h b/modern-cpp-kafka/include/kafka/RdKafkaHelper.h new file mode 100644 index 00000000..a7217266 --- /dev/null +++ b/modern-cpp-kafka/include/kafka/RdKafkaHelper.h @@ -0,0 +1,122 @@ +#pragma once + +#include + +#include + +#include + +#include + +namespace KAFKA_API { + +// define smart pointers for rk_kafka_xxx datatypes + +struct RkQueueDeleter { void operator()(rd_kafka_queue_t* p) { rd_kafka_queue_destroy(p); } }; +using rd_kafka_queue_unique_ptr = std::unique_ptr; + +struct RkEventDeleter { void operator()(rd_kafka_event_t* p) { rd_kafka_event_destroy(p); } }; +using rd_kafka_event_unique_ptr = std::unique_ptr; + +struct RkTopicDeleter { void operator()(rd_kafka_topic_t* p) { rd_kafka_topic_destroy(p); } }; +using rd_kafka_topic_unique_ptr = std::unique_ptr; + +struct RkTopicPartitionListDeleter { void operator()(rd_kafka_topic_partition_list_t* p) { rd_kafka_topic_partition_list_destroy(p); } }; +using rd_kafka_topic_partition_list_unique_ptr = std::unique_ptr; + +struct RkConfDeleter { void operator()(rd_kafka_conf_t* p) { rd_kafka_conf_destroy(p); } }; +using rd_kafka_conf_unique_ptr = std::unique_ptr; + +struct RkMetadataDeleter { void operator()(const rd_kafka_metadata_t* p) { rd_kafka_metadata_destroy(p); } }; +using rd_kafka_metadata_unique_ptr = std::unique_ptr; + +struct RkDeleter { void operator()(rd_kafka_t* p) { rd_kafka_destroy(p); } }; +using rd_kafka_unique_ptr = std::unique_ptr; + +struct RkNewTopicDeleter { void operator()(rd_kafka_NewTopic_t* p) { rd_kafka_NewTopic_destroy(p); } }; +using rd_kafka_NewTopic_unique_ptr = std::unique_ptr; + +struct RkDeleteTopicDeleter { void operator()(rd_kafka_DeleteTopic_t* p) { rd_kafka_DeleteTopic_destroy(p); } }; +using rd_kafka_DeleteTopic_unique_ptr = std::unique_ptr; + +struct RkDeleteRecordsDeleter { void operator()(rd_kafka_DeleteRecords_t* p) { rd_kafka_DeleteRecords_destroy(p); } }; +using rd_kafka_DeleteRecords_unique_ptr = std::unique_ptr; + +struct RkConsumerGroupMetadataDeleter { void operator()(rd_kafka_consumer_group_metadata_t* p) { rd_kafka_consumer_group_metadata_destroy(p) ; } }; +using rd_kafka_consumer_group_metadata_unique_ptr = std::unique_ptr; + +inline void RkErrorDeleter(rd_kafka_error_t* p) { rd_kafka_error_destroy(p); } +using rd_kafka_error_shared_ptr = std::shared_ptr; + +// Convert from rd_kafka_xxx datatypes +inline TopicPartitionOffsets getTopicPartitionOffsets(const rd_kafka_topic_partition_list_t* rk_tpos) +{ + TopicPartitionOffsets ret; + int count = rk_tpos ? rk_tpos->cnt : 0; + for (int i = 0; i < count; ++i) + { + const Topic t = rk_tpos->elems[i].topic; + const Partition p = rk_tpos->elems[i].partition; + const Offset o = rk_tpos->elems[i].offset; + + ret[TopicPartition(t, p)] = o; + } + return ret; +} + +inline Topics getTopics(const rd_kafka_topic_partition_list_t* rk_topics) +{ + Topics result; + for (int i = 0; i < (rk_topics ? rk_topics->cnt : 0); ++i) + { + result.insert(rk_topics->elems[i].topic); + } + return result; +} + +inline TopicPartitions getTopicPartitions(const rd_kafka_topic_partition_list_t* rk_tpos) +{ + TopicPartitions result; + for (int i = 0; i < (rk_tpos ? rk_tpos->cnt : 0); ++i) + { + result.insert(TopicPartition{rk_tpos->elems[i].topic, rk_tpos->elems[i].partition}); + } + return result; +} + +// Convert to rd_kafka_xxx datatypes +inline rd_kafka_topic_partition_list_t* createRkTopicPartitionList(const TopicPartitionOffsets& tpos) +{ + rd_kafka_topic_partition_list_t* rk_tpos = rd_kafka_topic_partition_list_new(static_cast(tpos.size())); + for (const auto& tp_o: tpos) + { + const auto& tp = tp_o.first; + const auto& o = tp_o.second; + rd_kafka_topic_partition_t* rk_tp = rd_kafka_topic_partition_list_add(rk_tpos, tp.first.c_str(), tp.second); + rk_tp->offset = o; + } + return rk_tpos; +} + +inline rd_kafka_topic_partition_list_t* createRkTopicPartitionList(const TopicPartitions& tps) +{ + TopicPartitionOffsets tpos; + for (const auto& tp: tps) + { + tpos[TopicPartition(tp.first, tp.second)] = RD_KAFKA_OFFSET_INVALID; + } + return createRkTopicPartitionList(tpos); +} + +inline rd_kafka_topic_partition_list_t* createRkTopicPartitionList(const Topics& topics) +{ + TopicPartitionOffsets tpos; + for (const auto& topic: topics) + { + tpos[TopicPartition(topic, RD_KAFKA_PARTITION_UA)] = RD_KAFKA_OFFSET_INVALID; + } + return createRkTopicPartitionList(tpos); +} + +} // end of KAFKA_API + diff --git a/modern-cpp-kafka/include/kafka/Timestamp.h b/modern-cpp-kafka/include/kafka/Timestamp.h new file mode 100644 index 00000000..4579eb52 --- /dev/null +++ b/modern-cpp-kafka/include/kafka/Timestamp.h @@ -0,0 +1,92 @@ +#pragma once + +#include + +#include + +#include +#include +#include +#include +#include +#include + + +namespace KAFKA_API { + +/** + * The time point together with the type. + */ +struct Timestamp +{ + using Value = std::int64_t; + + enum class Type { NotAvailable, CreateTime, LogAppendTime }; + + /** + * The milliseconds since epoch. + */ + Value msSinceEpoch; + + /** + * The type shows what the `msSinceEpoch` means (CreateTime or LogAppendTime). + */ + Type type; + + explicit Timestamp(Value v = 0, Type t = Type::NotAvailable): msSinceEpoch(v), type(t) {} + Timestamp(Value v, rd_kafka_timestamp_type_t t): Timestamp(v, convertType(t)) {} + + static Type convertType(rd_kafka_timestamp_type_t tstype) + { + return (tstype == RD_KAFKA_TIMESTAMP_CREATE_TIME) ? Type::CreateTime : + (tstype == RD_KAFKA_TIMESTAMP_LOG_APPEND_TIME ? Type::LogAppendTime : Type::NotAvailable); + } + + operator std::chrono::time_point() const // NOLINT + { + return std::chrono::time_point(std::chrono::milliseconds(msSinceEpoch)); + } + + static std::string toString(Type t) + { + switch (t) + { + case Type::CreateTime: + return "CreateTime"; + case Type::LogAppendTime: + return "LogAppendTime"; + default: + assert(t == Type::NotAvailable); + return ""; + } + } + + static std::string toString(Value v) + { + auto ms = std::chrono::milliseconds(v); + auto timepoint = std::chrono::time_point(ms); + std::time_t time = std::chrono::system_clock::to_time_t(timepoint); + std::ostringstream oss; + std::tm tmBuf = {}; +#if !defined(WIN32) + oss << std::put_time(localtime_r(&time, &tmBuf), "%F %T") << "." << std::setfill('0') << std::setw(3) << (v % 1000); +#else + localtime_s(&tmBuf, &time); + oss << std::put_time(&tmBuf, "%F %T") << "." << std::setfill('0') << std::setw(3) << (v % 1000); +#endif + return oss.str(); + } + + /** + * Obtains explanatory string. + */ + std::string toString() const + { + auto typeString = toString(type); + auto timeString = toString(msSinceEpoch); + return typeString.empty() ? timeString : (typeString + "[" + timeString + "]"); + } +}; + +} // end of KAFKA_API + diff --git a/modern-cpp-kafka/include/kafka/Types.h b/modern-cpp-kafka/include/kafka/Types.h new file mode 100644 index 00000000..e843f902 --- /dev/null +++ b/modern-cpp-kafka/include/kafka/Types.h @@ -0,0 +1,192 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +// Use `boost::optional` for C++14, which doesn't support `std::optional` +#if COMPILER_SUPPORTS_CPP_17 +#include +template +using Optional = std::optional; +#else +#include +#include +template +using Optional = boost::optional; +#endif + + +namespace KAFKA_API { + +// Which is similar with `boost::const_buffer` (thus avoid the dependency towards `boost`) +class ConstBuffer +{ +public: + explicit ConstBuffer(const void* data = nullptr, std::size_t size = 0): _data(data), _size(size) {} + const void* data() const { return _data; } + std::size_t size() const { return _size; } + std::string toString() const + { + if (_size == 0) return _data ? "[empty]" : "[null]"; + + std::ostringstream oss; + + auto printChar = [&oss](const unsigned char c) { + if (std::isprint(c)) { + oss << c; + } else { + oss << "[0x" << std::hex << std::setfill('0') << std::setw(2) << static_cast(c) << "]"; + } + }; + const auto* beg = static_cast(_data); + std::for_each(beg, beg + _size, printChar); + + return oss.str(); + } +private: + const void* _data; + std::size_t _size; +}; + +/** + * Topic name. + */ +using Topic = std::string; + +/** + * Partition number. + */ +using Partition = std::int32_t; + +/** + * Record offset. + */ +using Offset = std::int64_t; + +/** + * Record key. + */ +using Key = ConstBuffer; +using KeySize = std::size_t; + +/** + * Null Key. + */ +#if COMPILER_SUPPORTS_CPP_17 +const inline Key NullKey = Key{}; +#else +const static Key NullKey = Key{}; +#endif + +/** + * Record value. + */ +using Value = ConstBuffer; +using ValueSize = std::size_t; + +/** + * Null Value. + */ +#if COMPILER_SUPPORTS_CPP_17 +const inline Value NullValue = Value{}; +#else +const static Value NullValue = Value{}; +#endif + +/** + * Topic set. + */ +using Topics = std::set; + +/** + * Topic Partition pair. + */ +using TopicPartition = std::pair; + +/** + * TopicPartition set. + */ +using TopicPartitions = std::set; + +/** + * Topic/Partition/Offset tuple + */ +using TopicPartitionOffset = std::tuple; + +/** + * TopicPartition to Offset map. + */ +using TopicPartitionOffsets = std::map; + + +/** + * Obtains explanatory string for Topics. + */ +inline std::string toString(const Topics& topics) +{ + std::string ret; + std::for_each(topics.cbegin(), topics.cend(), + [&ret](const auto& topic) { + ret.append(ret.empty() ? "" : ",").append(topic); + }); + return ret; +} + +/** + * Obtains explanatory string for TopicPartition. + */ +inline std::string toString(const TopicPartition& tp) +{ + return tp.first + std::string("-") + std::to_string(tp.second); +} + +/** + * Obtains explanatory string for TopicPartitions. + */ +inline std::string toString(const TopicPartitions& tps) +{ + std::string ret; + std::for_each(tps.cbegin(), tps.cend(), + [&ret](const auto& tp) { + ret.append((ret.empty() ? "" : ",") + tp.first + "-" + std::to_string(tp.second)); + }); + return ret; +} + +/** + * Obtains explanatory string for TopicPartitionOffset. + */ +inline std::string toString(const TopicPartitionOffset& tpo) +{ + return std::get<0>(tpo) + "-" + std::to_string(std::get<1>(tpo)) + ":" + std::to_string(std::get<2>(tpo)); +} + +/** + * Obtains explanatory string for TopicPartitionOffsets. + */ +inline std::string toString(const TopicPartitionOffsets& tpos) +{ + std::string ret; + std::for_each(tpos.cbegin(), tpos.cend(), + [&ret](const auto& tp_o) { + const TopicPartition& tp = tp_o.first; + const Offset& o = tp_o.second; + ret.append((ret.empty() ? "" : ",") + tp.first + "-" + std::to_string(tp.second) + ":" + std::to_string(o)); + }); + return ret; +} + +} // end of KAFKA_API + diff --git a/modern-cpp-kafka/include/kafka/Utility.h b/modern-cpp-kafka/include/kafka/Utility.h new file mode 100644 index 00000000..88e62717 --- /dev/null +++ b/modern-cpp-kafka/include/kafka/Utility.h @@ -0,0 +1,81 @@ +#pragma once + +#include + +#include + +#include +#include +#include +#include +#include +#include + + +namespace KAFKA_API { namespace utility { + +/** + * Get local time as string. + */ +inline std::string getLocalTimeString(const std::chrono::system_clock::time_point& timePoint) +{ + auto time = std::chrono::system_clock::to_time_t(timePoint); + std::tm tmBuf = {}; + +#if !defined(WIN32) + localtime_r(&time, &tmBuf); +#else + localtime_s(&tmBuf, &time); +#endif + + std::ostringstream oss; + oss << std::put_time(&tmBuf, "%F %T") << "." << std::setfill('0') << std::setw(6) + << std::chrono::duration_cast(timePoint.time_since_epoch()).count() % 1000000; + + return oss.str(); +} + +/** + * Get current local time as string. + */ +inline std::string getCurrentTime() +{ + return getLocalTimeString(std::chrono::system_clock::now()); +} + +/** + * Get random string. + */ +inline std::string getRandomString() +{ + using namespace std::chrono; + std::uint32_t timestamp = static_cast(duration_cast(system_clock::now().time_since_epoch()).count()); + + std::random_device r; + std::default_random_engine e(r()); + std::uniform_int_distribution uniform_dist(0, 0xFFFFFFFF); + std::uint64_t rand = uniform_dist(e); + + std::ostringstream oss; + oss << std::setfill('0') << std::setw(sizeof(std::uint32_t) * 2) << std::hex << timestamp << "-" << rand; + return oss.str(); +} + +/** + * Get librdkafka version string. + */ +inline std::string getLibRdKafkaVersion() +{ + return rd_kafka_version_str(); +} + +/** + * Current number of threads created by rdkafka. + */ +inline int getLibRdKafkaThreadCount() +{ + return rd_kafka_thread_cnt(); +} + +} } // end of KAFKA_API::utility + diff --git a/modern-cpp-kafka/include/kafka/addons/KafkaMetrics.h b/modern-cpp-kafka/include/kafka/addons/KafkaMetrics.h new file mode 100644 index 00000000..cd31538a --- /dev/null +++ b/modern-cpp-kafka/include/kafka/addons/KafkaMetrics.h @@ -0,0 +1,208 @@ +#pragma once + +#include + +// https://github.com/Tencent/rapidjson/releases/tag/v1.1.0 +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + + +namespace KAFKA_API { + +/** + * \brief Helps to parse the metrics string with JSON format. + */ +class KafkaMetrics +{ +public: + /** + * \brief Initilize with the metrics string. + */ + explicit KafkaMetrics(std::string jsonMetrics); + + static const constexpr char* WILDCARD = "*"; + + using KeysType = std::vector; + + /** + * \brief The matched keys (for wildcards) and the value. + */ + template + using ResultsType = std::vector>; + + /** + * \brief Get integer value(s) for the specified metrics. + * Note: the wildcard ("*") is supported. + */ + ResultsType getInt(const KeysType& keys) { return get(keys); } + + /** + * \brief Get string value(s) for the specified metrics. + * Note: the wildcard ("*") is supported. + */ + ResultsType getString(const KeysType& keys) { return get(keys); } + + static std::string toString(const KafkaMetrics::KeysType& keys); + + template + static std::string toString(const KafkaMetrics::ResultsType& results); + +private: + template + ResultsType get(const KeysType& keys); + + template + void getResults(ResultsType& results, + KeysType& keysForWildcards, + rapidjson::Value::ConstMemberIterator iter, + KeysType::const_iterator keysToParse, + KeysType::const_iterator keysEnd); + + template + static ValueType getValue(rapidjson::Value::ConstMemberIterator iter); + +#if COMPILER_SUPPORTS_CPP_17 + std::string _decodeBuf; +#else + std::vector _decodeBuf; +#endif + rapidjson::Document _jsonDoc; +}; + +inline +KafkaMetrics::KafkaMetrics(std::string jsonMetrics) +#if COMPILER_SUPPORTS_CPP_17 + : _decodeBuf(std::move(jsonMetrics)) +#else + : _decodeBuf(jsonMetrics.cbegin(), jsonMetrics.cend() + 1) +#endif +{ + if (_jsonDoc.ParseInsitu(_decodeBuf.data()).HasParseError()) + { + throw std::runtime_error("Failed to parse string with JSON format!"); + } +} + +template<> +inline std::int64_t +KafkaMetrics::getValue(rapidjson::Value::ConstMemberIterator iter) +{ + return iter->value.GetInt(); +} + +template<> +inline std::string +KafkaMetrics::getValue(rapidjson::Value::ConstMemberIterator iter) +{ + return iter->value.GetString(); +} + +template +inline KafkaMetrics::ResultsType +KafkaMetrics::get(const KeysType& keys) +{ + if (keys.empty()) throw std::invalid_argument("Input keys cannot be empty!"); + if (keys.front() == WILDCARD) throw std::invalid_argument("The first key cannot be wildcard!"); + if (keys.back() == WILDCARD) throw std::invalid_argument("The last key cannot be wildcard!"); + + ResultsType results; + + rapidjson::Value::ConstMemberIterator iter = _jsonDoc.FindMember(keys.front().c_str()); + if (iter == _jsonDoc.MemberEnd()) return results; + + if (keys.size() == 1) + { + if (std::is_same::value ? iter->value.IsString() : iter->value.IsInt()) + { + results.emplace_back(KeysType{}, getValue(iter)); + } + + return results; + } + + KeysType keysForWildcards; + + getResults(results, keysForWildcards, iter, keys.cbegin() + 1, keys.cend()); + return results; +} + +template +inline void +KafkaMetrics::getResults(KafkaMetrics::ResultsType& results, + KeysType& keysForWildcards, + rapidjson::Value::ConstMemberIterator iter, + KeysType::const_iterator keysToParse, + KeysType::const_iterator keysEnd) +{ + if (!iter->value.IsObject()) return; + + const auto& key = *(keysToParse++); + const bool isTheEnd = (keysToParse == keysEnd); + + if (key == WILDCARD) + { + for (rapidjson::Value::ConstMemberIterator subIter = iter->value.MemberBegin(); subIter != iter->value.MemberEnd(); ++subIter) + { + KeysType newKeysForWildcards = keysForWildcards; + newKeysForWildcards.emplace_back(subIter->name.GetString()); + + getResults(results, newKeysForWildcards, subIter, keysToParse, keysEnd); + } + } + else + { + rapidjson::Value::ConstMemberIterator subIter = iter->value.FindMember(key.c_str()); + if (subIter == iter->value.MemberEnd()) return; + + if (!isTheEnd) + { + getResults(results, keysForWildcards, subIter, keysToParse, keysEnd); + } + else if (std::is_same::value ? subIter->value.IsString() : subIter->value.IsInt()) + { + results.emplace_back(keysForWildcards, getValue(subIter)); + } + } +} + +inline std::string +KafkaMetrics::toString(const KafkaMetrics::KeysType& keys) +{ + std::string ret; + + std::for_each(keys.cbegin(), keys.cend(), + [&ret](const auto& key){ ret.append((ret.empty() ? std::string() : std::string(", ")) + "\"" + key + "\""); }); + + return ret; +} + +template +inline std::string +KafkaMetrics::toString(const KafkaMetrics::ResultsType& results) +{ + std::ostringstream oss; + bool isTheFirstOne = true; + + std::for_each(results.cbegin(), results.cend(), + [&oss, &isTheFirstOne](const auto& result) { + const auto keysString = toString(result.first); + + oss << (isTheFirstOne ? (isTheFirstOne = false, "") : ", ") + << (keysString.empty() ? "" : (std::string("[") + keysString + "]:")); + oss << (std::is_same::value ? "\"" : "") << result.second << (std::is_same::value ? "\"" : ""); + }); + + return oss.str(); +} + +} // end of KAFKA_API + diff --git a/modern-cpp-kafka/include/kafka/addons/KafkaRecoverableProducer.h b/modern-cpp-kafka/include/kafka/addons/KafkaRecoverableProducer.h new file mode 100644 index 00000000..911f832f --- /dev/null +++ b/modern-cpp-kafka/include/kafka/addons/KafkaRecoverableProducer.h @@ -0,0 +1,360 @@ +#pragma once + +#include + +#include +#include +#include + +#include +#include +#include + +namespace KAFKA_API { namespace clients { + +class KafkaRecoverableProducer +{ +public: + explicit KafkaRecoverableProducer(const Properties& properties) + : _properties(properties), _running(true) + { + _errorCb = [this](const Error& error) { + if (error.isFatal()) _fatalError = std::make_unique(error); + }; + + _producer = createProducer(); + + _pollThread = std::thread([this]() { keepPolling(); }); + } + + ~KafkaRecoverableProducer() + { + if (_running) close(); + } + + /** + * Get the client id. + */ + const std::string& clientId() const + { + std::lock_guard lock(_producerMutex); + + return _producer->clientId(); + } + + /** + * Get the client name (i.e. client type + id). + */ + const std::string& name() const + { + std::lock_guard lock(_producerMutex); + + return _producer->name(); + } + + /** + * Set the log callback for the kafka client (it's a per-client setting). + */ + void setLogger(const Logger& logger) + { + std::lock_guard lock(_producerMutex); + + _logger = logger; + _producer->setLogger(*_logger); + } + + /** + * Set log level for the kafka client (the default value: 5). + */ + void setLogLevel(int level) + { + std::lock_guard lock(_producerMutex); + + _logLevel = level; + _producer->setLogLevel(*_logLevel); + } + + /** + * Set callback to receive the periodic statistics info. + * Note: 1) It only works while the "statistics.interval.ms" property is configured with a non-0 value. + * 2) The callback would be triggered periodically, receiving the internal statistics info (with JSON format) emited from librdkafka. + */ + void setStatsCallback(const KafkaClient::StatsCallback& cb) + { + std::lock_guard lock(_producerMutex); + + _statsCb = cb; + _producer->setStatsCallback(*_statsCb); + } + + void setErrorCallback(const KafkaClient::ErrorCallback& cb) + { + std::lock_guard lock(_producerMutex); + + _errorCb = [cb, this](const Error& error) { + cb(error); + + if (error.isFatal()) _fatalError = std::make_unique(error); + }; + _producer->setErrorCallback(*_errorCb); + } + + /** + * Return the properties which took effect. + */ + const Properties& properties() const + { + std::lock_guard lock(_producerMutex); + + return _producer->properties(); + } + + /** + * Fetch the effected property (including the property internally set by librdkafka). + */ + Optional getProperty(const std::string& name) const + { + std::lock_guard lock(_producerMutex); + + return _producer->getProperty(name); + } + + /** + * Fetch matadata from a available broker. + * Note: the Metadata response information may trigger a re-join if any subscribed topic has changed partition count or existence state. + */ + Optional fetchBrokerMetadata(const std::string& topic, + std::chrono::milliseconds timeout = std::chrono::milliseconds(KafkaClient::DEFAULT_METADATA_TIMEOUT_MS), + bool disableErrorLogging = false) + { + std::lock_guard lock(_producerMutex); + + return _producer->fetchBrokerMetadata(topic, timeout, disableErrorLogging); + } + + /** + * Invoking this method makes all buffered records immediately available to send, and blocks on the completion of the requests associated with these records. + * + * Possible error values: + * - RD_KAFKA_RESP_ERR__TIMED_OUT: The `timeout` was reached before all outstanding requests were completed. + */ + Error flush(std::chrono::milliseconds timeout = std::chrono::milliseconds::max()) + { + std::lock_guard lock(_producerMutex); + + return _producer->flush(timeout); + } + + /** + * Purge messages currently handled by the KafkaProducer. + */ + Error purge() + { + std::lock_guard lock(_producerMutex); + + return _producer->purge(); + } + + /** + * Close this producer. This method would wait up to timeout for the producer to complete the sending of all incomplete requests (before purging them). + */ + void close(std::chrono::milliseconds timeout = std::chrono::milliseconds::max()) + { + std::lock_guard lock(_producerMutex); + + _running = false; + if (_pollThread.joinable()) _pollThread.join(); + + _producer->close(timeout); + } + + /** + * Synchronously send a record to a topic. + * Throws KafkaException with errors: + * Local errors, + * - RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC: The topic doesn't exist + * - RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION: The partition doesn't exist + * - RD_KAFKA_RESP_ERR__INVALID_ARG: Invalid topic(topic is null, or the length is too long (> 512) + * - RD_KAFKA_RESP_ERR__MSG_TIMED_OUT: No ack received within the time limit + * Broker errors, + * - [Error Codes] (https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-ErrorCodes) + */ + producer::RecordMetadata syncSend(const producer::ProducerRecord& record) + { + std::lock_guard lock(_producerMutex); + + return _producer->syncSend(record); + } + + /** + * Asynchronously send a record to a topic. + * + * Note: + * - If a callback is provided, it's guaranteed to be triggered (before closing the producer). + * - If any error occured, an exception would be thrown. + * - Make sure the memory block (for ProducerRecord's value) is valid until the delivery callback finishes; Otherwise, should be with option `KafkaProducer::SendOption::ToCopyRecordValue`. + * + * Possible errors: + * Local errors, + * - RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC: The topic doesn't exist + * - RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION: The partition doesn't exist + * - RD_KAFKA_RESP_ERR__INVALID_ARG: Invalid topic(topic is null, or the length is too long (> 512) + * - RD_KAFKA_RESP_ERR__MSG_TIMED_OUT: No ack received within the time limit + * - RD_KAFKA_RESP_ERR__QUEUE_FULL: The message buffing queue is full + * Broker errors, + * - [Error Codes] (https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-ErrorCodes) + */ + void send(const producer::ProducerRecord& record, + const producer::Callback& deliveryCb, + KafkaProducer::SendOption option = KafkaProducer::SendOption::NoCopyRecordValue, + KafkaProducer::ActionWhileQueueIsFull action = KafkaProducer::ActionWhileQueueIsFull::Block) + { + std::lock_guard lock(_producerMutex); + + _producer->send(record, deliveryCb, option, action); + } + + /** + * Asynchronously send a record to a topic. + * + * Note: + * - If a callback is provided, it's guaranteed to be triggered (before closing the producer). + * - The input reference parameter `error` will be set if an error occurred. + * - Make sure the memory block (for ProducerRecord's value) is valid until the delivery callback finishes; Otherwise, should be with option `KafkaProducer::SendOption::ToCopyRecordValue`. + * + * Possible errors: + * Local errors, + * - RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC: The topic doesn't exist + * - RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION: The partition doesn't exist + * - RD_KAFKA_RESP_ERR__INVALID_ARG: Invalid topic(topic is null, or the length is too long (> 512) + * - RD_KAFKA_RESP_ERR__MSG_TIMED_OUT: No ack received within the time limit + * - RD_KAFKA_RESP_ERR__QUEUE_FULL: The message buffing queue is full + * Broker errors, + * - [Error Codes] (https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-ErrorCodes) + */ + + void send(const producer::ProducerRecord& record, + const producer::Callback& deliveryCb, + Error& error, + KafkaProducer::SendOption option = KafkaProducer::SendOption::NoCopyRecordValue, + KafkaProducer::ActionWhileQueueIsFull action = KafkaProducer::ActionWhileQueueIsFull::Block) + { + std::lock_guard lock(_producerMutex); + + _producer->send(record, deliveryCb, error, option, action); + } + + /** + * Needs to be called before any other methods when the transactional.id is set in the configuration. + */ + void initTransactions(std::chrono::milliseconds timeout = std::chrono::milliseconds(KafkaProducer::DEFAULT_INIT_TRANSACTIONS_TIMEOUT_MS)) + { + std::lock_guard lock(_producerMutex); + + _producer->initTransactions(timeout); + } + + /** + * Should be called before the start of each new transaction. + */ + void beginTransaction() + { + std::lock_guard lock(_producerMutex); + + _producer->beginTransaction(); + } + + /** + * Commit the ongoing transaction. + */ + void commitTransaction(std::chrono::milliseconds timeout = std::chrono::milliseconds(KafkaProducer::DEFAULT_COMMIT_TRANSACTION_TIMEOUT_MS)) + { + std::lock_guard lock(_producerMutex); + + _producer->commitTransaction(timeout); + } + + /** + * Abort the ongoing transaction. + */ + void abortTransaction(std::chrono::milliseconds timeout = std::chrono::milliseconds::max()) + { + std::lock_guard lock(_producerMutex); + + _producer->abortTransaction(timeout); + } + + /** + * Send a list of specified offsets to the consumer group coodinator, and also marks those offsets as part of the current transaction. + */ + void sendOffsetsToTransaction(const TopicPartitionOffsets& topicPartitionOffsets, + const consumer::ConsumerGroupMetadata& groupMetadata, + std::chrono::milliseconds timeout) + { + std::lock_guard lock(_producerMutex); + + _producer->sendOffsetsToTransaction(topicPartitionOffsets, groupMetadata, timeout); + } + +#ifdef KAFKA_API_ENABLE_UNIT_TEST_STUBS + void mockFatalError() + { + _fatalError = std::make_unique(RD_KAFKA_RESP_ERR__FATAL, "fake fatal error", true); + } +#endif + +private: + void keepPolling() + { + while (_running) + { + _producer->pollEvents(std::chrono::milliseconds(1)); + if (_fatalError) + { + const std::string errStr = _fatalError->toString(); + KAFKA_API_LOG(Log::Level::Notice, "met fatal error[%s], will re-initialize the internal producer", errStr.c_str()); + + std::lock_guard lock(_producerMutex); + + if (!_running) return; + + _producer->purge(); + _producer->close(); + + _fatalError.reset(); + + _producer = createProducer(); + } + } + } + + std::unique_ptr createProducer() + { + auto producer = std::make_unique(_properties, KafkaClient::EventsPollingOption::Manual); + + if (_logger) producer->setLogger(*_logger); + if (_logLevel) producer->setLogLevel(*_logLevel); + if (_statsCb) producer->setStatsCallback(*_statsCb); + if (_errorCb) producer->setErrorCallback(*_errorCb); + + return producer; + } + + // Configurations for producer + Properties _properties; + Optional _logger; + Optional _logLevel; + Optional _statsCb; + Optional _errorCb; + + std::unique_ptr _fatalError; + + std::atomic _running; + std::thread _pollThread; + + mutable std::mutex _producerMutex; + std::unique_ptr _producer; +}; + +} } // end of KAFKA_API::clients + diff --git a/modern-cpp-kafka/include/kafka/addons/UnorderedOffsetCommitQueue.h b/modern-cpp-kafka/include/kafka/addons/UnorderedOffsetCommitQueue.h new file mode 100644 index 00000000..756b00fb --- /dev/null +++ b/modern-cpp-kafka/include/kafka/addons/UnorderedOffsetCommitQueue.h @@ -0,0 +1,178 @@ +#pragma once + +#include + +#include +#include + +#include +#include + +namespace KAFKA_API { namespace clients { namespace consumer { + +template +class Heap +{ +public: + bool empty() const { return data.empty(); } + std::size_t size() const { return data.size(); } + + const T& front() const { return data[0]; } + + void push(const T& t) + { + data.emplace_back(t); + + for (std::size_t indexCurrent = data.size() - 1; indexCurrent > 0;) + { + std::size_t indexParent = (indexCurrent + 1) / 2 - 1; + + if (!(data[indexCurrent] < data[indexParent])) return; + + std::swap(data[indexCurrent], data[indexParent]); + indexCurrent = indexParent; + } + } + + void pop_front() + { + data[0] = data.back(); + data.pop_back(); + + if (data.empty()) return; + + for (std::size_t indexCurrent = 0;;) + { + std::size_t indexRightChild = (indexCurrent + 1) * 2; + std::size_t indexLeftChild = indexRightChild - 1; + + if (indexLeftChild >= data.size()) return; + + std::size_t indexMinChild = (indexRightChild >= data.size() || data[indexLeftChild] < data[indexRightChild]) ? indexLeftChild : indexRightChild; + + if (!(data[indexMinChild] < data[indexCurrent])) return; + + std::swap(data[indexCurrent], data[indexMinChild]); + indexCurrent = indexMinChild; + } + } + +private: + std::vector data; +}; + + +/** + * \brief The queue can be used to determine the right offset to commit. + * A `KafkaManuallyCommitConsumer` might forward the received records to different handlers, while these handlers could not ack the records in order. + * Then, the `UnorderedOffsetCommitQueue` would help, + * 1. Prepare an `UnorderedOffsetCommitQueue` for each topic-partition. + * 2. Make sure call `waitOffset()` for each record received. + * 3. Make sure call `ackOffset()` while a handler acks for an record. + * 4. Figure out whether there's offset to commit with `popOffsetToCommit()` and commit the offset then. + */ +class UnorderedOffsetCommitQueue +{ +public: + UnorderedOffsetCommitQueue(const Topic& topic, Partition partition) + : _partitionInfo(std::string("topic[").append(topic).append("], paritition[").append(std::to_string(partition)).append("]")) + { + } + UnorderedOffsetCommitQueue() = default; + + /** + * \brief Return how many received offsets have not been popped to commit (with `popOffsetToCommit()`). + */ + std::size_t size() const { return _offsetsReceived.size(); } + + /** + * \brief Add an offset (for a ConsumerRecord) to the waiting list, until it being acked (with `ackOffset`). + * Note: Make sure the offset would be `ack` later with `ackOffset()`. + */ + void waitOffset(Offset offset) + { + if (offset < 0 || (!_offsetsReceived.empty() && offset <= _offsetsReceived.back())) + { + // Invalid offset (might be fetched from the record which had no valid offset) + KAFKA_API_LOG(Log::Level::Err, "Got invalid offset to wait[%lld]! %s", offset, (_partitionInfo.empty() ? "" : _partitionInfo.c_str())); + return; + } + + _offsetsReceived.emplace_back(offset); + } + + /** + * \brief Ack the record has been handled and ready to be committed. + * Note: If all offsets ahead has been acked, then with `popOffsetToCommit()`, we'd get `offset + 1`, which is ready to be committed for the consumer. + */ + void ackOffset(Offset offset) + { + Offset maxOffsetReceived = _offsetsReceived.back(); + if (offset > maxOffsetReceived) + { + // Runtime error + KAFKA_API_LOG(Log::Level::Err, "Got invalid ack offset[%lld]! Even larger than all offsets received[%lld]! %s", offset, maxOffsetReceived, (_partitionInfo.empty() ? "" : _partitionInfo.c_str())); + } + + _offsetsToCommit.push(offset); + do + { + Offset minOffsetToCommit = _offsetsToCommit.front(); + Offset expectedOffset = _offsetsReceived.front(); + if (minOffsetToCommit == expectedOffset) + { + _toCommit = expectedOffset + 1; + _offsetsToCommit.pop_front(); + _offsetsReceived.pop_front(); + } + else if (minOffsetToCommit < expectedOffset) + { + // Inconsist error (might be caused by duplicated ack) + KAFKA_API_LOG(Log::Level::Err, "Got invalid ack offset[%lld]! Even smaller than expected[%lld]! %s", minOffsetToCommit, expectedOffset, (_partitionInfo.empty() ? "" : _partitionInfo.c_str())); + _offsetsToCommit.pop_front(); + } + else + { + break; + } + } while (!_offsetsToCommit.empty()); + } + + /** + * \brief Pop the offset which is ready for the consumer (if any). + */ + Optional popOffsetToCommit() + { + Optional ret; + if (_committed != _toCommit) + { + ret = _committed = _toCommit; + } + return ret; + } + + /** + * \brief Return the offset last popped. + */ + Optional lastPoppedOffset() + { + Optional ret; + if (_committed != INVALID_OFFSET) + { + ret = _committed; + } + return ret; + } + +private: + std::deque _offsetsReceived; + Heap _offsetsToCommit; + Offset _toCommit = {INVALID_OFFSET}; + Offset _committed = {INVALID_OFFSET}; + std::string _partitionInfo; + + static constexpr Offset INVALID_OFFSET = -1; +}; + +} } } // end of KAFKA_API::clients::consumer +