diff --git a/CMakeLists.txt b/CMakeLists.txt index 6cf96f5..997c2ef 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,6 +28,7 @@ string(REPLACE ";" "${AWS_MODULE_DIR};" AWS_MODULE_PATH "${CMAKE_PREFIX_PATH}${A # Append that generated list to the module search path list(APPEND CMAKE_MODULE_PATH ${AWS_MODULE_PATH}) +include(AwsSIMD) include(AwsCFlags) include(AwsCheckHeaders) include(AwsSharedLibSetup) @@ -58,17 +59,48 @@ file(GLOB AWS_ARCH_SRC ) if (USE_CPU_EXTENSIONS) - if(AWS_ARCH_INTEL) - # First, check if inline assembly is available. Inline assembly can also be supported by MSVC if the compiler in use is Clang. - if(AWS_HAVE_GCC_INLINE_ASM) - file(GLOB AWS_ARCH_SRC - "source/intel/asm/*.c" + if (AWS_ARCH_INTEL) + file (GLOB AWS_ARCH_INTEL_SRC + "source/intel/*.c" + ) + + if (AWS_HAVE_AVX512_INTRINSICS AND CMAKE_SIZEOF_VOID_P EQUAL 8) + if (MSVC) + file(GLOB AWS_ARCH_INTRIN_SRC + "source/intel/intrin/*.c" + "source/intel/visualc/*.c" ) - elseif (MSVC) - file(GLOB AWS_ARCH_SRC + else() + file(GLOB AWS_ARCH_INTRIN_SRC + "source/intel/intrin/*.c" + ) + endif() + else() + if (MSVC) + file(GLOB AWS_ARCH_INTRIN_SRC "source/intel/visualc/*.c" + ) + endif() + endif() + + source_group("Source Files\\intel" FILES ${AWS_ARCH_INTEL_SRC}) + source_group("Source Files\\intel\\intrin" FILES ${AWS_ARCH_INTRIN_SRC}) + + if (AWS_HAVE_GCC_INLINE_ASM) + file(GLOB AWS_ARCH_ASM_SRC + "source/intel/asm/*.c" + ) + + file(GLOB AWS_ARCH_SRC + ${AWS_ARCH_INTEL_SRC} + ${AWS_ARCH_INTRIN_SRC} + ${AWS_ARCH_ASM_SRC} + ) + else() + file(GLOB AWS_ARCH_SRC + ${AWS_ARCH_INTEL_SRC} + ${AWS_ARCH_INTRIN_SRC} ) - source_group("Source Files\\intel\\visualc" FILES ${AWS_ARCH_SRC}) endif() endif() @@ -115,6 +147,7 @@ file(GLOB CHECKSUMS_COMBINED_SRC add_library(${PROJECT_NAME} ${CHECKSUMS_COMBINED_HEADERS} ${CHECKSUMS_COMBINED_SRC}) + aws_set_common_properties(${PROJECT_NAME}) aws_prepare_symbol_visibility_args(${PROJECT_NAME} "AWS_CHECKSUMS") aws_check_headers(${PROJECT_NAME} ${AWS_CHECKSUMS_HEADERS}) @@ -124,6 +157,10 @@ aws_add_sanitizers(${PROJECT_NAME}) # We are not ABI stable yet set_target_properties(${PROJECT_NAME} PROPERTIES VERSION 1.0.0) +if (USE_CPU_EXTENSIONS AND AWS_ARCH_INTEL) + simd_add_source_avx(${PROJECT_NAME} ${AWS_ARCH_SRC}) +endif() + target_include_directories(${PROJECT_NAME} PUBLIC $ $) diff --git a/include/aws/checksums/private/intel/crc32c_compiler_shims.h b/include/aws/checksums/private/intel/crc32c_compiler_shims.h new file mode 100644 index 0000000..21002de --- /dev/null +++ b/include/aws/checksums/private/intel/crc32c_compiler_shims.h @@ -0,0 +1,25 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +#include + +#include +#include + +#if _WIN64 || __x86_64__ || __ppc64_ +typedef uint64_t *slice_ptr_type; +typedef uint64_t slice_ptr_int_type; +# define crc_intrin_fn _mm_crc32_u64 +#else +typedef uint32_t *slice_ptr_type; +typedef uint32_t slice_ptr_int_type; +# define crc_intrin_fn _mm_crc32_u32 +#endif + +#ifdef AWS_HAVE_AVX512_INTRINSICS +uint32_t aws_checksums_crc32c_avx512(const uint8_t *input, int length, uint32_t crc); +#endif + +uint32_t aws_checksums_crc32c_sse42(const uint8_t *input, int length, uint32_t crc); diff --git a/source/intel/asm/crc32c_sse42_asm.c b/source/intel/asm/crc32c_sse42_asm.c index 35e1d09..bc79597 100644 --- a/source/intel/asm/crc32c_sse42_asm.c +++ b/source/intel/asm/crc32c_sse42_asm.c @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0. */ -#include +#include #include @@ -283,7 +283,7 @@ static bool detected_clmul = false; * Pass 0 in the previousCrc32 parameter as an initial value unless continuing to update a running CRC in a subsequent * call. */ -uint32_t aws_checksums_crc32c_hw(const uint8_t *input, int length, uint32_t previousCrc32) { +uint32_t aws_checksums_crc32c_sse42(const uint8_t *input, int length, uint32_t previousCrc32) { if (AWS_UNLIKELY(!detection_performed)) { detected_clmul = aws_cpu_has_feature(AWS_CPU_FEATURE_CLMUL); @@ -293,7 +293,8 @@ uint32_t aws_checksums_crc32c_hw(const uint8_t *input, int length, uint32_t prev detection_performed = true; } - uint32_t crc = ~previousCrc32; + /* this is called by a higher-level shim and previousCRC32 is already ~ */ + uint32_t crc = previousCrc32; /* For small input, forget about alignment checks - simply compute the CRC32c one byte at a time */ if (AWS_UNLIKELY(length < 8)) { @@ -358,22 +359,17 @@ uint32_t aws_checksums_crc32c_hw(const uint8_t *input, int length, uint32_t prev return ~crc; } -uint32_t aws_checksums_crc32_hw(const uint8_t *input, int length, uint32_t previousCrc32) { - return aws_checksums_crc32_sw(input, length, previousCrc32); -} # if defined(__clang__) # pragma clang diagnostic pop # endif #else -uint32_t aws_checksums_crc32_hw(const uint8_t *input, int length, uint32_t previousCrc32) { - return aws_checksums_crc32_sw(input, length, previousCrc32); -} - -uint32_t aws_checksums_crc32c_hw(const uint8_t *input, int length, uint32_t previousCrc32) { - return aws_checksums_crc32c_sw(input, length, previousCrc32); +uint32_t aws_checksums_crc32c_sse42(const uint8_t *input, int length, uint32_t previousCrc32) { + /* these are nested in a larger computation. As a result the crc doesn't need to be bit flipped. + However, the sw function is also used as a standalone implementation that does need to do the + bit flip. So go ahead and flip it here, so the sw implementation flips it back. */ + return aws_checksums_crc32c_sw(input, length, ~previousCrc32); } - #endif /* clang-format on */ diff --git a/source/intel/crc_hw.c b/source/intel/crc_hw.c new file mode 100644 index 0000000..d571cc0 --- /dev/null +++ b/source/intel/crc_hw.c @@ -0,0 +1,101 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include +#include + +static bool detection_performed = false; +static bool detected_sse42 = false; +static bool detected_avx512 = false; +static bool detected_clmul = false; +static bool detected_vpclmulqdq = false; + +/* + * Computes the Castagnoli CRC32c (iSCSI) of the specified data buffer using the Intel CRC32Q (64-bit quad word) and + * PCLMULQDQ machine instructions (if present). + * Handles data that isn't 8-byte aligned as well as any trailing data with the CRC32B (byte) instruction. + * Pass 0 in the previousCrc32 parameter as an initial value unless continuing to update a running CRC in a subsequent + * call. + */ +uint32_t aws_checksums_crc32c_hw(const uint8_t *input, int length, uint32_t previousCrc32) { + + if (AWS_UNLIKELY(!detection_performed)) { + detected_sse42 = aws_cpu_has_feature(AWS_CPU_FEATURE_SSE_4_2); + detected_avx512 = aws_cpu_has_feature(AWS_CPU_FEATURE_AVX512); + detected_clmul = aws_cpu_has_feature(AWS_CPU_FEATURE_CLMUL); + detected_vpclmulqdq = aws_cpu_has_feature(AWS_CPU_FEATURE_VPCLMULQDQ); + + /* Simply setting the flag true to skip HW detection next time + Not using memory barriers since the worst that can + happen is a fallback to the non HW accelerated code. */ + detection_performed = true; + } + + /* this is the entry point. We should only do the bit flip once. It should not be done for the subfunctions and + * branches.*/ + uint32_t crc = ~previousCrc32; + + /* For small input, forget about alignment checks - simply compute the CRC32c one byte at a time */ + if (length < (int)sizeof(slice_ptr_int_type)) { + while (length-- > 0) { + crc = (uint32_t)_mm_crc32_u8(crc, *input++); + } + return ~crc; + } + + /* Get the 8-byte memory alignment of our input buffer by looking at the least significant 3 bits */ + int input_alignment = (uintptr_t)(input)&0x7; + + /* Compute the number of unaligned bytes before the first aligned 8-byte chunk (will be in the range 0-7) */ + int leading = (8 - input_alignment) & 0x7; + + /* reduce the length by the leading unaligned bytes we are about to process */ + length -= leading; + + /* spin through the leading unaligned input bytes (if any) one-by-one */ + while (leading-- > 0) { + crc = (uint32_t)_mm_crc32_u8(crc, *input++); + } + +#if defined(AWS_HAVE_AVX512_INTRINSICS) && (INTPTR_MAX == INT64_MAX) + int chunk_size = length & ~63; + + if (detected_avx512 && detected_vpclmulqdq && detected_clmul) { + if (length >= 256) { + crc = aws_checksums_crc32c_avx512(input, length, crc); + /* check remaining data */ + length -= chunk_size; + if (!length) { + return ~crc; + } + + /* Fall into the default crc32 for the remaining data. */ + input += chunk_size; + } + } +#endif + + if (detected_sse42 && detected_clmul) { + return aws_checksums_crc32c_sse42(input, length, crc); + } + + /* Spin through remaining (aligned) 8-byte chunks using the CRC32Q quad word instruction */ + while (length >= (int)sizeof(slice_ptr_int_type)) { + crc = (uint32_t)crc_intrin_fn(crc, *input); + input += sizeof(slice_ptr_int_type); + length -= (int)sizeof(slice_ptr_int_type); + } + + /* Finish up with any trailing bytes using the CRC32B single byte instruction one-by-one */ + while (length-- > 0) { + crc = (uint32_t)_mm_crc32_u8(crc, *input); + input++; + } + + return ~crc; +} + +uint32_t aws_checksums_crc32_hw(const uint8_t *input, int length, uint32_t previousCrc32) { + return aws_checksums_crc32_sw(input, length, previousCrc32); +} diff --git a/source/intel/intrin/crc32c_sse42_avx512.c b/source/intel/intrin/crc32c_sse42_avx512.c new file mode 100644 index 0000000..837a1ba --- /dev/null +++ b/source/intel/intrin/crc32c_sse42_avx512.c @@ -0,0 +1,152 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +#include + +#include +#include + +#include +#include +#include +#include + +AWS_ALIGNED_TYPEDEF(const uint64_t, zalign_8, 64); +AWS_ALIGNED_TYPEDEF(const uint64_t, zalign_2, 16); + +/* + * crc32c_avx512(): compute the crc32c of the buffer, where the buffer + * length must be at least 256, and a multiple of 64. Based on: + * + * "Fast CRC Computation for Generic Polynomials Using PCLMULQDQ Instruction" + * V. Gopal, E. Ozturk, et al., 2009, http://download.intel.com/design/intarch/papers/323102.pdf + */ +uint32_t aws_checksums_crc32c_avx512(const uint8_t *input, int length, uint32_t previous_crc) { + AWS_ASSERT( + length >= 256 && "invariant violated. length must be greater than 255 bytes to use avx512 to compute crc."); + + uint32_t crc = ~previous_crc; + /* + * Definitions of the bit-reflected domain constants k1,k2,k3,k4,k5,k6 + * are similar to those given at the end of the paper + * + * k1 = ( x ^ ( 512 * 4 + 32 ) mod P(x) << 32 )' << 1 + * k2 = ( x ^ ( 512 * 4 - 32 ) mod P(x) << 32 )' << 1 + * k3 = ( x ^ ( 512 + 32 ) mod P(x) << 32 )' << 1 + * k4 = ( x ^ ( 512 - 32 ) mod P(x) << 32 )' << 1 + * k5 = ( x ^ ( 128 + 32 ) mod P(x) << 32 )' << 1 + * k6 = ( x ^ ( 128 - 32 ) mod P(x) << 32 )' << 1 + */ + + static zalign_8 k1k2[8] = { + 0xdcb17aa4, 0xb9e02b86, 0xdcb17aa4, 0xb9e02b86, 0xdcb17aa4, 0xb9e02b86, 0xdcb17aa4, 0xb9e02b86}; + + static zalign_8 k3k4[8] = { + 0x740eef02, 0x9e4addf8, 0x740eef02, 0x9e4addf8, 0x740eef02, 0x9e4addf8, 0x740eef02, 0x9e4addf8}; + static zalign_8 k9k10[8] = { + 0x6992cea2, 0x0d3b6092, 0x6992cea2, 0x0d3b6092, 0x6992cea2, 0x0d3b6092, 0x6992cea2, 0x0d3b6092}; + static zalign_8 k1k4[8] = { + 0x1c291d04, 0xddc0152b, 0x3da6d0cb, 0xba4fc28e, 0xf20c0dfe, 0x493c7d27, 0x00000000, 0x00000000}; + + + __m512i x0, x1, x2, x3, x4, x5, x6, x7, x8, y5, y6, y7, y8; + __m128i a1, a2; + + /* + * There's at least one block of 256. + */ + x1 = _mm512_loadu_si512((__m512i *)(input + 0x00)); + x2 = _mm512_loadu_si512((__m512i *)(input + 0x40)); + x3 = _mm512_loadu_si512((__m512i *)(input + 0x80)); + x4 = _mm512_loadu_si512((__m512i *)(input + 0xC0)); + + x1 = _mm512_xor_si512(x1, _mm512_castsi128_si512(_mm_cvtsi32_si128(crc))); + + x0 = _mm512_load_si512((__m512i *)k1k2); + + input += 256; + length -= 256; + + /* + * Parallel fold blocks of 256, if any. + */ + while (length >= 256) { + x5 = _mm512_clmulepi64_epi128(x1, x0, 0x00); + x6 = _mm512_clmulepi64_epi128(x2, x0, 0x00); + x7 = _mm512_clmulepi64_epi128(x3, x0, 0x00); + x8 = _mm512_clmulepi64_epi128(x4, x0, 0x00); + + x1 = _mm512_clmulepi64_epi128(x1, x0, 0x11); + x2 = _mm512_clmulepi64_epi128(x2, x0, 0x11); + x3 = _mm512_clmulepi64_epi128(x3, x0, 0x11); + x4 = _mm512_clmulepi64_epi128(x4, x0, 0x11); + + y5 = _mm512_loadu_si512((__m512i *)(input + 0x00)); + y6 = _mm512_loadu_si512((__m512i *)(input + 0x40)); + y7 = _mm512_loadu_si512((__m512i *)(input + 0x80)); + y8 = _mm512_loadu_si512((__m512i *)(input + 0xC0)); + + x1 = _mm512_ternarylogic_epi64(x1, x5, y5, 0x96); + x2 = _mm512_ternarylogic_epi64(x2, x6, y6, 0x96); + x3 = _mm512_ternarylogic_epi64(x3, x7, y7, 0x96); + x4 = _mm512_ternarylogic_epi64(x4, x8, y8, 0x96); + + input += 256; + length -= 256; + } + + /* + * Fold 256 bytes into 64 bytes. + */ + x0 = _mm512_load_si512((__m512i *)k9k10); + x5 = _mm512_clmulepi64_epi128(x1, x0, 0x00); + x6 = _mm512_clmulepi64_epi128(x1, x0, 0x11); + x3 = _mm512_ternarylogic_epi64(x3, x5, x6, 0x96); + + x7 = _mm512_clmulepi64_epi128(x2, x0, 0x00); + x8 = _mm512_clmulepi64_epi128(x2, x0, 0x11); + x4 = _mm512_ternarylogic_epi64(x4, x7, x8, 0x96); + + x0 = _mm512_load_si512((__m512i *)k3k4); + y5 = _mm512_clmulepi64_epi128(x3, x0, 0x00); + y6 = _mm512_clmulepi64_epi128(x3, x0, 0x11); + x1 = _mm512_ternarylogic_epi64(x4, y5, y6, 0x96); + + /* + * Single fold blocks of 64, if any. + */ + while (length >= 64) { + x2 = _mm512_loadu_si512((__m512i *)input); + + x5 = _mm512_clmulepi64_epi128(x1, x0, 0x00); + x1 = _mm512_clmulepi64_epi128(x1, x0, 0x11); + x1 = _mm512_ternarylogic_epi64(x1, x2, x5, 0x96); + + input += 64; + length -= 64; + } + + /* + * Fold 512-bits to 128-bits. + */ + x0 = _mm512_loadu_si512((__m512i *)k1k4); + + a2 = _mm512_extracti32x4_epi32(x1, 3); + x5 = _mm512_clmulepi64_epi128(x1, x0, 0x00); + x1 = _mm512_clmulepi64_epi128(x1, x0, 0x11); + x1 = _mm512_ternarylogic_epi64(x1, x5, _mm512_castsi128_si512(a2), 0x96); + + x0 = _mm512_shuffle_i64x2(x1, x1, 0x4E); + x0 = _mm512_xor_epi64(x1, x0); + a1 = _mm512_extracti32x4_epi32(x0, 1); + a1 = _mm_xor_epi64(a1, _mm512_castsi512_si128(x0)); + + /* + * Fold 128-bits to 32-bits. + */ + uint64_t val; + val = _mm_crc32_u64(0, _mm_extract_epi64(a1, 0)); + return (uint32_t) _mm_crc32_u64(val, _mm_extract_epi64(a1, 1)); +} diff --git a/source/intel/visualc/visualc_crc32c_sse42.c b/source/intel/visualc/visualc_crc32c_sse42.c index ca1aca4..707f2ba 100644 --- a/source/intel/visualc/visualc_crc32c_sse42.c +++ b/source/intel/visualc/visualc_crc32c_sse42.c @@ -3,26 +3,15 @@ * SPDX-License-Identifier: Apache-2.0. */ -#include -#include - -#if defined(_M_X64) || defined(_M_IX86) - -# if defined(_M_X64) -typedef uint64_t *slice_ptr_type; -typedef uint64_t slice_ptr_int_type; -# else -typedef uint32_t *slice_ptr_type; -typedef uint32_t slice_ptr_int_type; -# endif +#include /** * This implements crc32c via the intel sse 4.2 instructions. * This is separate from the straight asm version, because visual c does not allow * inline assembly for x64. */ -uint32_t aws_checksums_crc32c_hw(const uint8_t *data, int length, uint32_t previousCrc32) { - uint32_t crc = ~previousCrc32; +uint32_t aws_checksums_crc32c_sse42(const uint8_t *data, int length, uint32_t previousCrc32) { + uint32_t crc = previousCrc32; int length_to_process = length; slice_ptr_type temp = (slice_ptr_type)data; @@ -54,11 +43,11 @@ uint32_t aws_checksums_crc32c_hw(const uint8_t *data, int length, uint32_t previ uint32_t remainder = length_to_process % sizeof(temp); while (slices--) { -# if defined(_M_X64) +#if defined(_M_X64) crc = (uint32_t)_mm_crc32_u64(crc, *temp++); -# else +#else crc = _mm_crc32_u32(crc, *temp++); -# endif +#endif } /* process the remaining parts that can't be done on the slice size. */ @@ -70,8 +59,3 @@ uint32_t aws_checksums_crc32c_hw(const uint8_t *data, int length, uint32_t previ return ~crc; } - -uint32_t aws_checksums_crc32_hw(const uint8_t *input, int length, uint32_t previousCrc32) { - return aws_checksums_crc32_sw(input, length, previousCrc32); -} -#endif /* x64 || x86 */ diff --git a/tests/crc_test.c b/tests/crc_test.c index ec9d2a4..0d8cfeb 100644 --- a/tests/crc_test.c +++ b/tests/crc_test.c @@ -5,6 +5,9 @@ #include #include + +#include + #include static const uint8_t DATA_32_ZEROS[32] = {0}; @@ -99,6 +102,17 @@ static int s_test_crc32c(struct aws_allocator *allocator, void *ctx) { res |= s_test_known_crc32c(CRC_FUNC_NAME(aws_checksums_crc32c)); res |= s_test_known_crc32c(CRC_FUNC_NAME(aws_checksums_crc32c_sw)); + struct aws_byte_buf avx_buf; + /* enough for two avx512 runs */ + aws_byte_buf_init(&avx_buf, allocator, 512); + aws_device_random_buffer(&avx_buf); + + uint32_t crc = aws_checksums_crc32c_sw(avx_buf.buffer, (int)avx_buf.len, 0); + uint32_t hw_crc = aws_checksums_crc32c_hw(avx_buf.buffer, (int)avx_buf.len, 0); + + aws_byte_buf_clean_up(&avx_buf); + ASSERT_UINT_EQUALS(hw_crc, crc); + return res; } AWS_TEST_CASE(test_crc32c, s_test_crc32c)