diff --git a/README.md b/README.md index 79c3ca3..b847371 100644 --- a/README.md +++ b/README.md @@ -1,74 +1,149 @@ -# NitroBase64 +# react-native-nitro-base64 ⚡ -A high-performance, cross-platform Base64 encoding/decoding module for React Native, powered by C++ and simdutf. Implements WHATWG forgiving-base64 and supports both standard and URL-safe variants. +The fastest Base64 library for React Native — powered by [simdutf](https://github.com/simdutf/simdutf) SIMD C++ and [Nitro Modules](https://github.com/mrousavy/nitro). -| iPhone | Android | -| ------------------------------------------------- | --------------------------------------------------- | +Processes a 1.3MB image **26x faster** than `atob`/`btoa` and **1.8x faster** than `react-native-quick-base64`. + +| iPhone | Android | +| --- | --- | | ![iPhone](./docs/ios.png) | ![Android](./docs/android.png) | -## Features -- Fast C++ implementation using [simdutf](https://github.com/simdutf/simdutf) -- WHATWG forgiving-base64 compliance (removes whitespace, optional padding) -- Supports both standard and URL-safe base64 -- Consistent API for iOS and Android -- Detailed error handling -## Installation +--- + +## 🤔 Why this library? + +Most React Native base64 libraries hit the same bottlenecks: + +- 🐢 **Pure JS** (`base64-js`) — slow. No native acceleration. +- ⚠️ **`atob`/`btoa`** — fast for tiny strings, falls apart on binary data at scale. +- ⚠️ **String-based native** — crosses the JSI bridge as UTF-16, doubling memory for binary payloads. + +`react-native-nitro-base64` uses SIMD instructions (AVX-512 on x86, NEON on Apple Silicon) to process 64 bytes per CPU instruction, and exposes `ArrayBuffer`/`Uint8Array` APIs that bypass string marshaling entirely. + +### 📊 Benchmark — 30 iterations, iPhone (Apple Silicon) + +> Tested against `react-native-quick-base64@3.0.0` + + +#### Small image (7KB) + +| Library | Time | vs. nitro-base64 | +|---|---|---| +| ⚡ **nitro-base64** `decodeBuffer` / `fromByteArray` | **0.06ms** | — | +| react-native-quick-base64 | 0.11ms | 1.8x slower | +| atob / btoa (Hermes) | 1.04ms | 17x slower | + +#### Large image (1.3MB) + +| Library | Time | vs. nitro-base64 | +|---|---|---| +| ⚡ **nitro-base64** `decodeBuffer` / `fromByteArray` | **7.5ms** | — | +| react-native-quick-base64 | 13.6ms | 1.8x slower | +| atob / btoa (Hermes) | 203ms | 26x slower | + +--- + +## 📦 Installation ```sh yarn add react-native-nitro-base64 -yarn add react-native-nitro-modules@0.29.8 +yarn add react-native-nitro-modules ``` -### Linking -This module uses [react-native-nitro-modules](https://github.com/mrousavy/nitro). Follow Nitro's setup instructions for autolinking and native builds. +Then follow [Nitro's setup guide](https://nitro.margelo.com) for autolinking. -## API - -```typescript -export interface NitroBase64 extends HybridObject<{ ios: 'c++'; android: 'c++' }> { - install(): void; - encode(input: string, urlSafe: boolean): string; - decode(base64: string, urlSafe: boolean): string; -} -``` +### 🔧 Setup -- `install(): void` — Initializes the native module for faster first-time encoding/decoding. Call this early in your app lifecycle via index.js or root -- `encode(input: string, urlSafe: boolean): string` — Encodes binary data to base64. Set `urlSafe` to `true` for base64url encoding. -- `decode(base64: string, urlSafe: boolean): string` — Decodes base64 (WHATWG forgiving) to binary. Set `urlSafe` to `true` for base64url decoding. +Call `install()` once before your app mounts: -## Usage -```javascript -index.js +```js +// index.js import { install } from 'react-native-nitro-base64'; import { AppRegistry } from 'react-native'; -import { install } from 'react-native-nitro-base64'; -import { name as appName } from './app.json'; import App from './src/App'; -install(); // <=== Call +import { name as appName } from './app.json'; + +install(); AppRegistry.registerComponent(appName, () => App); +``` +--- +## 📖 API -```typescript -import { encode,decode } from 'react-native-nitro-base64'; +| Function | Input → Output | Notes | +|---|---|---| +| `encode(str, urlSafe?)` | string → base64 string | URL-safe optional | +| `decode(base64)` | base64 string → string | Auto-detects standard & URL-safe | +| `encodeBuffer(ArrayBuffer, urlSafe?)` | ArrayBuffer → base64 string | 🚀 Zero-copy, fastest for images | +| `decodeBuffer(base64)` | base64 string → ArrayBuffer | 🚀 Zero-copy, fastest for images | +| `fromByteArray(Uint8Array, urlSafe?)` | Uint8Array → base64 string | Drop-in for `react-native-quick-base64` | +| `toByteArray(base64)` | base64 string → Uint8Array | Drop-in for `react-native-quick-base64` | -const encoded = encode('Hello World!', false); // "SGVsbG8gV29ybGQh" -const urlEncoded = encode('Hello World!', true); // "SGVsbG8gV29ybGQh" +--- -const decoded = decode(encoded, false); // "Hello World!" -const urlDecoded = decode(urlEncoded, true); // "Hello World!" +## 🚀 Usage + +### 🖼️ Images & files — `encodeBuffer` / `decodeBuffer` + +Images are binary data. Sending binary through a JS string crosses the JSI bridge as UTF-16, doubling memory. `ArrayBuffer` skips that entirely — raw bytes go straight to C++. + +```ts +import { encodeBuffer, decodeBuffer } from 'react-native-nitro-base64'; + +// Upload image to API +const response = await fetch(imageUri); +const buffer = await response.arrayBuffer(); +const base64 = encodeBuffer(buffer); // → "iVBORw0KGgo..." + +// Decode API response for component +const buffer = decodeBuffer(base64String); // → ArrayBuffer +``` + +### 🔑 Text & JWT — `encode` / `decode` + +```ts +import { encode, decode } from 'react-native-nitro-base64'; + +const encoded = encode('Hello World!'); // "SGVsbG8gV29ybGQh" +const decoded = decode(encoded); // "Hello World!" + +// URL-safe for JWT / URL params +const token = encode(payload, true); ``` -## Platform Support -- **iOS:** C++ implementation via simdutf -- **Android:** C++ implementation via simdutf +### 🔐 Web Crypto & hashing — `fromByteArray` / `toByteArray` -## Error Handling -Throws descriptive errors for invalid base64 input, remainder issues, or extra bits in padding. +Drop-in replacement for `react-native-quick-base64`. Same API, faster engine. + +```ts +import { fromByteArray, toByteArray } from 'react-native-nitro-base64'; + +// SHA-256 hash → base64 +const hash = new Uint8Array(await crypto.subtle.digest('SHA-256', data)); +const encoded = fromByteArray(hash); + +// base64 → text +const bytes = toByteArray(base64String); +const text = new TextDecoder().decode(bytes); +``` + +--- + +## 📱 Platform Support + +| Platform | Engine | +|---|---| +| iOS | C++ (simdutf, NEON SIMD) | +| Android | C++ (simdutf, NEON / AVX2 SIMD) | + +--- + +## 📄 License -## License MIT -## Credits -- [simdutf](https://github.com/simdutf/simdutf) -- [Nitro Modules](https://github.com/mrousavy/nitro) +## 🙏 Credits + +- [simdutf](https://github.com/simdutf/simdutf) — SIMD-accelerated Unicode & Base64 +- [Nitro Modules](https://github.com/mrousavy/nitro) — JSI native module framework diff --git a/cpp/NitroBase64.hpp b/cpp/NitroBase64.hpp index c33ddcd..f71ce64 100644 --- a/cpp/NitroBase64.hpp +++ b/cpp/NitroBase64.hpp @@ -2,6 +2,7 @@ #include "HybridNitroBase64Spec.hpp" #include "simdutf.h" +#include namespace margelo::nitro::nitrobase64 { using namespace margelo::nitro; @@ -27,7 +28,7 @@ namespace margelo::nitro::nitrobase64 { return output; } - std::string decode(const std::string &base64,bool urlSafe = false) override { + std::string decode(const std::string &base64) override { size_t max_decoded_size = simdutf::maximal_binary_length_from_base64( base64.data(), base64.size() @@ -39,7 +40,7 @@ namespace margelo::nitro::nitrobase64 { base64.data(), base64.size(), output.data(), - urlSafe ? simdutf::base64_url : simdutf::base64_default + simdutf::base64_default_or_url ); if (result.error != simdutf::error_code::SUCCESS) { @@ -50,6 +51,45 @@ namespace margelo::nitro::nitrobase64 { return output; } + std::string encodeBuffer(const std::shared_ptr& input, bool urlSafe = false) override { + auto opts = urlSafe ? simdutf::base64_url : simdutf::base64_default; + size_t base64_size = simdutf::base64_length_from_binary(input->size(), opts); + std::string output(base64_size, '\0'); + + size_t actual_size = simdutf::binary_to_base64( + reinterpret_cast(input->data()), + input->size(), + output.data(), + opts + ); + + output.resize(actual_size); + return output; + } + + std::shared_ptr decodeBuffer(const std::string &base64) override { + size_t max_decoded_size = simdutf::maximal_binary_length_from_base64( + base64.data(), + base64.size() + ); + + std::vector buffer(max_decoded_size); + + auto result = simdutf::base64_to_binary( + base64.data(), + base64.size(), + reinterpret_cast(buffer.data()), + simdutf::base64_default_or_url + ); + + if (result.error != simdutf::error_code::SUCCESS) { + throw std::runtime_error("Base64 decode failed"); + } + + buffer.resize(result.count); + return ArrayBuffer::move(std::move(buffer)); + } + private: static constexpr auto TAG = "NitroBase64"; }; diff --git a/cpp/simdutf.cpp b/cpp/simdutf.cpp index 28843c9..5bba36d 100644 --- a/cpp/simdutf.cpp +++ b/cpp/simdutf.cpp @@ -1,910 +1,10 @@ -/* auto-generated on 2025-09-10 16:01:38 -0400. Do not edit! */ +/* auto-generated on 2026-05-07 06:54:06 +0200. Do not edit! */ /* begin file src/simdutf.cpp */ #include "simdutf.h" - // We include base64_tables once. -/* begin file src/tables/base64_tables.h */ -#ifndef SIMDUTF_BASE64_TABLES_H -#define SIMDUTF_BASE64_TABLES_H -#include -#include - -namespace simdutf { -namespace { -namespace tables { -namespace base64 { -namespace base64_default { - -const char e0[256] = { - 'A', 'A', 'A', 'A', 'B', 'B', 'B', 'B', 'C', 'C', 'C', 'C', 'D', 'D', 'D', - 'D', 'E', 'E', 'E', 'E', 'F', 'F', 'F', 'F', 'G', 'G', 'G', 'G', 'H', 'H', - 'H', 'H', 'I', 'I', 'I', 'I', 'J', 'J', 'J', 'J', 'K', 'K', 'K', 'K', 'L', - 'L', 'L', 'L', 'M', 'M', 'M', 'M', 'N', 'N', 'N', 'N', 'O', 'O', 'O', 'O', - 'P', 'P', 'P', 'P', 'Q', 'Q', 'Q', 'Q', 'R', 'R', 'R', 'R', 'S', 'S', 'S', - 'S', 'T', 'T', 'T', 'T', 'U', 'U', 'U', 'U', 'V', 'V', 'V', 'V', 'W', 'W', - 'W', 'W', 'X', 'X', 'X', 'X', 'Y', 'Y', 'Y', 'Y', 'Z', 'Z', 'Z', 'Z', 'a', - 'a', 'a', 'a', 'b', 'b', 'b', 'b', 'c', 'c', 'c', 'c', 'd', 'd', 'd', 'd', - 'e', 'e', 'e', 'e', 'f', 'f', 'f', 'f', 'g', 'g', 'g', 'g', 'h', 'h', 'h', - 'h', 'i', 'i', 'i', 'i', 'j', 'j', 'j', 'j', 'k', 'k', 'k', 'k', 'l', 'l', - 'l', 'l', 'm', 'm', 'm', 'm', 'n', 'n', 'n', 'n', 'o', 'o', 'o', 'o', 'p', - 'p', 'p', 'p', 'q', 'q', 'q', 'q', 'r', 'r', 'r', 'r', 's', 's', 's', 's', - 't', 't', 't', 't', 'u', 'u', 'u', 'u', 'v', 'v', 'v', 'v', 'w', 'w', 'w', - 'w', 'x', 'x', 'x', 'x', 'y', 'y', 'y', 'y', 'z', 'z', 'z', 'z', '0', '0', - '0', '0', '1', '1', '1', '1', '2', '2', '2', '2', '3', '3', '3', '3', '4', - '4', '4', '4', '5', '5', '5', '5', '6', '6', '6', '6', '7', '7', '7', '7', - '8', '8', '8', '8', '9', '9', '9', '9', '+', '+', '+', '+', '/', '/', '/', - '/'}; - -const char e1[256] = { - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', - 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', - 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', - 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', - 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', - 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', - 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', - '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', - 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', - 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', - 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', - 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', - 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', - 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', - 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', - '/'}; - -const char e2[256] = { - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', - 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', - 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', - 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', - 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', - 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', - 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', - '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', - 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', - 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', - 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', - 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', - 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', - 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', - 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', - '/'}; - -const uint32_t d0[256] = { - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x000000f8, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000fc, - 0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4, - 0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, - 0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018, - 0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030, - 0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048, - 0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060, - 0x00000064, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078, - 0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090, - 0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8, - 0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0, - 0x000000c4, 0x000000c8, 0x000000cc, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; - -const uint32_t d1[256] = { - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x0000e003, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000f003, - 0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003, - 0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, - 0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000, - 0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000, - 0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001, - 0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001, - 0x00009001, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001, - 0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002, - 0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002, - 0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003, - 0x00001003, 0x00002003, 0x00003003, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; - -const uint32_t d2[256] = { - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x00800f00, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00c00f00, - 0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00, - 0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, - 0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100, - 0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300, - 0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400, - 0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600, - 0x00400600, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700, - 0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900, - 0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00, - 0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00, - 0x00400c00, 0x00800c00, 0x00c00c00, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; - -const uint32_t d3[256] = { - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x003e0000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003f0000, - 0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000, - 0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, - 0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000, - 0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000, - 0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000, - 0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000, - 0x00190000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000, - 0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000, - 0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000, - 0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000, - 0x00310000, 0x00320000, 0x00330000, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; -} // namespace base64_default - -namespace base64_url { - -const char e0[256] = { - 'A', 'A', 'A', 'A', 'B', 'B', 'B', 'B', 'C', 'C', 'C', 'C', 'D', 'D', 'D', - 'D', 'E', 'E', 'E', 'E', 'F', 'F', 'F', 'F', 'G', 'G', 'G', 'G', 'H', 'H', - 'H', 'H', 'I', 'I', 'I', 'I', 'J', 'J', 'J', 'J', 'K', 'K', 'K', 'K', 'L', - 'L', 'L', 'L', 'M', 'M', 'M', 'M', 'N', 'N', 'N', 'N', 'O', 'O', 'O', 'O', - 'P', 'P', 'P', 'P', 'Q', 'Q', 'Q', 'Q', 'R', 'R', 'R', 'R', 'S', 'S', 'S', - 'S', 'T', 'T', 'T', 'T', 'U', 'U', 'U', 'U', 'V', 'V', 'V', 'V', 'W', 'W', - 'W', 'W', 'X', 'X', 'X', 'X', 'Y', 'Y', 'Y', 'Y', 'Z', 'Z', 'Z', 'Z', 'a', - 'a', 'a', 'a', 'b', 'b', 'b', 'b', 'c', 'c', 'c', 'c', 'd', 'd', 'd', 'd', - 'e', 'e', 'e', 'e', 'f', 'f', 'f', 'f', 'g', 'g', 'g', 'g', 'h', 'h', 'h', - 'h', 'i', 'i', 'i', 'i', 'j', 'j', 'j', 'j', 'k', 'k', 'k', 'k', 'l', 'l', - 'l', 'l', 'm', 'm', 'm', 'm', 'n', 'n', 'n', 'n', 'o', 'o', 'o', 'o', 'p', - 'p', 'p', 'p', 'q', 'q', 'q', 'q', 'r', 'r', 'r', 'r', 's', 's', 's', 's', - 't', 't', 't', 't', 'u', 'u', 'u', 'u', 'v', 'v', 'v', 'v', 'w', 'w', 'w', - 'w', 'x', 'x', 'x', 'x', 'y', 'y', 'y', 'y', 'z', 'z', 'z', 'z', '0', '0', - '0', '0', '1', '1', '1', '1', '2', '2', '2', '2', '3', '3', '3', '3', '4', - '4', '4', '4', '5', '5', '5', '5', '6', '6', '6', '6', '7', '7', '7', '7', - '8', '8', '8', '8', '9', '9', '9', '9', '-', '-', '-', '-', '_', '_', '_', - '_'}; - -const char e1[256] = { - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', - 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', - 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', - 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', - 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', - 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', - 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', - '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', - 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', - 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', - 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', - 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', - 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', - 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', - 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', - '_'}; - -const char e2[256] = { - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', - 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', - 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', - 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', - 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', - 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', - 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', - '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', - 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', - 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', - 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', - 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', - 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', - 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', - 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', - '_'}; - -const uint32_t d0[256] = { - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000f8, 0x01ffffff, 0x01ffffff, - 0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4, - 0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, - 0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018, - 0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030, - 0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048, - 0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060, - 0x00000064, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000fc, - 0x01ffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078, - 0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090, - 0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8, - 0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0, - 0x000000c4, 0x000000c8, 0x000000cc, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; -const uint32_t d1[256] = { - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000e003, 0x01ffffff, 0x01ffffff, - 0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003, - 0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, - 0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000, - 0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000, - 0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001, - 0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001, - 0x00009001, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000f003, - 0x01ffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001, - 0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002, - 0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002, - 0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003, - 0x00001003, 0x00002003, 0x00003003, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; -const uint32_t d2[256] = { - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00800f00, 0x01ffffff, 0x01ffffff, - 0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00, - 0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, - 0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100, - 0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300, - 0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400, - 0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600, - 0x00400600, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00c00f00, - 0x01ffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700, - 0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900, - 0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00, - 0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00, - 0x00400c00, 0x00800c00, 0x00c00c00, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; -const uint32_t d3[256] = { - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003e0000, 0x01ffffff, 0x01ffffff, - 0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000, - 0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, - 0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000, - 0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000, - 0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000, - 0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000, - 0x00190000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003f0000, - 0x01ffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000, - 0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000, - 0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000, - 0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000, - 0x00310000, 0x00320000, 0x00330000, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; -} // namespace base64_url - -namespace base64_default_or_url { -const uint32_t d0[256] = { - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x000000f8, 0x01ffffff, 0x000000f8, 0x01ffffff, 0x000000fc, - 0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4, - 0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, - 0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018, - 0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030, - 0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048, - 0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060, - 0x00000064, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000fc, - 0x01ffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078, - 0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090, - 0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8, - 0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0, - 0x000000c4, 0x000000c8, 0x000000cc, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; -const uint32_t d1[256] = { - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x0000e003, 0x01ffffff, 0x0000e003, 0x01ffffff, 0x0000f003, - 0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003, - 0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, - 0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000, - 0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000, - 0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001, - 0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001, - 0x00009001, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000f003, - 0x01ffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001, - 0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002, - 0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002, - 0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003, - 0x00001003, 0x00002003, 0x00003003, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; -const uint32_t d2[256] = { - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x00800f00, 0x01ffffff, 0x00800f00, 0x01ffffff, 0x00c00f00, - 0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00, - 0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, - 0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100, - 0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300, - 0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400, - 0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600, - 0x00400600, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00c00f00, - 0x01ffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700, - 0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900, - 0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00, - 0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00, - 0x00400c00, 0x00800c00, 0x00c00c00, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; -const uint32_t d3[256] = { - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x003e0000, 0x01ffffff, 0x003e0000, 0x01ffffff, 0x003f0000, - 0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000, - 0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, - 0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000, - 0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000, - 0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000, - 0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000, - 0x00190000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003f0000, - 0x01ffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000, - 0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000, - 0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000, - 0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000, - 0x00310000, 0x00320000, 0x00330000, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; -} // namespace base64_default_or_url -const uint64_t thintable_epi8[256] = { - 0x0706050403020100, 0x0007060504030201, 0x0007060504030200, - 0x0000070605040302, 0x0007060504030100, 0x0000070605040301, - 0x0000070605040300, 0x0000000706050403, 0x0007060504020100, - 0x0000070605040201, 0x0000070605040200, 0x0000000706050402, - 0x0000070605040100, 0x0000000706050401, 0x0000000706050400, - 0x0000000007060504, 0x0007060503020100, 0x0000070605030201, - 0x0000070605030200, 0x0000000706050302, 0x0000070605030100, - 0x0000000706050301, 0x0000000706050300, 0x0000000007060503, - 0x0000070605020100, 0x0000000706050201, 0x0000000706050200, - 0x0000000007060502, 0x0000000706050100, 0x0000000007060501, - 0x0000000007060500, 0x0000000000070605, 0x0007060403020100, - 0x0000070604030201, 0x0000070604030200, 0x0000000706040302, - 0x0000070604030100, 0x0000000706040301, 0x0000000706040300, - 0x0000000007060403, 0x0000070604020100, 0x0000000706040201, - 0x0000000706040200, 0x0000000007060402, 0x0000000706040100, - 0x0000000007060401, 0x0000000007060400, 0x0000000000070604, - 0x0000070603020100, 0x0000000706030201, 0x0000000706030200, - 0x0000000007060302, 0x0000000706030100, 0x0000000007060301, - 0x0000000007060300, 0x0000000000070603, 0x0000000706020100, - 0x0000000007060201, 0x0000000007060200, 0x0000000000070602, - 0x0000000007060100, 0x0000000000070601, 0x0000000000070600, - 0x0000000000000706, 0x0007050403020100, 0x0000070504030201, - 0x0000070504030200, 0x0000000705040302, 0x0000070504030100, - 0x0000000705040301, 0x0000000705040300, 0x0000000007050403, - 0x0000070504020100, 0x0000000705040201, 0x0000000705040200, - 0x0000000007050402, 0x0000000705040100, 0x0000000007050401, - 0x0000000007050400, 0x0000000000070504, 0x0000070503020100, - 0x0000000705030201, 0x0000000705030200, 0x0000000007050302, - 0x0000000705030100, 0x0000000007050301, 0x0000000007050300, - 0x0000000000070503, 0x0000000705020100, 0x0000000007050201, - 0x0000000007050200, 0x0000000000070502, 0x0000000007050100, - 0x0000000000070501, 0x0000000000070500, 0x0000000000000705, - 0x0000070403020100, 0x0000000704030201, 0x0000000704030200, - 0x0000000007040302, 0x0000000704030100, 0x0000000007040301, - 0x0000000007040300, 0x0000000000070403, 0x0000000704020100, - 0x0000000007040201, 0x0000000007040200, 0x0000000000070402, - 0x0000000007040100, 0x0000000000070401, 0x0000000000070400, - 0x0000000000000704, 0x0000000703020100, 0x0000000007030201, - 0x0000000007030200, 0x0000000000070302, 0x0000000007030100, - 0x0000000000070301, 0x0000000000070300, 0x0000000000000703, - 0x0000000007020100, 0x0000000000070201, 0x0000000000070200, - 0x0000000000000702, 0x0000000000070100, 0x0000000000000701, - 0x0000000000000700, 0x0000000000000007, 0x0006050403020100, - 0x0000060504030201, 0x0000060504030200, 0x0000000605040302, - 0x0000060504030100, 0x0000000605040301, 0x0000000605040300, - 0x0000000006050403, 0x0000060504020100, 0x0000000605040201, - 0x0000000605040200, 0x0000000006050402, 0x0000000605040100, - 0x0000000006050401, 0x0000000006050400, 0x0000000000060504, - 0x0000060503020100, 0x0000000605030201, 0x0000000605030200, - 0x0000000006050302, 0x0000000605030100, 0x0000000006050301, - 0x0000000006050300, 0x0000000000060503, 0x0000000605020100, - 0x0000000006050201, 0x0000000006050200, 0x0000000000060502, - 0x0000000006050100, 0x0000000000060501, 0x0000000000060500, - 0x0000000000000605, 0x0000060403020100, 0x0000000604030201, - 0x0000000604030200, 0x0000000006040302, 0x0000000604030100, - 0x0000000006040301, 0x0000000006040300, 0x0000000000060403, - 0x0000000604020100, 0x0000000006040201, 0x0000000006040200, - 0x0000000000060402, 0x0000000006040100, 0x0000000000060401, - 0x0000000000060400, 0x0000000000000604, 0x0000000603020100, - 0x0000000006030201, 0x0000000006030200, 0x0000000000060302, - 0x0000000006030100, 0x0000000000060301, 0x0000000000060300, - 0x0000000000000603, 0x0000000006020100, 0x0000000000060201, - 0x0000000000060200, 0x0000000000000602, 0x0000000000060100, - 0x0000000000000601, 0x0000000000000600, 0x0000000000000006, - 0x0000050403020100, 0x0000000504030201, 0x0000000504030200, - 0x0000000005040302, 0x0000000504030100, 0x0000000005040301, - 0x0000000005040300, 0x0000000000050403, 0x0000000504020100, - 0x0000000005040201, 0x0000000005040200, 0x0000000000050402, - 0x0000000005040100, 0x0000000000050401, 0x0000000000050400, - 0x0000000000000504, 0x0000000503020100, 0x0000000005030201, - 0x0000000005030200, 0x0000000000050302, 0x0000000005030100, - 0x0000000000050301, 0x0000000000050300, 0x0000000000000503, - 0x0000000005020100, 0x0000000000050201, 0x0000000000050200, - 0x0000000000000502, 0x0000000000050100, 0x0000000000000501, - 0x0000000000000500, 0x0000000000000005, 0x0000000403020100, - 0x0000000004030201, 0x0000000004030200, 0x0000000000040302, - 0x0000000004030100, 0x0000000000040301, 0x0000000000040300, - 0x0000000000000403, 0x0000000004020100, 0x0000000000040201, - 0x0000000000040200, 0x0000000000000402, 0x0000000000040100, - 0x0000000000000401, 0x0000000000000400, 0x0000000000000004, - 0x0000000003020100, 0x0000000000030201, 0x0000000000030200, - 0x0000000000000302, 0x0000000000030100, 0x0000000000000301, - 0x0000000000000300, 0x0000000000000003, 0x0000000000020100, - 0x0000000000000201, 0x0000000000000200, 0x0000000000000002, - 0x0000000000000100, 0x0000000000000001, 0x0000000000000000, - 0x0000000000000000, -}; - -const uint8_t pshufb_combine_table[272] = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, - 0x0c, 0x0d, 0x0e, 0x0f, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08, - 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0x00, 0x01, 0x02, 0x03, - 0x04, 0x05, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, - 0x0f, 0xff, 0xff, 0xff, 0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b, - 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, 0x00, 0x01, 0x02, 0x08, - 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x00, 0x01, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0x00, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, - 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x08, 0x09, 0x0a, 0x0b, - 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, -}; - -const unsigned char BitsSetTable256mul2[256] = { - 0, 2, 2, 4, 2, 4, 4, 6, 2, 4, 4, 6, 4, 6, 6, 8, 2, 4, 4, - 6, 4, 6, 6, 8, 4, 6, 6, 8, 6, 8, 8, 10, 2, 4, 4, 6, 4, 6, - 6, 8, 4, 6, 6, 8, 6, 8, 8, 10, 4, 6, 6, 8, 6, 8, 8, 10, 6, - 8, 8, 10, 8, 10, 10, 12, 2, 4, 4, 6, 4, 6, 6, 8, 4, 6, 6, 8, - 6, 8, 8, 10, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, 10, 8, 10, 10, - 12, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, 10, 8, 10, 10, 12, 6, 8, - 8, 10, 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, 12, 14, 2, 4, 4, 6, 4, - 6, 6, 8, 4, 6, 6, 8, 6, 8, 8, 10, 4, 6, 6, 8, 6, 8, 8, 10, - 6, 8, 8, 10, 8, 10, 10, 12, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, - 10, 8, 10, 10, 12, 6, 8, 8, 10, 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, - 12, 14, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, 10, 8, 10, 10, 12, 6, - 8, 8, 10, 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, 12, 14, 6, 8, 8, 10, - 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, 12, 14, 8, 10, 10, 12, 10, 12, 12, - 14, 10, 12, 12, 14, 12, 14, 14, 16}; - -constexpr uint8_t to_base64_value[] = { - 255, 255, 255, 255, 255, 255, 255, 255, 255, 64, 64, 255, 64, 64, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 64, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, - 255, 255, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, - 255, 255, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, - 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, - 25, 255, 255, 255, 255, 255, 255, 26, 27, 28, 29, 30, 31, 32, 33, - 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, - 49, 50, 51, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255}; - -constexpr uint8_t to_base64_url_value[] = { - 255, 255, 255, 255, 255, 255, 255, 255, 255, 64, 64, 255, 64, 64, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 64, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 62, 255, 255, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, - 255, 255, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, - 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, - 25, 255, 255, 255, 255, 63, 255, 26, 27, 28, 29, 30, 31, 32, 33, - 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, - 49, 50, 51, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255}; - -constexpr uint8_t to_base64_default_or_url_value[] = { - 255, 255, 255, 255, 255, 255, 255, 255, 255, 64, 64, 255, 64, 64, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 64, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, - 62, 255, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, - 255, 255, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, - 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, - 25, 255, 255, 255, 255, 63, 255, 26, 27, 28, 29, 30, 31, 32, 33, - 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, - 49, 50, 51, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255}; - -static_assert(sizeof(to_base64_value) == 256, - "to_base64_value must have 256 elements"); -static_assert(sizeof(to_base64_url_value) == 256, - "to_base64_url_value must have 256 elements"); -static_assert(to_base64_value[uint8_t(' ')] == 64, - "space must be == 64 in to_base64_value"); -static_assert(to_base64_url_value[uint8_t(' ')] == 64, - "space must be == 64 in to_base64_url_value"); -static_assert(to_base64_value[uint8_t('\t')] == 64, - "tab must be == 64 in to_base64_value"); -static_assert(to_base64_url_value[uint8_t('\t')] == 64, - "tab must be == 64 in to_base64_url_value"); -static_assert(to_base64_value[uint8_t('\r')] == 64, - "cr must be == 64 in to_base64_value"); -static_assert(to_base64_url_value[uint8_t('\r')] == 64, - "cr must be == 64 in to_base64_url_value"); -static_assert(to_base64_value[uint8_t('\n')] == 64, - "lf must be == 64 in to_base64_value"); -static_assert(to_base64_url_value[uint8_t('\n')] == 64, - "lf must be == 64 in to_base64_url_value"); -static_assert(to_base64_value[uint8_t('\f')] == 64, - "ff must be == 64 in to_base64_value"); -static_assert(to_base64_url_value[uint8_t('\f')] == 64, - "ff must be == 64 in to_base64_url_value"); -static_assert(to_base64_value[uint8_t('+')] == 62, - "+ must be == 62 in to_base64_value"); -static_assert(to_base64_url_value[uint8_t('-')] == 62, - "- must be == 62 in to_base64_url_value"); -static_assert(to_base64_value[uint8_t('/')] == 63, - "/ must be == 62 in to_base64_value"); -static_assert(to_base64_url_value[uint8_t('_')] == 63, - "_ must be == 62 in to_base64_url_value"); -} // namespace base64 -} // namespace tables -} // unnamed namespace -} // namespace simdutf - -#endif // SIMDUTF_BASE64_TABLES_H -/* end file src/tables/base64_tables.h */ - /* begin file src/encoding_types.cpp */ namespace simdutf { -bool match_system(endianness e) { -#if SIMDUTF_IS_BIG_ENDIAN - return e == endianness::BIG; -#else - return e == endianness::LITTLE; -#endif -} - -std::string to_string(encoding_type bom) { +std::string_view to_string(encoding_type bom) { switch (bom) { case UTF16_LE: return "UTF16 little-endian"; @@ -2688,27 +1788,35 @@ class implementation final : public simdutf::implementation { simdutf_warn_unused result base64_to_binary( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; simdutf_warn_unused full_result base64_to_binary_details( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; - simdutf_warn_unused result - base64_to_binary(const char16_t *input, size_t length, char *output, - base64_options options, - last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; + simdutf_warn_unused result base64_to_binary( + const char16_t *input, size_t length, char *output, + base64_options options, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept override; simdutf_warn_unused full_result base64_to_binary_details( const char16_t *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; size_t binary_to_base64(const char *input, size_t length, char *output, - base64_options options) const noexcept; + base64_options options) const noexcept override; + size_t + binary_to_base64_with_lines(const char *input, size_t length, char *output, + size_t line_length, + base64_options options) const noexcept override; const char *find(const char *start, const char *end, - char character) const noexcept; + char character) const noexcept override; const char16_t *find(const char16_t *start, const char16_t *end, - char16_t character) const noexcept; + char16_t character) const noexcept override; + simdutf_warn_unused size_t binary_length_from_base64( + const char *input, size_t length) const noexcept override; + simdutf_warn_unused size_t binary_length_from_base64( + const char16_t *input, size_t length) const noexcept override; }; } // namespace arm64 @@ -2720,7 +1828,6 @@ class implementation final : public simdutf::implementation { /* begin file src/simdutf/arm64/begin.h */ // redefining SIMDUTF_IMPLEMENTATION to "arm64" // #define SIMDUTF_IMPLEMENTATION arm64 -#define SIMDUTF_SIMD_HAS_BYTEMASK 1 /* end file src/simdutf/arm64/begin.h */ // Declarations @@ -3082,9 +2189,10 @@ template <> struct simd8 { // ... template simdutf_really_inline void store_ascii_as_utf16(char16_t *p) const { - int8x16x2_t pair = match_system(big_endian) - ? int8x16x2_t{{this->value, vmovq_n_s8(0)}} - : int8x16x2_t{{vmovq_n_s8(0), this->value}}; + constexpr auto matches = match_system(big_endian); + const int8x16x2_t pair = matches + ? int8x16x2_t{{this->value, vmovq_n_s8(0)}} + : int8x16x2_t{{vmovq_n_s8(0), this->value}}; vst2q_s8(reinterpret_cast(p), pair); } @@ -3283,6 +2391,12 @@ template struct simd8x64 { this->chunks[2] > mask, this->chunks[3] > mask) .to_bitmask(); } + simdutf_really_inline uint64_t gteq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64(this->chunks[0] >= mask, this->chunks[1] >= mask, + this->chunks[2] >= mask, this->chunks[3] >= mask) + .to_bitmask(); + } simdutf_really_inline uint64_t gteq_unsigned(const uint8_t m) const { const simd8 mask = simd8::splat(m); return simd8x64(simd8(uint8x16_t(this->chunks[0])) >= mask, @@ -3652,7 +2766,12 @@ template struct simd16x32 { this->chunks[2] > mask, this->chunks[3] > mask) .to_bitmask(); } - + simdutf_really_inline uint64_t gteq(const T m) const { + const simd16 mask = simd16::splat(m); + return simd16x32(this->chunks[0] >= mask, this->chunks[1] >= mask, + this->chunks[2] >= mask, this->chunks[3] >= mask) + .to_bitmask(); + } simdutf_really_inline uint64_t lteq(const T m) const { const simd16 mask = simd16::splat(m); return simd16x32(this->chunks[0] <= mask, this->chunks[1] <= mask, @@ -3871,7 +2990,6 @@ simdutf_really_inline simd64 sum_8bytes(const simd8 v) { /* end file src/simdutf/arm64/simd.h */ /* begin file src/simdutf/arm64/end.h */ -#undef SIMDUTF_SIMD_HAS_BYTEMASK /* end file src/simdutf/arm64/end.h */ #endif // SIMDUTF_IMPLEMENTATION_ARM64 @@ -4113,27 +3231,35 @@ class implementation final : public simdutf::implementation { simdutf_warn_unused result base64_to_binary( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; simdutf_warn_unused full_result base64_to_binary_details( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; - simdutf_warn_unused result - base64_to_binary(const char16_t *input, size_t length, char *output, - base64_options options, - last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; + simdutf_warn_unused result base64_to_binary( + const char16_t *input, size_t length, char *output, + base64_options options, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept override; simdutf_warn_unused full_result base64_to_binary_details( const char16_t *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; size_t binary_to_base64(const char *input, size_t length, char *output, - base64_options options) const noexcept; + base64_options options) const noexcept override; + size_t + binary_to_base64_with_lines(const char *input, size_t length, char *output, + size_t line_length, + base64_options options) const noexcept override; const char *find(const char *start, const char *end, - char character) const noexcept; + char character) const noexcept override; const char16_t *find(const char16_t *start, const char16_t *end, - char16_t character) const noexcept; + char16_t character) const noexcept override; + simdutf_warn_unused size_t binary_length_from_base64( + const char *input, size_t length) const noexcept override; + simdutf_warn_unused size_t binary_length_from_base64( + const char16_t *input, size_t length) const noexcept override; }; } // namespace icelake @@ -4182,6 +3308,17 @@ simdutf_really_inline long long int count_ones(uint64_t input_num) { } #endif +#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO +simdutf_really_inline unsigned __int64 count_ones32(uint32_t input_num) { + // note: we do not support legacy 32-bit Windows + return __popcnt(input_num); // Visual Studio wants two underscores +} +#else +simdutf_really_inline long long int count_ones32(uint32_t input_num) { + return _popcnt32(input_num); +} +#endif + #if SIMDUTF_NEED_TRAILING_ZEROES // simdutf_really_inline int trailing_zeroes(uint64_t input_num) { // #if SIMDUTF_REGULAR_VISUAL_STUDIO @@ -4445,27 +3582,35 @@ class implementation final : public simdutf::implementation { simdutf_warn_unused result base64_to_binary( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; simdutf_warn_unused full_result base64_to_binary_details( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; - simdutf_warn_unused result - base64_to_binary(const char16_t *input, size_t length, char *output, - base64_options options, - last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; + simdutf_warn_unused result base64_to_binary( + const char16_t *input, size_t length, char *output, + base64_options options, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept override; simdutf_warn_unused full_result base64_to_binary_details( const char16_t *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; size_t binary_to_base64(const char *input, size_t length, char *output, - base64_options options) const noexcept; + base64_options options) const noexcept override; + size_t + binary_to_base64_with_lines(const char *input, size_t length, char *output, + size_t line_length, + base64_options options) const noexcept override; const char *find(const char *start, const char *end, - char character) const noexcept; + char character) const noexcept override; const char16_t *find(const char16_t *start, const char16_t *end, - char16_t character) const noexcept; + char16_t character) const noexcept override; + simdutf_warn_unused size_t binary_length_from_base64( + const char *input, size_t length) const noexcept override; + simdutf_warn_unused size_t binary_length_from_base64( + const char16_t *input, size_t length) const noexcept override; }; } // namespace haswell @@ -5460,27 +4605,35 @@ class implementation final : public simdutf::implementation { simdutf_warn_unused result base64_to_binary( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; simdutf_warn_unused full_result base64_to_binary_details( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; - simdutf_warn_unused result - base64_to_binary(const char16_t *input, size_t length, char *output, - base64_options options, - last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; + simdutf_warn_unused result base64_to_binary( + const char16_t *input, size_t length, char *output, + base64_options options, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept override; simdutf_warn_unused full_result base64_to_binary_details( const char16_t *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; size_t binary_to_base64(const char *input, size_t length, char *output, - base64_options options) const noexcept; + base64_options options) const noexcept override; + size_t + binary_to_base64_with_lines(const char *input, size_t length, char *output, + size_t line_length, + base64_options options) const noexcept override; const char *find(const char *start, const char *end, - char character) const noexcept; + char character) const noexcept override; const char16_t *find(const char16_t *start, const char16_t *end, - char16_t character) const noexcept; + char16_t character) const noexcept override; + simdutf_warn_unused size_t binary_length_from_base64( + const char *input, size_t length) const noexcept override; + simdutf_warn_unused size_t binary_length_from_base64( + const char16_t *input, size_t length) const noexcept override; }; } // namespace westmere @@ -5923,7 +5076,12 @@ template struct simd8x64 { this->chunks[2] > mask, this->chunks[3] > mask) .to_bitmask(); } - + simdutf_really_inline uint64_t gteq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64(this->chunks[0] >= mask, this->chunks[1] >= mask, + this->chunks[2] >= mask, this->chunks[3] >= mask) + .to_bitmask(); + } simdutf_really_inline uint64_t eq(const T m) const { const simd8 mask = simd8::splat(m); return simd8x64(this->chunks[0] == mask, this->chunks[1] == mask, @@ -6155,6 +5313,13 @@ template struct simd16x32 { .to_bitmask(); } + simdutf_really_inline uint64_t gteq(const T m) const { + const simd16 mask = simd16::splat(m); + return simd16x32(this->chunks[0] >= mask, this->chunks[1] >= mask, + this->chunks[2] >= mask, this->chunks[3] >= mask) + .to_bitmask(); + } + simdutf_really_inline uint64_t eq(const T m) const { const simd16 mask = simd16::splat(m); return simd16x32(this->chunks[0] == mask, this->chunks[1] == mask, @@ -6448,29 +5613,33 @@ class implementation final : public simdutf::implementation { simdutf_warn_unused result base64_to_binary( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; simdutf_warn_unused full_result base64_to_binary_details( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; - simdutf_warn_unused result - base64_to_binary(const char16_t *input, size_t length, char *output, - base64_options options, - last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; + simdutf_warn_unused result base64_to_binary( + const char16_t *input, size_t length, char *output, + base64_options options, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept override; simdutf_warn_unused full_result base64_to_binary_details( const char16_t *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; size_t binary_to_base64(const char *input, size_t length, char *output, - base64_options options) const noexcept; + base64_options options) const noexcept override; + size_t + binary_to_base64_with_lines(const char *input, size_t length, char *output, + size_t line_length, + base64_options options) const noexcept override; const char *find(const char *start, const char *end, - char character) const noexcept; + char character) const noexcept override; const char16_t *find(const char16_t *start, const char16_t *end, - char16_t character) const noexcept; + char16_t character) const noexcept override; #ifdef SIMDUTF_INTERNAL_TESTS virtual std::vector internal_tests() const override; @@ -7976,28 +7145,32 @@ class implementation final : public simdutf::implementation { simdutf_warn_unused result base64_to_binary( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; simdutf_warn_unused full_result base64_to_binary_details( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; - simdutf_warn_unused result - base64_to_binary(const char16_t *input, size_t length, char *output, - base64_options options, - last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; + simdutf_warn_unused result base64_to_binary( + const char16_t *input, size_t length, char *output, + base64_options options, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept override; simdutf_warn_unused full_result base64_to_binary_details( const char16_t *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; size_t binary_to_base64(const char *input, size_t length, char *output, - base64_options options) const noexcept; + base64_options options) const noexcept override; + size_t + binary_to_base64_with_lines(const char *input, size_t length, char *output, + size_t line_length, + base64_options options) const noexcept override; const char *find(const char *start, const char *end, - char character) const noexcept; + char character) const noexcept override; const char16_t *find(const char16_t *start, const char16_t *end, - char16_t character) const noexcept; + char16_t character) const noexcept override; private: const bool _supports_zvbb; @@ -8170,42 +7343,47 @@ SIMDUTF_UNTARGET_REGION #endif // SIMDUTF_RVV_H /* end file src/simdutf/rvv.h */ -/* begin file src/simdutf/lsx.h */ -#ifndef SIMDUTF_LSX_H -#define SIMDUTF_LSX_H +/* begin file src/simdutf/lasx.h */ +#ifndef SIMDUTF_LASX_H +#define SIMDUTF_LASX_H #ifdef SIMDUTF_FALLBACK_H - #error "lsx.h must be included before fallback.h" + #error "lasx.h must be included before fallback.h" #endif -#ifndef SIMDUTF_IMPLEMENTATION_LSX - #define SIMDUTF_IMPLEMENTATION_LSX (SIMDUTF_IS_LSX) +#ifndef SIMDUTF_IMPLEMENTATION_LASX + #define SIMDUTF_IMPLEMENTATION_LASX (SIMDUTF_IS_LSX) #endif -#if SIMDUTF_IMPLEMENTATION_LSX && SIMDUTF_IS_LSX - #define SIMDUTF_CAN_ALWAYS_RUN_LSX 1 +#if SIMDUTF_IMPLEMENTATION_LASX && SIMDUTF_IS_LASX + #define SIMDUTF_CAN_ALWAYS_RUN_LASX 1 #else - #define SIMDUTF_CAN_ALWAYS_RUN_LSX 0 + #define SIMDUTF_CAN_ALWAYS_RUN_LASX 0 #endif #define SIMDUTF_CAN_ALWAYS_RUN_FALLBACK (SIMDUTF_IMPLEMENTATION_FALLBACK) -#if SIMDUTF_IMPLEMENTATION_LSX +#if SIMDUTF_IMPLEMENTATION_LASX + #define SIMDUTF_TARGET_LASX SIMDUTF_TARGET_REGION("lasx,lsx") + + // For runtime dispatching to work, we need the lsxintrin to appear + // before we call SIMDUTF_TARGET_LASX. It is unclear why. + #include namespace simdutf { /** - * Implementation for LoongArch SX. + * Implementation for LoongArch ASX. */ -namespace lsx {} // namespace lsx +namespace lasx {} // namespace lasx } // namespace simdutf -/* begin file src/simdutf/lsx/implementation.h */ -#ifndef SIMDUTF_LSX_IMPLEMENTATION_H -#define SIMDUTF_LSX_IMPLEMENTATION_H +/* begin file src/simdutf/lasx/implementation.h */ +#ifndef SIMDUTF_LASX_IMPLEMENTATION_H +#define SIMDUTF_LASX_IMPLEMENTATION_H namespace simdutf { -namespace lsx { +namespace lasx { namespace { using namespace simdutf; @@ -8214,56 +7392,162 @@ using namespace simdutf; class implementation final : public simdutf::implementation { public: simdutf_really_inline implementation() - : simdutf::implementation("lsx", "LOONGARCH SX", - internal::instruction_set::LSX) {} + : simdutf::implementation("lasx", "LOONGARCH ASX", + internal::instruction_set::LSX | + internal::instruction_set::LASX) {} simdutf_warn_unused result base64_to_binary( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; simdutf_warn_unused full_result base64_to_binary_details( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; - simdutf_warn_unused result - base64_to_binary(const char16_t *input, size_t length, char *output, - base64_options options, - last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; + simdutf_warn_unused result base64_to_binary( + const char16_t *input, size_t length, char *output, + base64_options options, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept override; simdutf_warn_unused full_result base64_to_binary_details( const char16_t *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; size_t binary_to_base64(const char *input, size_t length, char *output, - base64_options options) const noexcept; + base64_options options) const noexcept override; + size_t + binary_to_base64_with_lines(const char *input, size_t length, char *output, + size_t line_length, + base64_options options) const noexcept override; const char *find(const char *start, const char *end, - char character) const noexcept; + char character) const noexcept override; const char16_t *find(const char16_t *start, const char16_t *end, - char16_t character) const noexcept; + char16_t character) const noexcept override; + simdutf_warn_unused size_t binary_length_from_base64( + const char *input, size_t length) const noexcept override; + simdutf_warn_unused size_t binary_length_from_base64( + const char16_t *input, size_t length) const noexcept override; }; -} // namespace lsx +} // namespace lasx } // namespace simdutf -#endif // SIMDUTF_LSX_IMPLEMENTATION_H -/* end file src/simdutf/lsx/implementation.h */ +#endif // SIMDUTF_LASX_IMPLEMENTATION_H +/* end file src/simdutf/lasx/implementation.h */ -/* begin file src/simdutf/lsx/begin.h */ -// redefining SIMDUTF_IMPLEMENTATION to "lsx" -// #define SIMDUTF_IMPLEMENTATION lsx +/* begin file src/simdutf/lasx/begin.h */ +// redefining SIMDUTF_IMPLEMENTATION to "lasx" +// #define SIMDUTF_IMPLEMENTATION lasx #define SIMDUTF_SIMD_HAS_UNSIGNED_CMP 1 -/* end file src/simdutf/lsx/begin.h */ + +#if SIMDUTF_CAN_ALWAYS_RUN_LASX +// nothing needed. +#else +SIMDUTF_TARGET_LASX +#endif +/* end file src/simdutf/lasx/begin.h */ // Declarations -/* begin file src/simdutf/lsx/intrinsics.h */ -#ifndef SIMDUTF_LSX_INTRINSICS_H -#define SIMDUTF_LSX_INTRINSICS_H +/* begin file src/simdutf/lasx/intrinsics.h */ +#ifndef SIMDUTF_LASX_INTRINSICS_H +#define SIMDUTF_LASX_INTRINSICS_H // This should be the correct header whether // you use visual studio or other compilers. #include +#include + +#if defined(__loongarch_asx) + #ifdef __clang__ + #define VREGS_PREFIX "$vr" + #define XREGS_PREFIX "$xr" + #else // GCC + #define VREGS_PREFIX "$f" + #define XREGS_PREFIX "$f" + #endif + #define __ALL_REGS \ + "0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26," \ + "27,28,29,30,31" +// Convert __m128i to __m256i +static inline __m256i ____m256i(__m128i in) { + __m256i out = __lasx_xvldi(0); + __asm__ volatile(".irp i," __ALL_REGS "\n\t" + " .ifc %[out], " XREGS_PREFIX "\\i \n\t" + " .irp j," __ALL_REGS "\n\t" + " .ifc %[in], " VREGS_PREFIX "\\j \n\t" + " xvpermi.q $xr\\i, $xr\\j, 0x0 \n\t" + " .endif \n\t" + " .endr \n\t" + " .endif \n\t" + ".endr \n\t" + : [out] "+f"(out) + : [in] "f"(in)); + return out; +} +// Convert two __m128i to __m256i +static inline __m256i lasx_set_q(__m128i inhi, __m128i inlo) { + __m256i out; + __asm__ volatile(".irp i," __ALL_REGS "\n\t" + " .ifc %[hi], " VREGS_PREFIX "\\i \n\t" + " .irp j," __ALL_REGS "\n\t" + " .ifc %[lo], " VREGS_PREFIX "\\j \n\t" + " xvpermi.q $xr\\i, $xr\\j, 0x20 \n\t" + " .endif \n\t" + " .endr \n\t" + " .endif \n\t" + ".endr \n\t" + ".ifnc %[out], %[hi] \n\t" + ".irp i," __ALL_REGS "\n\t" + " .ifc %[out], " XREGS_PREFIX "\\i \n\t" + " .irp j," __ALL_REGS "\n\t" + " .ifc %[hi], " VREGS_PREFIX "\\j \n\t" + " xvori.b $xr\\i, $xr\\j, 0 \n\t" + " .endif \n\t" + " .endr \n\t" + " .endif \n\t" + ".endr \n\t" + ".endif \n\t" + : [out] "=f"(out), [hi] "+f"(inhi) + : [lo] "f"(inlo)); + return out; +} +// Convert __m256i low part to __m128i +static inline __m128i lasx_extracti128_lo(__m256i in) { + __m128i out; + __asm__ volatile(".ifnc %[out], %[in] \n\t" + ".irp i," __ALL_REGS "\n\t" + " .ifc %[out], " VREGS_PREFIX "\\i \n\t" + " .irp j," __ALL_REGS "\n\t" + " .ifc %[in], " XREGS_PREFIX "\\j \n\t" + " vori.b $vr\\i, $vr\\j, 0 \n\t" + " .endif \n\t" + " .endr \n\t" + " .endif \n\t" + ".endr \n\t" + ".endif \n\t" + : [out] "=f"(out) + : [in] "f"(in)); + return out; +} +// Convert __m256i high part to __m128i +static inline __m128i lasx_extracti128_hi(__m256i in) { + __m128i out; + __asm__ volatile(".irp i," __ALL_REGS "\n\t" + " .ifc %[out], " VREGS_PREFIX "\\i \n\t" + " .irp j," __ALL_REGS "\n\t" + " .ifc %[in], " XREGS_PREFIX "\\j \n\t" + " xvpermi.q $xr\\i, $xr\\j, 0x11 \n\t" + " .endif \n\t" + " .endr \n\t" + " .endif \n\t" + ".endr \n\t" + : [out] "=f"(out) + : [in] "f"(in)); + return out; +} +#endif /* Encoding of argument for LoongArch64 xvldi instruction. See: @@ -8293,7 +7577,7 @@ all lanes the result as 64-bit elements to all lanes */ -namespace vldi { +namespace lasx_vldi { template class const_u16 { constexpr static const uint8_t b0 = ((v >> 0 * 8) & 0xff); @@ -8410,7 +7694,8 @@ template class const_u64 { constexpr static int value = int((operation << 8) | byte) - 8192; constexpr static bool valid = operation != 0xffff; }; -} // namespace vldi + +} // namespace lasx_vldi // Uncomment when running under QEMU affected // by bug https://gitlab.com/qemu-project/qemu/-/issues/2865 @@ -8420,45 +7705,78 @@ template class const_u64 { #endif #ifdef QEMU_VLDI_BUG - #define lsx_splat_u16(v) __lsx_vreplgr2vr_h(v) - #define lsx_splat_u32(v) __lsx_vreplgr2vr_w(v) + #define lasx_splat_u16(v) __lasx_xvreplgr2vr_h(v) + #define lasx_splat_u32(v) __lasx_xvreplgr2vr_w(v) #else -template constexpr __m128i lsx_splat_u16_aux() { +template constexpr __m256i lasx_splat_u16_aux() { constexpr bool is_imm10 = (int16_t(x) < 512) && (int16_t(x) > -512); constexpr uint16_t imm10 = is_imm10 ? x : 0; - constexpr bool is_vldi = vldi::const_u16::valid; - constexpr int vldi_imm = is_vldi ? vldi::const_u16::value : 0; + constexpr bool is_vldi = lasx_vldi::const_u16::valid; + constexpr int vldi_imm = is_vldi ? lasx_vldi::const_u16::value : 0; - return is_imm10 ? __lsx_vrepli_h(int16_t(imm10)) - : is_vldi ? __lsx_vldi(vldi_imm) - : __lsx_vreplgr2vr_h(x); + return is_imm10 ? __lasx_xvrepli_h(int16_t(imm10)) + : is_vldi ? __lasx_xvldi(vldi_imm) + : __lasx_xvreplgr2vr_h(x); } -template constexpr __m128i lsx_splat_u32_aux() { +template constexpr __m256i lasx_splat_u32_aux() { constexpr bool is_imm10 = (int32_t(x) < 512) && (int32_t(x) > -512); constexpr uint32_t imm10 = is_imm10 ? x : 0; - constexpr bool is_vldi = vldi::const_u32::valid; - constexpr int vldi_imm = is_vldi ? vldi::const_u32::value : 0; + constexpr bool is_vldi = lasx_vldi::const_u32::valid; + constexpr int vldi_imm = is_vldi ? lasx_vldi::const_u32::value : 0; - return is_imm10 ? __lsx_vrepli_w(int32_t(imm10)) - : is_vldi ? __lsx_vldi(vldi_imm) - : __lsx_vreplgr2vr_w(x); + return is_imm10 ? __lasx_xvrepli_w(int32_t(imm10)) + : is_vldi ? __lasx_xvldi(vldi_imm) + : __lasx_xvreplgr2vr_w(x); } - #define lsx_splat_u16(v) lsx_splat_u16_aux<(v)>() - #define lsx_splat_u32(v) lsx_splat_u32_aux<(v)>() + #define lasx_splat_u16(v) lasx_splat_u16_aux<(v)>() + #define lasx_splat_u32(v) lasx_splat_u32_aux<(v)>() #endif // QEMU_VLDI_BUG -#endif // SIMDUTF_LSX_INTRINSICS_H -/* end file src/simdutf/lsx/intrinsics.h */ -/* begin file src/simdutf/lsx/bitmanipulation.h */ -#ifndef SIMDUTF_LSX_BITMANIPULATION_H -#define SIMDUTF_LSX_BITMANIPULATION_H +#ifndef lsx_splat_u16 + #ifdef QEMU_VLDI_BUG + #define lsx_splat_u16(v) __lsx_vreplgr2vr_h(v) + #define lsx_splat_u32(v) __lsx_vreplgr2vr_w(v) + #else +namespace { +template constexpr __m128i lsx_splat_u16_aux() { + return ((int16_t(x) < 512) && (int16_t(x) > -512)) + ? __lsx_vrepli_h( + ((int16_t(x) < 512) && (int16_t(x) > -512)) ? int16_t(x) : 0) + : (lasx_vldi::const_u16::valid + ? __lsx_vldi(lasx_vldi::const_u16::valid + ? lasx_vldi::const_u16::value + : 0) + : __lsx_vreplgr2vr_h(x)); +} + +template constexpr __m128i lsx_splat_u32_aux() { + return ((int32_t(x) < 512) && (int32_t(x) > -512)) + ? __lsx_vrepli_w( + ((int32_t(x) < 512) && (int32_t(x) > -512)) ? int32_t(x) : 0) + : (lasx_vldi::const_u32::valid + ? __lsx_vldi(lasx_vldi::const_u32::valid + ? lasx_vldi::const_u32::value + : 0) + : __lsx_vreplgr2vr_w(x)); +} +} // namespace + #define lsx_splat_u16(v) lsx_splat_u16_aux<(v)>() + #define lsx_splat_u32(v) lsx_splat_u32_aux<(v)>() + #endif // QEMU_VLDI_BUG +#endif // lsx_splat_u16 + +#endif // SIMDUTF_LASX_INTRINSICS_H +/* end file src/simdutf/lasx/intrinsics.h */ +/* begin file src/simdutf/lasx/bitmanipulation.h */ +#ifndef SIMDUTF_LASX_BITMANIPULATION_H +#define SIMDUTF_LASX_BITMANIPULATION_H #include namespace simdutf { -namespace lsx { +namespace lasx { namespace { simdutf_really_inline int count_ones(uint64_t input_num) { @@ -8472,171 +7790,349 @@ simdutf_really_inline int trailing_zeroes(uint64_t input_num) { #endif } // unnamed namespace -} // namespace lsx +} // namespace lasx } // namespace simdutf -#endif // SIMDUTF_LSX_BITMANIPULATION_H -/* end file src/simdutf/lsx/bitmanipulation.h */ -/* begin file src/simdutf/lsx/simd.h */ -#ifndef SIMDUTF_LSX_SIMD_H -#define SIMDUTF_LSX_SIMD_H +#endif // SIMDUTF_LASX_BITMANIPULATION_H +/* end file src/simdutf/lasx/bitmanipulation.h */ +/* begin file src/simdutf/lasx/simd.h */ +#ifndef SIMDUTF_LASX_SIMD_H +#define SIMDUTF_LASX_SIMD_H namespace simdutf { -namespace lsx { +namespace lasx { namespace { namespace simd { -template struct simd8; +__attribute__((aligned(32))) static const uint8_t prev_shuf_table[32][32] = { + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, + 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}, + {0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, + 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}, + {0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, + 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, + {0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + {0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, + {0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, + 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8}, + {0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, + 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7}, + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, + 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6}, + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, + 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5}, + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, + 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4}, + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3}, + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, + 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2}, + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, + 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1}, + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0}, + {15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, + 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + {14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + {13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, + 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + {12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, + 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + {11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, + 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + {10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, + 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + {9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + {8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, + 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0}, + {7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, + 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0}, + {6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, + 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0}, + {5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0}, + {4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0}, + {3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, + 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0}, + {2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, + 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0}, + {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, +}; -// -// Base class of simd8 and simd8, both of which use __m128i -// internally. -// -template > struct base_u8 { - __m128i value; - static const int SIZE = sizeof(value); +__attribute__((aligned(32))) static const uint8_t bitsel_mask_table[32][32] = { + {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0}}; - // Conversion from/to SIMD register - simdutf_really_inline base_u8(const __m128i _value) : value(_value) {} - simdutf_really_inline operator const __m128i &() const { return this->value; } - simdutf_really_inline operator __m128i &() { return this->value; } +// Forward-declared so they can be used by splat and friends. +template struct base { + __m256i value; + + // Zero constructor + simdutf_really_inline base() : value{__m256i()} {} + + // Conversion from SIMD register + simdutf_really_inline base(const __m256i _value) : value(_value) {} + // Conversion to SIMD register + simdutf_really_inline operator const __m256i &() const { return this->value; } + simdutf_really_inline operator __m256i &() { return this->value; } + template + simdutf_really_inline void store_ascii_as_utf16(char16_t *ptr) const { + if (big_endian) { + __m256i zero = __lasx_xvldi(0); + __m256i in8 = __lasx_xvpermi_d(this->value, 0b11011000); + __m256i inlow = __lasx_xvilvl_b(in8, zero); + __m256i inhigh = __lasx_xvilvh_b(in8, zero); + __lasx_xvst(inlow, reinterpret_cast(ptr), 0); + __lasx_xvst(inhigh, reinterpret_cast(ptr), 32); + } else { + __m256i inlow = __lasx_vext2xv_hu_bu(this->value); + __m256i inhigh = __lasx_vext2xv_hu_bu( + __lasx_xvpermi_q(this->value, this->value, 0b00000001)); + __lasx_xvst(inlow, reinterpret_cast<__m256i *>(ptr), 0); + __lasx_xvst(inhigh, reinterpret_cast<__m256i *>(ptr), 32); + } + } + simdutf_really_inline void store_ascii_as_utf32(char32_t *ptr) const { + __m256i in32_0 = __lasx_vext2xv_wu_bu(this->value); + __lasx_xvst(in32_0, reinterpret_cast(ptr), 0); + __m256i in8_1 = __lasx_xvpermi_d(this->value, 0b00000001); + __m256i in32_1 = __lasx_vext2xv_wu_bu(in8_1); + __lasx_xvst(in32_1, reinterpret_cast(ptr), 32); + + __m256i in8_2 = __lasx_xvpermi_d(this->value, 0b00000010); + __m256i in32_2 = __lasx_vext2xv_wu_bu(in8_2); + __lasx_xvst(in32_2, reinterpret_cast(ptr), 64); + + __m256i in8_3 = __lasx_xvpermi_d(this->value, 0b00000011); + __m256i in32_3 = __lasx_vext2xv_wu_bu(in8_3); + __lasx_xvst(in32_3, reinterpret_cast(ptr), 96); + } // Bit operations - simdutf_really_inline simd8 operator|(const simd8 other) const { - return __lsx_vor_v(this->value, other); + simdutf_really_inline Child operator|(const Child other) const { + return __lasx_xvor_v(this->value, other); } - simdutf_really_inline simd8 operator&(const simd8 other) const { - return __lsx_vand_v(this->value, other); + simdutf_really_inline Child operator&(const Child other) const { + return __lasx_xvand_v(this->value, other); } - simdutf_really_inline simd8 operator^(const simd8 other) const { - return __lsx_vxor_v(this->value, other); + simdutf_really_inline Child operator^(const Child other) const { + return __lasx_xvxor_v(this->value, other); } - simdutf_really_inline simd8 operator~() const { return *this ^ 0xFFu; } - simdutf_really_inline simd8 &operator|=(const simd8 other) { - auto this_cast = static_cast *>(this); + simdutf_really_inline Child &operator|=(const Child other) { + auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } +}; + +template struct simd8; +template > +struct base8 : base> { + simdutf_really_inline base8() : base>() {} + simdutf_really_inline base8(const __m256i _value) : base>(_value) {} friend simdutf_really_inline Mask operator==(const simd8 lhs, const simd8 rhs) { - return __lsx_vseq_b(lhs, rhs); + return __lasx_xvseq_b(lhs, rhs); } - template + static const int SIZE = sizeof(base::value); + + template simdutf_really_inline simd8 prev(const simd8 prev_chunk) const { - return __lsx_vor_v(__lsx_vbsll_v(this->value, N), - __lsx_vbsrl_v(prev_chunk.value, 16 - N)); - } -}; + static_assert(N <= 16, "unsupported shift value"); -// SIMD byte mask type (returned by things like eq and gt) -template <> struct simd8 : base_u8 { - typedef uint16_t bitmask_t; - typedef uint32_t bitmask2_t; + if (!N) + return this->value; - static simdutf_really_inline simd8 splat(bool _value) { - return __lsx_vreplgr2vr_b(uint8_t(-(!!_value))); - } + __m256i zero = __lasx_xvldi(0); + __m256i result, shuf; + if (N < 16) { + shuf = __lasx_xvld(prev_shuf_table[N], 0); - simdutf_really_inline simd8(const __m128i _value) : base_u8(_value) {} - // False constructor - simdutf_really_inline simd8() : simd8(__lsx_vldi(0)) {} - // Splat constructor - simdutf_really_inline simd8(bool _value) : simd8(splat(_value)) {} - simdutf_really_inline void store(uint8_t dst[16]) const { - return __lsx_vst(this->value, dst, 0); - } + result = __lasx_xvshuf_b( + __lasx_xvpermi_q(this->value, this->value, 0b00000001), this->value, + shuf); + __m256i srl_prev = __lasx_xvbsrl_v( + __lasx_xvpermi_q(zero, prev_chunk.value, 0b00110001), (16 - N)); + __m256i mask = __lasx_xvld(bitsel_mask_table[N], 0); + result = __lasx_xvbitsel_v(result, srl_prev, mask); - simdutf_really_inline uint32_t to_bitmask() const { - return __lsx_vpickve2gr_wu(__lsx_vmsknz_b(*this), 0); + return result; + } else if (N == 16) { + return __lasx_xvpermi_q(this->value, prev_chunk.value, 0b00100001); + } } }; -// Unsigned bytes -template <> struct simd8 : base_u8 { - static simdutf_really_inline simd8 splat(uint8_t _value) { - return __lsx_vreplgr2vr_b(_value); - } - static simdutf_really_inline simd8 zero() { return __lsx_vldi(0); } - static simdutf_really_inline simd8 load(const uint8_t *values) { - return __lsx_vld(values, 0); - } - simdutf_really_inline simd8(const __m128i _value) - : base_u8(_value) {} - // Zero constructor - simdutf_really_inline simd8() : simd8(zero()) {} - // Array constructor - simdutf_really_inline simd8(const uint8_t values[16]) : simd8(load(values)) {} - // Splat constructor - simdutf_really_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Member-by-member initialization - simdutf_really_inline - simd8(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, - uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, - uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15) - : simd8((__m128i)v16u8{v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, - v12, v13, v14, v15}) {} - - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdutf_really_inline static simd8 - repeat_16(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, - uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, - uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, - uint8_t v15) { - return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, - v13, v14, v15); +// SIMD byte mask type (returned by things like eq and gt) +template <> struct simd8 : base8 { + static simdutf_really_inline simd8 splat(bool _value) { + return __lasx_xvreplgr2vr_b(uint8_t(-(!!_value))); } - // Store to array - simdutf_really_inline void store(uint8_t dst[16]) const { - return __lsx_vst(this->value, dst, 0); - } + simdutf_really_inline simd8() : base8() {} + simdutf_really_inline simd8(const __m256i _value) : base8(_value) {} + // Splat constructor + simdutf_really_inline simd8(bool _value) : base8(splat(_value)) {} - // Order-specific operations - simdutf_really_inline simd8 - operator>=(const simd8 other) const { - return __lsx_vsle_bu(other, *this); - } - simdutf_really_inline simd8 - operator>(const simd8 other) const { - return __lsx_vslt_bu(other, *this); - } - simdutf_really_inline simd8 &operator-=(const simd8 other) { - value = __lsx_vsub_b(value, other.value); - return *this; + simdutf_really_inline uint32_t to_bitmask() const { + __m256i mask = __lasx_xvmsknz_b(this->value); + uint32_t mask0 = __lasx_xvpickve2gr_wu(mask, 0); + uint32_t mask1 = __lasx_xvpickve2gr_wu(mask, 4); + return (mask0 | (mask1 << 16)); } - // Same as >, but instead of guaranteeing all 1's == true, false = 0 and true - // = nonzero. For ARM, returns all 1's. - simdutf_really_inline simd8 - gt_bits(const simd8 other) const { - return simd8(*this > other); + simdutf_really_inline bool any() const { + if (__lasx_xbz_b(this->value)) + return false; + return true; } + simdutf_really_inline simd8 operator~() const { return *this ^ true; } +}; - // Bit-specific operations - simdutf_really_inline simd8 any_bits_set(simd8 bits) const { - return __lsx_vslt_bu(__lsx_vldi(0), __lsx_vand_v(this->value, bits)); - } - simdutf_really_inline bool is_ascii() const { - return __lsx_vpickve2gr_hu(__lsx_vmskgez_b(this->value), 0) == 0xFFFF; +template struct base8_numeric : base8 { + static simdutf_really_inline simd8 splat(T _value) { + return __lasx_xvreplgr2vr_b(_value); } - - simdutf_really_inline bool any_bits_set_anywhere() const { - return __lsx_vpickve2gr_hu(__lsx_vmsknz_b(this->value), 0) > 0; + static simdutf_really_inline simd8 zero() { return __lasx_xvldi(0); } + static simdutf_really_inline simd8 load(const T values[32]) { + return __lasx_xvld(reinterpret_cast(values), 0); } - template simdutf_really_inline simd8 shr() const { - return __lsx_vsrli_b(this->value, N); + // Repeat 16 values as many times as necessary (usually for lookup tables) + static simdutf_really_inline simd8 repeat_16(T v0, T v1, T v2, T v3, T v4, + T v5, T v6, T v7, T v8, T v9, + T v10, T v11, T v12, T v13, + T v14, T v15) { + return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, + v14, v15, v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, + v12, v13, v14, v15); } - template simdutf_really_inline simd8 shl() const { - return __lsx_vslli_b(this->value, N); + + simdutf_really_inline base8_numeric() : base8() {} + simdutf_really_inline base8_numeric(const __m256i _value) + : base8(_value) {} + + // Store to array + simdutf_really_inline void store(T dst[32]) const { + return __lasx_xvst(this->value, reinterpret_cast<__m256i *>(dst), 0); } + // Override to distinguish from bool version + simdutf_really_inline simd8 operator~() const { return *this ^ 0xFFu; } + // Perform a lookup assuming the value is between 0 and 16 (undefined behavior // for out of range values) template simdutf_really_inline simd8 lookup_16(simd8 lookup_table) const { - return lookup_table.apply_lookup_16_to(*this); + __m256i origin = __lasx_xvand_v(this->value, __lasx_xvldi(0x1f)); + return __lasx_xvshuf_b(__lasx_xvldi(0), lookup_table, origin); } template @@ -8650,150 +8146,119 @@ template <> struct simd8 : base_u8 { replace7, replace8, replace9, replace10, replace11, replace12, replace13, replace14, replace15)); } - - template - simdutf_really_inline simd8 - apply_lookup_16_to(const simd8 original) const { - __m128i original_tmp = __lsx_vand_v(original, __lsx_vldi(0x1f)); - return __lsx_vshuf_b(__lsx_vldi(0), *this, simd8(original_tmp)); - } - - simdutf_really_inline uint64_t sum_bytes() const { - const auto sum_u16 = __lsx_vhaddw_hu_bu(value, value); - const auto sum_u32 = __lsx_vhaddw_wu_hu(sum_u16, sum_u16); - const auto sum_u64 = __lsx_vhaddw_du_wu(sum_u32, sum_u32); - - return uint64_t(__lsx_vpickve2gr_du(sum_u64, 0)) + - uint64_t(__lsx_vpickve2gr_du(sum_u64, 1)); - } }; // Signed bytes -template <> struct simd8 { - __m128i value; - - static const int SIZE = sizeof(value); - - static simdutf_really_inline simd8 splat(int8_t _value) { - return __lsx_vreplgr2vr_b(_value); - } - static simdutf_really_inline simd8 zero() { return __lsx_vldi(0); } - static simdutf_really_inline simd8 load(const int8_t values[16]) { - return __lsx_vld(values, 0); - } +template <> struct simd8 : base8_numeric { + simdutf_really_inline simd8() : base8_numeric() {} + simdutf_really_inline simd8(const __m256i _value) + : base8_numeric(_value) {} - template - simdutf_really_inline void store_ascii_as_utf16(char16_t *p) const { - __m128i zero = __lsx_vldi(0); - if (match_system(big_endian)) { - __lsx_vst(__lsx_vilvl_b(zero, (__m128i)this->value), - reinterpret_cast(p), 0); - __lsx_vst(__lsx_vilvh_b(zero, (__m128i)this->value), - reinterpret_cast(p + 8), 0); - } else { - __lsx_vst(__lsx_vilvl_b((__m128i)this->value, zero), - reinterpret_cast(p), 0); - __lsx_vst(__lsx_vilvh_b((__m128i)this->value, zero), - reinterpret_cast(p + 8), 0); - } + // Splat constructor + simdutf_really_inline simd8(int8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdutf_really_inline simd8(const int8_t values[32]) : simd8(load(values)) {} + simdutf_really_inline operator simd8() const; + simdutf_really_inline bool is_ascii() const { + __m256i ascii_mask = __lasx_xvslti_b(this->value, 0); + if (__lasx_xbnz_v(ascii_mask)) + return false; + return true; } - - simdutf_really_inline void store_ascii_as_utf32(char32_t *p) const { - __m128i zero = __lsx_vldi(0); - __m128i in16low = __lsx_vilvl_b(zero, (__m128i)this->value); - __m128i in16high = __lsx_vilvh_b(zero, (__m128i)this->value); - __m128i in32_0 = __lsx_vilvl_h(zero, in16low); - __m128i in32_1 = __lsx_vilvh_h(zero, in16low); - __m128i in32_2 = __lsx_vilvl_h(zero, in16high); - __m128i in32_3 = __lsx_vilvh_h(zero, in16high); - __lsx_vst(in32_0, reinterpret_cast(p), 0); - __lsx_vst(in32_1, reinterpret_cast(p + 4), 0); - __lsx_vst(in32_2, reinterpret_cast(p + 8), 0); - __lsx_vst(in32_3, reinterpret_cast(p + 12), 0); + // Order-sensitive comparisons + simdutf_really_inline simd8 operator>(const simd8 other) const { + return __lasx_xvslt_b(other, this->value); } - - // In places where the table can be reused, which is most uses in simdutf, it - // is worth it to do 4 table lookups, as there is no direct zero extension - // from u8 to u32. - simdutf_really_inline void store_ascii_as_utf32_tbl(char32_t *p) const { - const simd8 tb1{0, 255, 255, 255, 1, 255, 255, 255, - 2, 255, 255, 255, 3, 255, 255, 255}; - const simd8 tb2{4, 255, 255, 255, 5, 255, 255, 255, - 6, 255, 255, 255, 7, 255, 255, 255}; - const simd8 tb3{8, 255, 255, 255, 9, 255, 255, 255, - 10, 255, 255, 255, 11, 255, 255, 255}; - const simd8 tb4{12, 255, 255, 255, 13, 255, 255, 255, - 14, 255, 255, 255, 15, 255, 255, 255}; - - // encourage store pairing and interleaving - const auto shuf1 = this->apply_lookup_16_to(tb1); - const auto shuf2 = this->apply_lookup_16_to(tb2); - shuf1.store(reinterpret_cast(p)); - shuf2.store(reinterpret_cast(p + 4)); - - const auto shuf3 = this->apply_lookup_16_to(tb3); - const auto shuf4 = this->apply_lookup_16_to(tb4); - shuf3.store(reinterpret_cast(p + 8)); - shuf4.store(reinterpret_cast(p + 12)); + simdutf_really_inline simd8 operator<(const simd8 other) const { + return __lasx_xvslt_b(this->value, other); } - // Conversion from/to SIMD register - simdutf_really_inline simd8(const __m128i _value) : value(_value) {} +}; - // Zero constructor - simdutf_really_inline simd8() : simd8(zero()) {} +// Unsigned bytes +template <> struct simd8 : base8_numeric { + simdutf_really_inline simd8() : base8_numeric() {} + simdutf_really_inline simd8(const __m256i _value) + : base8_numeric(_value) {} // Splat constructor - simdutf_really_inline simd8(int8_t _value) : simd8(splat(_value)) {} + simdutf_really_inline simd8(uint8_t _value) : simd8(splat(_value)) {} // Array constructor - simdutf_really_inline simd8(const int8_t *values) : simd8(load(values)) {} + simdutf_really_inline simd8(const uint8_t values[32]) : simd8(load(values)) {} + // Member-by-member initialization + simdutf_really_inline + simd8(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, + uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, + uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15, + uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20, + uint8_t v21, uint8_t v22, uint8_t v23, uint8_t v24, uint8_t v25, + uint8_t v26, uint8_t v27, uint8_t v28, uint8_t v29, uint8_t v30, + uint8_t v31) + : simd8((__m256i)v32u8{v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v16, v17, v18, v19, v20, v21, v22, v23, + v24, v25, v26, v27, v28, v29, v30, v31}) {} - // Store to array - simdutf_really_inline void store(int8_t dst[16]) const { - return __lsx_vst(value, dst, 0); + // Saturated math + simdutf_really_inline simd8 + saturating_sub(const simd8 other) const { + return __lasx_xvssub_bu(this->value, other); } - simdutf_really_inline operator simd8() const { - return ((__m128i)this->value); + // Same as >, but only guarantees true is nonzero (< guarantees true = -1) + simdutf_really_inline simd8 + gt_bits(const simd8 other) const { + return this->saturating_sub(other); } - - simdutf_really_inline simd8 - operator|(const simd8 other) const { - return __lsx_vor_v((__m128i)value, (__m128i)other.value); + simdutf_really_inline simd8 + operator>=(const simd8 other) const { + return __lasx_xvsle_bu(other, *this); + } + simdutf_really_inline simd8 + operator>(const simd8 other) const { + return __lasx_xvslt_bu(other, *this); + } + simdutf_really_inline simd8 &operator-=(const simd8 other) { + value = __lasx_xvsub_b(value, other.value); + return *this; } + // Bit-specific operations simdutf_really_inline bool is_ascii() const { - return (__lsx_vpickve2gr_hu(__lsx_vmskgez_b((__m128i)this->value), 0) == - 0xffff); + __m256i ascii_mask = __lasx_xvslti_b(this->value, 0); + if (__lasx_xbnz_v(ascii_mask)) + return false; + return true; } - - // Order-sensitive comparisons - simdutf_really_inline simd8 operator>(const simd8 other) const { - return __lsx_vslt_b((__m128i)other.value, (__m128i)value); + simdutf_really_inline bool any_bits_set_anywhere() const { + if (__lasx_xbnz_v(this->value)) + return true; + return false; } - simdutf_really_inline simd8 operator<(const simd8 other) const { - return __lsx_vslt_b((__m128i)value, (__m128i)other.value); + template simdutf_really_inline simd8 shr() const { + return __lasx_xvsrli_b(this->value, N); } - - template - simdutf_really_inline simd8 - prev(const simd8 prev_chunk) const { - return __lsx_vor_v(__lsx_vbsll_v(this->value, N), - __lsx_vbsrl_v(prev_chunk.value, 16 - N)); + template simdutf_really_inline simd8 shl() const { + return __lasx_xvslli_b(this->value, N); } - template - simdutf_really_inline simd8 - apply_lookup_16_to(const simd8 original) const { - __m128i original_tmp = __lsx_vand_v(original, __lsx_vldi(0x1f)); - return __lsx_vshuf_b(__lsx_vldi(0), (__m128i)this->value, - simd8(original_tmp)); + simdutf_really_inline uint64_t sum_bytes() const { + const auto sum_u16 = __lasx_xvhaddw_hu_bu(value, value); + const auto sum_u32 = __lasx_xvhaddw_wu_hu(sum_u16, sum_u16); + const auto sum_u64 = __lasx_xvhaddw_du_wu(sum_u32, sum_u32); + + return uint64_t(__lasx_xvpickve2gr_du(sum_u64, 0)) + + uint64_t(__lasx_xvpickve2gr_du(sum_u64, 1)) + + uint64_t(__lasx_xvpickve2gr_du(sum_u64, 2)) + + uint64_t(__lasx_xvpickve2gr_du(sum_u64, 3)); } }; +simdutf_really_inline simd8::operator simd8() const { + return this->value; +} template struct simd8x64 { static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert( - NUM_CHUNKS == 4, - "LoongArch kernel should use four registers per 64-byte block."); + static_assert(NUM_CHUNKS == 2, + "LASX kernel should use two registers per 64-byte block."); simd8 chunks[NUM_CHUNKS]; simd8x64(const simd8x64 &o) = delete; // no copy allowed @@ -8801,36 +8266,36 @@ template struct simd8x64 { operator=(const simd8 other) = delete; // no assignment allowed simd8x64() = delete; // no default constructor allowed - simdutf_really_inline simd8x64(const simd8 chunk0, const simd8 chunk1, - const simd8 chunk2, const simd8 chunk3) - : chunks{chunk0, chunk1, chunk2, chunk3} {} + simdutf_really_inline simd8x64(const simd8 chunk0, const simd8 chunk1) + : chunks{chunk0, chunk1} {} simdutf_really_inline simd8x64(const T *ptr) : chunks{simd8::load(ptr), - simd8::load(ptr + sizeof(simd8) / sizeof(T)), - simd8::load(ptr + 2 * sizeof(simd8) / sizeof(T)), - simd8::load(ptr + 3 * sizeof(simd8) / sizeof(T))} {} + simd8::load(ptr + sizeof(simd8) / sizeof(T))} {} simdutf_really_inline void store(T *ptr) const { this->chunks[0].store(ptr + sizeof(simd8) * 0 / sizeof(T)); this->chunks[1].store(ptr + sizeof(simd8) * 1 / sizeof(T)); - this->chunks[2].store(ptr + sizeof(simd8) * 2 / sizeof(T)); - this->chunks[3].store(ptr + sizeof(simd8) * 3 / sizeof(T)); + } + + simdutf_really_inline uint64_t to_bitmask() const { + uint64_t r_lo = uint32_t(this->chunks[0].to_bitmask()); + uint64_t r_hi = this->chunks[1].to_bitmask(); + return r_lo | (r_hi << 32); } simdutf_really_inline simd8x64 &operator|=(const simd8x64 &other) { this->chunks[0] |= other.chunks[0]; this->chunks[1] |= other.chunks[1]; - this->chunks[2] |= other.chunks[2]; - this->chunks[3] |= other.chunks[3]; return *this; } simdutf_really_inline simd8 reduce_or() const { - return (this->chunks[0] | this->chunks[1]) | - (this->chunks[2] | this->chunks[3]); + return this->chunks[0] | this->chunks[1]; } - simdutf_really_inline bool is_ascii() const { return reduce_or().is_ascii(); } + simdutf_really_inline bool is_ascii() const { + return this->reduce_or().is_ascii(); + } template simdutf_really_inline void store_ascii_as_utf16(char16_t *ptr) const { @@ -8838,152 +8303,123 @@ template struct simd8x64 { sizeof(simd8) * 0); this->chunks[1].template store_ascii_as_utf16(ptr + sizeof(simd8) * 1); - this->chunks[2].template store_ascii_as_utf16(ptr + - sizeof(simd8) * 2); - this->chunks[3].template store_ascii_as_utf16(ptr + - sizeof(simd8) * 3); } simdutf_really_inline void store_ascii_as_utf32(char32_t *ptr) const { - this->chunks[0].store_ascii_as_utf32_tbl(ptr + sizeof(simd8) * 0); - this->chunks[1].store_ascii_as_utf32_tbl(ptr + sizeof(simd8) * 1); - this->chunks[2].store_ascii_as_utf32_tbl(ptr + sizeof(simd8) * 2); - this->chunks[3].store_ascii_as_utf32_tbl(ptr + sizeof(simd8) * 3); + this->chunks[0].store_ascii_as_utf32(ptr + sizeof(simd8) * 0); + this->chunks[1].store_ascii_as_utf32(ptr + sizeof(simd8) * 1); } - simdutf_really_inline uint64_t to_bitmask() const { - __m128i mask = __lsx_vbsll_v(__lsx_vmsknz_b(this->chunks[3]), 6); - mask = __lsx_vor_v(mask, __lsx_vbsll_v(__lsx_vmsknz_b(this->chunks[2]), 4)); - mask = __lsx_vor_v(mask, __lsx_vbsll_v(__lsx_vmsknz_b(this->chunks[1]), 2)); - mask = __lsx_vor_v(mask, __lsx_vmsknz_b(this->chunks[0])); - return __lsx_vpickve2gr_du(mask, 0); + simdutf_really_inline uint64_t lt(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64(this->chunks[0] < mask, this->chunks[1] < mask) + .to_bitmask(); } - simdutf_really_inline uint64_t lt(const T m) const { + simdutf_really_inline uint64_t gteq(const T m) const { const simd8 mask = simd8::splat(m); - return simd8x64(this->chunks[0] < mask, this->chunks[1] < mask, - this->chunks[2] < mask, this->chunks[3] < mask) + return simd8x64(this->chunks[0] >= mask, this->chunks[1] >= mask) .to_bitmask(); } + simdutf_really_inline uint64_t gt(const T m) const { const simd8 mask = simd8::splat(m); - return simd8x64(this->chunks[0] > mask, this->chunks[1] > mask, - this->chunks[2] > mask, this->chunks[3] > mask) + return simd8x64(this->chunks[0] > mask, this->chunks[1] > mask) .to_bitmask(); } simdutf_really_inline uint64_t gteq_unsigned(const uint8_t m) const { const simd8 mask = simd8::splat(m); - return simd8x64(simd8(this->chunks[0].value) >= mask, - simd8(this->chunks[1].value) >= mask, - simd8(this->chunks[2].value) >= mask, - simd8(this->chunks[3].value) >= mask) + return simd8x64((simd8(__m256i(this->chunks[0])) >= mask), + (simd8(__m256i(this->chunks[1])) >= mask)) .to_bitmask(); } }; // struct simd8x64 -/* begin file src/simdutf/lsx/simd16-inl.h */ +/* begin file src/simdutf/lasx/simd16-inl.h */ template struct simd16; -template > struct base_u16 { - __m128i value; - static const size_t SIZE = sizeof(value); - static const size_t ELEMENTS = sizeof(value) / sizeof(T); - - // Conversion from/to SIMD register - simdutf_really_inline base_u16() = default; - simdutf_really_inline base_u16(const __m128i _value) : value(_value) {} - // Bit operations - simdutf_really_inline simd16 operator|(const simd16 other) const { - return __lsx_vor_v(this->value, other.value); - } - simdutf_really_inline simd16 operator&(const simd16 other) const { - return __lsx_vand_v(this->value, other.value); - } - simdutf_really_inline simd16 operator~() const { - return __lsx_vxori_b(this->value, 0xFF); - } - - friend simdutf_really_inline Mask operator==(const simd16 lhs, - const simd16 rhs) { - return __lsx_vseq_h(lhs.value, rhs.value); - } - - template - simdutf_really_inline simd16 byte_right_shift() const { - return __lsx_vbsrl_v(this->value, N); - } - - simdutf_really_inline uint16_t first() const { - return uint16_t(__lsx_vpickve2gr_w(value, 0)); - } -}; - template > -struct base16 : base_u16 { - using bitmask_type = uint16_t; +struct base16 : base> { + using bitmask_type = uint32_t; - simdutf_really_inline base16() : base_u16() {} - simdutf_really_inline base16(const __m128i _value) : base_u16(_value) {} + simdutf_really_inline base16() : base>() {} + simdutf_really_inline base16(const __m256i _value) + : base>(_value) {} template simdutf_really_inline base16(const Pointer *ptr) - : base16(__lsx_vld(ptr, 0)) {} + : base16(__lasx_xvld(reinterpret_cast(ptr), 0)) {} - static const int SIZE = sizeof(base_u16::value); + /// the size of vector in bytes + static const int SIZE = sizeof(base>::value); - template - simdutf_really_inline simd16 prev(const simd16 prev_chunk) const { - return __lsx_vor_v(__lsx_vbsll_v(*this, N * 2), - __lsx_vbsrl_v(prev_chunk, 16 - N * 2)); - } + /// the number of elements of type T a vector can hold + static const int ELEMENTS = SIZE / sizeof(T); }; // SIMD byte mask type (returned by things like eq and gt) template <> struct simd16 : base16 { static simdutf_really_inline simd16 splat(bool _value) { - return __lsx_vreplgr2vr_h(uint16_t(-(!!_value))); + return __lasx_xvreplgr2vr_h(uint16_t(-(!!_value))); } simdutf_really_inline simd16() : base16() {} - simdutf_really_inline simd16(const __m128i _value) : base16(_value) {} + simdutf_really_inline simd16(const __m256i _value) : base16(_value) {} + // Splat constructor + simdutf_really_inline simd16(bool _value) : base16(splat(_value)) {} simdutf_really_inline bitmask_type to_bitmask() const { - __m128i mask = __lsx_vmsknz_b(this->value); - bitmask_type mask0 = bitmask_type(__lsx_vpickve2gr_wu(mask, 0)); - return mask0; + __m256i mask = __lasx_xvmsknz_b(this->value); + bitmask_type mask0 = __lasx_xvpickve2gr_wu(mask, 0); + bitmask_type mask1 = __lasx_xvpickve2gr_wu(mask, 4); + return (mask0 | (mask1 << 16)); } + simdutf_really_inline simd16 operator~() const { return *this ^ true; } - simdutf_really_inline bool is_zero() const { return __lsx_bz_v(this->value); } + simdutf_really_inline bool is_zero() const { + return __lasx_xbz_v(this->value); + } + + template simdutf_really_inline simd16 byte_right_shift() const { + const auto t0 = __lasx_xvbsrl_v(this->value, N); + const auto t1 = __lasx_xvpermi_q(this->value, __lasx_xvldi(0), 0b00000011); + const auto t2 = __lasx_xvbsll_v(t1, 16 - N); + const auto t3 = __lasx_xvor_v(t0, t2); + return t3; + } + + simdutf_really_inline uint16_t first() const { + return uint16_t(__lasx_xvpickve2gr_w(value, 0)); + } }; template struct base16_numeric : base16 { static simdutf_really_inline simd16 splat(T _value) { - return __lsx_vreplgr2vr_h(_value); + return __lasx_xvreplgr2vr_h((uint16_t)_value); } - static simdutf_really_inline simd16 zero() { return __lsx_vldi(0); } - + static simdutf_really_inline simd16 zero() { return __lasx_xvldi(0); } template static simdutf_really_inline simd16 load(const Pointer values) { - return __lsx_vld(values, 0); + return __lasx_xvld(values, 0); } - simdutf_really_inline base16_numeric(const __m128i _value) + simdutf_really_inline base16_numeric() : base16() {} + simdutf_really_inline base16_numeric(const __m256i _value) : base16(_value) {} // Store to array simdutf_really_inline void store(T dst[8]) const { - return __lsx_vst(this->value, dst, 0); + return __lasx_xvst(this->value, reinterpret_cast<__m256i *>(dst), 0); } // Override to distinguish from bool version - simdutf_really_inline simd16 operator~() const { - return __lsx_vxori_b(this->value, 0xFF); - } + simdutf_really_inline simd16 operator~() const { return *this ^ 0xFFFFu; } }; -// Unsigned code unitstemplate<> +// Unsigned code units template <> struct simd16 : base16_numeric { - simdutf_really_inline simd16(const __m128i _value) - : base16_numeric((__m128i)_value) {} + simdutf_really_inline simd16() : base16_numeric() {} + simdutf_really_inline simd16(const __m256i _value) + : base16_numeric(_value) {} // Splat constructor simdutf_really_inline simd16(uint16_t _value) : simd16(splat(_value)) {} @@ -8993,67 +8429,71 @@ template <> struct simd16 : base16_numeric { simdutf_really_inline simd16(const char16_t *values) : simd16(load(reinterpret_cast(values))) {} - // Copy constructor - simdutf_really_inline simd16(const simd16 mask) : simd16(mask.value) {} - // Order-specific operations simdutf_really_inline simd16 &operator+=(const simd16 other) { - value = __lsx_vadd_h(value, other.value); + value = __lasx_xvadd_h(value, other.value); return *this; } + // Change the endianness + simdutf_really_inline simd16 swap_bytes() const { + return __lasx_xvshuf4i_b(this->value, 0b10110001); + } + template static simdutf_really_inline simd8 pack_shifted_right(const simd16 &v0, const simd16 &v1) { - return __lsx_vssrlni_bu_h(v1.value, v0.value, N); + return __lasx_xvpermi_d(__lasx_xvssrlni_bu_h(v1.value, v0.value, N), + 0b11011000); } // Pack with the unsigned saturation of two uint16_t code units into single // uint8_t vector static simdutf_really_inline simd8 pack(const simd16 &v0, const simd16 &v1) { - return pack_shifted_right<0>(v0, v1); - } - // Change the endianness - simdutf_really_inline simd16 swap_bytes() const { - return __lsx_vshuf4i_b(this->value, 0b10110001); + return pack_shifted_right<0>(v0, v1); } simdutf_really_inline uint64_t sum() const { - const auto sum_u32 = __lsx_vhaddw_wu_hu(value, value); - const auto sum_u64 = __lsx_vhaddw_du_wu(sum_u32, sum_u32); + const auto sum_u32 = __lasx_xvhaddw_wu_hu(value, value); + const auto sum_u64 = __lasx_xvhaddw_du_wu(sum_u32, sum_u32); - return uint64_t(__lsx_vpickve2gr_du(sum_u64, 0)) + - uint64_t(__lsx_vpickve2gr_du(sum_u64, 1)); + return uint64_t(__lasx_xvpickve2gr_du(sum_u64, 0)) + + uint64_t(__lasx_xvpickve2gr_du(sum_u64, 1)) + + uint64_t(__lasx_xvpickve2gr_du(sum_u64, 2)) + + uint64_t(__lasx_xvpickve2gr_du(sum_u64, 3)); + } + + template simdutf_really_inline simd16 byte_right_shift() const { + return __lasx_xvbsrl_v(this->value, N); } }; simdutf_really_inline simd16 operator<(const simd16 a, const simd16 b) { - return __lsx_vslt_hu(a.value, b.value); + return __lasx_xvslt_hu(a.value, b.value); } simdutf_really_inline simd16 operator>(const simd16 a, const simd16 b) { - return __lsx_vslt_hu(b.value, a.value); + return __lasx_xvslt_hu(b.value, a.value); } simdutf_really_inline simd16 operator<=(const simd16 a, const simd16 b) { - return __lsx_vsle_hu(a.value, b.value); + return __lasx_xvsle_hu(a.value, b.value); } simdutf_really_inline simd16 operator>=(const simd16 a, const simd16 b) { - return __lsx_vsle_hu(b.value, a.value); + return __lasx_xvsle_hu(b.value, a.value); } template struct simd16x32 { static constexpr int NUM_CHUNKS = 64 / sizeof(simd16); - static_assert( - NUM_CHUNKS == 4, - "LOONGARCH kernel should use four registers per 64-byte block."); + static_assert(NUM_CHUNKS == 2, + "LASX kernel should use two registers per 64-byte block."); simd16 chunks[NUM_CHUNKS]; simd16x32(const simd16x32 &o) = delete; // no copy allowed @@ -9061,127 +8501,144 @@ template struct simd16x32 { operator=(const simd16 other) = delete; // no assignment allowed simd16x32() = delete; // no default constructor allowed - simdutf_really_inline - simd16x32(const simd16 chunk0, const simd16 chunk1, - const simd16 chunk2, const simd16 chunk3) - : chunks{chunk0, chunk1, chunk2, chunk3} {} + simdutf_really_inline simd16x32(const simd16 chunk0, + const simd16 chunk1) + : chunks{chunk0, chunk1} {} simdutf_really_inline simd16x32(const T *ptr) : chunks{simd16::load(ptr), - simd16::load(ptr + sizeof(simd16) / sizeof(T)), - simd16::load(ptr + 2 * sizeof(simd16) / sizeof(T)), - simd16::load(ptr + 3 * sizeof(simd16) / sizeof(T))} {} + simd16::load(ptr + sizeof(simd16) / sizeof(T))} {} simdutf_really_inline void store(T *ptr) const { this->chunks[0].store(ptr + sizeof(simd16) * 0 / sizeof(T)); this->chunks[1].store(ptr + sizeof(simd16) * 1 / sizeof(T)); - this->chunks[2].store(ptr + sizeof(simd16) * 2 / sizeof(T)); - this->chunks[3].store(ptr + sizeof(simd16) * 3 / sizeof(T)); } simdutf_really_inline void swap_bytes() { this->chunks[0] = this->chunks[0].swap_bytes(); this->chunks[1] = this->chunks[1].swap_bytes(); - this->chunks[2] = this->chunks[2].swap_bytes(); - this->chunks[3] = this->chunks[3].swap_bytes(); } simdutf_really_inline uint64_t to_bitmask() const { - uint64_t r0 = uint32_t(this->chunks[0].to_bitmask()); - uint64_t r1 = this->chunks[1].to_bitmask(); - uint64_t r2 = this->chunks[2].to_bitmask(); - uint64_t r3 = this->chunks[3].to_bitmask(); - return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); + uint64_t r_lo = uint32_t(this->chunks[0].to_bitmask()); + uint64_t r_hi = this->chunks[1].to_bitmask(); + return r_lo | (r_hi << 32); + } + simdutf_really_inline uint64_t gteq(const T m) const { + const simd16 mask = simd16::splat(m); + return simd16x32(this->chunks[0] >= mask, this->chunks[1] >= mask) + .to_bitmask(); } simdutf_really_inline uint64_t lteq(const T m) const { const simd16 mask = simd16::splat(m); - return simd16x32(this->chunks[0] <= mask, this->chunks[1] <= mask, - this->chunks[2] <= mask, this->chunks[3] <= mask) + return simd16x32(this->chunks[0] <= mask, this->chunks[1] <= mask) .to_bitmask(); } }; // struct simd16x32 -simdutf_really_inline simd16 operator^(const simd16 a, +simdutf_really_inline simd16 min(const simd16 a, + const simd16 b) { + return __lasx_xvmin_hu(a.value, b.value); +} + +simdutf_really_inline simd16 operator==(const simd16 a, + uint16_t b) { + const auto bv = __lasx_xvreplgr2vr_h(b); + return __lasx_xvseq_h(a.value, bv); +} + +simdutf_really_inline simd16 as_vector_u16(const simd16 x) { + return x.value; +} + +simdutf_really_inline simd16 operator&(const simd16 a, uint16_t b) { - const auto bv = __lsx_vreplgr2vr_h(b); - return __lsx_vxor_v(a.value, bv); + const auto bv = __lasx_xvreplgr2vr_h(b); + return __lasx_xvand_v(a.value, bv); } -simdutf_really_inline simd16 operator^(const simd16 a, - const simd16 b) { - return __lsx_vxor_v(a.value, b.value); +simdutf_really_inline simd16 operator&(const simd16 a, + const simd16 b) { + return __lasx_xvand_v(a.value, b.value); } -simdutf_really_inline simd16 min(const simd16 a, - const simd16 b) { - return __lsx_vmin_hu(a.value, b.value); +simdutf_really_inline simd16 operator^(const simd16 a, + uint16_t b) { + const auto bv = __lasx_xvreplgr2vr_h(b); + return __lasx_xvxor_v(a.value, bv); } -simdutf_really_inline simd16 as_vector_u16(const simd16 x) { - return x.value; +simdutf_really_inline simd16 operator^(const simd16 a, + const simd16 b) { + return __lasx_xvxor_v(a.value, b.value); } -/* end file src/simdutf/lsx/simd16-inl.h */ -/* begin file src/simdutf/lsx/simd32-inl.h */ +/* end file src/simdutf/lasx/simd16-inl.h */ +/* begin file src/simdutf/lasx/simd32-inl.h */ template struct simd32; template <> struct simd32 { - __m128i value; + __m256i value; static const int SIZE = sizeof(value); static const int ELEMENTS = SIZE / sizeof(uint32_t); // constructors - simdutf_really_inline simd32(__m128i v) : value(v) {} + simdutf_really_inline simd32(__m256i v) : value(v) {} template - simdutf_really_inline simd32(Ptr *ptr) : value(__lsx_vld(ptr, 0)) {} + simdutf_really_inline simd32(Ptr *ptr) : value(__lasx_xvld(ptr, 0)) {} // in-place operators simdutf_really_inline simd32 &operator-=(const simd32 other) { - value = __lsx_vsub_w(value, other.value); + value = __lasx_xvsub_w(value, other.value); return *this; } // members simdutf_really_inline uint64_t sum() const { - return uint64_t(__lsx_vpickve2gr_wu(value, 0)) + - uint64_t(__lsx_vpickve2gr_wu(value, 1)) + - uint64_t(__lsx_vpickve2gr_wu(value, 2)) + - uint64_t(__lsx_vpickve2gr_wu(value, 3)); + const auto odd = __lasx_xvsrli_d(value, 32); + const auto even = __lasx_xvand_v(value, __lasx_xvreplgr2vr_d(0xffffffff)); + + const auto sum64 = __lasx_xvadd_d(odd, even); + + return uint64_t(__lasx_xvpickve2gr_du(sum64, 0)) + + uint64_t(__lasx_xvpickve2gr_du(sum64, 1)) + + uint64_t(__lasx_xvpickve2gr_du(sum64, 2)) + + uint64_t(__lasx_xvpickve2gr_du(sum64, 3)); } // static members static simdutf_really_inline simd32 splat(uint32_t x) { - return __lsx_vreplgr2vr_w(x); + return __lasx_xvreplgr2vr_w(x); } static simdutf_really_inline simd32 zero() { - return __lsx_vrepli_w(0); + return __lasx_xvrepli_w(0); } }; // ------------------------------------------------------------ template <> struct simd32 { - __m128i value; + __m256i value; static const int SIZE = sizeof(value); // constructors - simdutf_really_inline simd32(__m128i v) : value(v) {} + simdutf_really_inline simd32(__m256i v) : value(v) {} }; // ------------------------------------------------------------ simdutf_really_inline simd32 operator&(const simd32 a, const simd32 b) { - return __lsx_vor_v(a.value, b.value); + return __lasx_xvor_v(a.value, b.value); } simdutf_really_inline simd32 operator<(const simd32 a, const simd32 b) { - return __lsx_vslt_wu(a.value, b.value); + return __lasx_xvslt_wu(a.value, b.value); } simdutf_really_inline simd32 operator>(const simd32 a, const simd32 b) { - return __lsx_vslt_wu(b.value, a.value); + return __lasx_xvslt_wu(b.value, a.value); } // ------------------------------------------------------------ @@ -9189,112 +8646,128 @@ simdutf_really_inline simd32 operator>(const simd32 a, simdutf_really_inline simd32 as_vector_u32(const simd32 v) { return v.value; } -/* end file src/simdutf/lsx/simd32-inl.h */ -/* begin file src/simdutf/lsx/simd64-inl.h */ +/* end file src/simdutf/lasx/simd32-inl.h */ +/* begin file src/simdutf/lasx/simd64-inl.h */ template struct simd64; template <> struct simd64 { - __m128i value; + __m256i value; static const int SIZE = sizeof(value); static const int ELEMENTS = SIZE / sizeof(uint64_t); // constructors - simdutf_really_inline simd64(__m128i v) : value(v) {} + simdutf_really_inline simd64(__m256i v) : value(v) {} template - simdutf_really_inline simd64(Ptr *ptr) : value(__lsx_vld(ptr, 0)) {} + simdutf_really_inline simd64(Ptr *ptr) : value(__lasx_xvld(ptr, 0)) {} // in-place operators simdutf_really_inline simd64 &operator+=(const simd64 other) { - value = __lsx_vadd_d(value, other.value); + value = __lasx_xvadd_d(value, other.value); return *this; } // members simdutf_really_inline uint64_t sum() const { - return uint64_t(__lsx_vpickve2gr_du(value, 0)) + - uint64_t(__lsx_vpickve2gr_du(value, 1)); + return uint64_t(__lasx_xvpickve2gr_du(value, 0)) + + uint64_t(__lasx_xvpickve2gr_du(value, 1)) + + uint64_t(__lasx_xvpickve2gr_du(value, 2)) + + uint64_t(__lasx_xvpickve2gr_du(value, 3)); } // static members static simdutf_really_inline simd64 zero() { - return __lsx_vrepli_d(0); + return __lasx_xvrepli_d(0); } }; // ------------------------------------------------------------ template <> struct simd64 { - __m128i value; + __m256i value; static const int SIZE = sizeof(value); // constructors - simdutf_really_inline simd64(__m128i v) : value(v) {} + simdutf_really_inline simd64(__m256i v) : value(v) {} }; // ------------------------------------------------------------ simd64 sum_8bytes(const simd8 v) { - const auto sum_u16 = __lsx_vhaddw_hu_bu(v, v); - const auto sum_u32 = __lsx_vhaddw_wu_hu(sum_u16, sum_u16); - const auto sum_u64 = __lsx_vhaddw_du_wu(sum_u32, sum_u32); + const auto sum_u16 = __lasx_xvhaddw_hu_bu(v, v); + const auto sum_u32 = __lasx_xvhaddw_wu_hu(sum_u16, sum_u16); + const auto sum_u64 = __lasx_xvhaddw_du_wu(sum_u32, sum_u32); return simd64(sum_u64); } -/* end file src/simdutf/lsx/simd64-inl.h */ +/* end file src/simdutf/lasx/simd64-inl.h */ } // namespace simd } // unnamed namespace -} // namespace lsx +} // namespace lasx } // namespace simdutf -#endif // SIMDUTF_LSX_SIMD_H -/* end file src/simdutf/lsx/simd.h */ +#endif // SIMDUTF_LASX_SIMD_H +/* end file src/simdutf/lasx/simd.h */ -/* begin file src/simdutf/lsx/end.h */ +/* begin file src/simdutf/lasx/end.h */ #undef SIMDUTF_SIMD_HAS_UNSIGNED_CMP -/* end file src/simdutf/lsx/end.h */ -#endif // SIMDUTF_IMPLEMENTATION_LSX +#if SIMDUTF_CAN_ALWAYS_RUN_LASX +// nothing needed. +#else +SIMDUTF_UNTARGET_REGION +#endif +/* end file src/simdutf/lasx/end.h */ -#endif // SIMDUTF_LSX_H -/* end file src/simdutf/lsx.h */ -/* begin file src/simdutf/lasx.h */ -#ifndef SIMDUTF_LASX_H -#define SIMDUTF_LASX_H +#endif // SIMDUTF_IMPLEMENTATION_LASX + +#endif // SIMDUTF_LASX_H +/* end file src/simdutf/lasx.h */ +/* begin file src/simdutf/lsx.h */ +#ifndef SIMDUTF_LSX_H +#define SIMDUTF_LSX_H #ifdef SIMDUTF_FALLBACK_H - #error "lasx.h must be included before fallback.h" + #error "lsx.h must be included before fallback.h" +#endif + +#ifndef SIMDUTF_CAN_ALWAYS_RUN_LASX + #error "lsx.h must be included after lasx.h" #endif -#ifndef SIMDUTF_IMPLEMENTATION_LASX - #define SIMDUTF_IMPLEMENTATION_LASX (SIMDUTF_IS_LASX) +#ifndef SIMDUTF_IMPLEMENTATION_LSX + #if SIMDUTF_CAN_ALWAYS_RUN_LASX + #define SIMDUTF_IMPLEMENTATION_LSX 0 + #else + #define SIMDUTF_IMPLEMENTATION_LSX (SIMDUTF_IS_LSX) + #endif #endif -#if SIMDUTF_IMPLEMENTATION_LASX && SIMDUTF_IS_LASX - #define SIMDUTF_CAN_ALWAYS_RUN_LASX 1 +#if SIMDUTF_IMPLEMENTATION_LSX && SIMDUTF_IS_LSX + #define SIMDUTF_CAN_ALWAYS_RUN_LSX 1 #else - #define SIMDUTF_CAN_ALWAYS_RUN_LASX 0 + #define SIMDUTF_CAN_ALWAYS_RUN_LSX 0 #endif #define SIMDUTF_CAN_ALWAYS_RUN_FALLBACK (SIMDUTF_IMPLEMENTATION_FALLBACK) -#if SIMDUTF_IMPLEMENTATION_LASX +#if SIMDUTF_IMPLEMENTATION_LSX namespace simdutf { /** - * Implementation for LoongArch ASX. + * Implementation for LoongArch SX. */ -namespace lasx {} // namespace lasx +namespace lsx {} // namespace lsx } // namespace simdutf -/* begin file src/simdutf/lasx/implementation.h */ -#ifndef SIMDUTF_LASX_IMPLEMENTATION_H -#define SIMDUTF_LASX_IMPLEMENTATION_H +/* begin file src/simdutf/lsx/implementation.h */ +#ifndef SIMDUTF_LSX_IMPLEMENTATION_H +#define SIMDUTF_LSX_IMPLEMENTATION_H namespace simdutf { -namespace lasx { +namespace lsx { namespace { using namespace simdutf; @@ -9303,148 +8776,64 @@ using namespace simdutf; class implementation final : public simdutf::implementation { public: simdutf_really_inline implementation() - : simdutf::implementation("lasx", "LOONGARCH ASX", - internal::instruction_set::LSX | - internal::instruction_set::LASX) {} + : simdutf::implementation("lsx", "LOONGARCH SX", + internal::instruction_set::LSX) {} simdutf_warn_unused result base64_to_binary( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; simdutf_warn_unused full_result base64_to_binary_details( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; - simdutf_warn_unused result - base64_to_binary(const char16_t *input, size_t length, char *output, - base64_options options, - last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; + simdutf_warn_unused result base64_to_binary( + const char16_t *input, size_t length, char *output, + base64_options options, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept override; simdutf_warn_unused full_result base64_to_binary_details( const char16_t *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; size_t binary_to_base64(const char *input, size_t length, char *output, - base64_options options) const noexcept; + base64_options options) const noexcept override; + size_t + binary_to_base64_with_lines(const char *input, size_t length, char *output, + size_t line_length, + base64_options options) const noexcept override; const char *find(const char *start, const char *end, - char character) const noexcept; + char character) const noexcept override; const char16_t *find(const char16_t *start, const char16_t *end, - char16_t character) const noexcept; + char16_t character) const noexcept override; + simdutf_warn_unused size_t binary_length_from_base64( + const char *input, size_t length) const noexcept override; + simdutf_warn_unused size_t binary_length_from_base64( + const char16_t *input, size_t length) const noexcept override; }; -} // namespace lasx +} // namespace lsx } // namespace simdutf -#endif // SIMDUTF_LASX_IMPLEMENTATION_H -/* end file src/simdutf/lasx/implementation.h */ - -/* begin file src/simdutf/lasx/begin.h */ -// redefining SIMDUTF_IMPLEMENTATION to "lasx" -// #define SIMDUTF_IMPLEMENTATION lasx -#define SIMDUTF_SIMD_HAS_UNSIGNED_CMP 1 -/* end file src/simdutf/lasx/begin.h */ - - // Declarations -/* begin file src/simdutf/lasx/intrinsics.h */ -#ifndef SIMDUTF_LASX_INTRINSICS_H -#define SIMDUTF_LASX_INTRINSICS_H - - -// This should be the correct header whether -// you use visual studio or other compilers. -#include -#include +#endif // SIMDUTF_LSX_IMPLEMENTATION_H +/* end file src/simdutf/lsx/implementation.h */ + +/* begin file src/simdutf/lsx/begin.h */ +// redefining SIMDUTF_IMPLEMENTATION to "lsx" +// #define SIMDUTF_IMPLEMENTATION lsx +#define SIMDUTF_SIMD_HAS_UNSIGNED_CMP 1 +/* end file src/simdutf/lsx/begin.h */ + + // Declarations +/* begin file src/simdutf/lsx/intrinsics.h */ +#ifndef SIMDUTF_LSX_INTRINSICS_H +#define SIMDUTF_LSX_INTRINSICS_H -#if defined(__loongarch_asx) - #ifdef __clang__ - #define VREGS_PREFIX "$vr" - #define XREGS_PREFIX "$xr" - #else // GCC - #define VREGS_PREFIX "$f" - #define XREGS_PREFIX "$f" - #endif - #define __ALL_REGS \ - "0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26," \ - "27,28,29,30,31" -// Convert __m128i to __m256i -static inline __m256i ____m256i(__m128i in) { - __m256i out = __lasx_xvldi(0); - __asm__ volatile(".irp i," __ALL_REGS "\n\t" - " .ifc %[out], " XREGS_PREFIX "\\i \n\t" - " .irp j," __ALL_REGS "\n\t" - " .ifc %[in], " VREGS_PREFIX "\\j \n\t" - " xvpermi.q $xr\\i, $xr\\j, 0x0 \n\t" - " .endif \n\t" - " .endr \n\t" - " .endif \n\t" - ".endr \n\t" - : [out] "+f"(out) - : [in] "f"(in)); - return out; -} -// Convert two __m128i to __m256i -static inline __m256i lasx_set_q(__m128i inhi, __m128i inlo) { - __m256i out; - __asm__ volatile(".irp i," __ALL_REGS "\n\t" - " .ifc %[hi], " VREGS_PREFIX "\\i \n\t" - " .irp j," __ALL_REGS "\n\t" - " .ifc %[lo], " VREGS_PREFIX "\\j \n\t" - " xvpermi.q $xr\\i, $xr\\j, 0x20 \n\t" - " .endif \n\t" - " .endr \n\t" - " .endif \n\t" - ".endr \n\t" - ".ifnc %[out], %[hi] \n\t" - ".irp i," __ALL_REGS "\n\t" - " .ifc %[out], " XREGS_PREFIX "\\i \n\t" - " .irp j," __ALL_REGS "\n\t" - " .ifc %[hi], " VREGS_PREFIX "\\j \n\t" - " xvori.b $xr\\i, $xr\\j, 0 \n\t" - " .endif \n\t" - " .endr \n\t" - " .endif \n\t" - ".endr \n\t" - ".endif \n\t" - : [out] "=f"(out), [hi] "+f"(inhi) - : [lo] "f"(inlo)); - return out; -} -// Convert __m256i low part to __m128i -static inline __m128i lasx_extracti128_lo(__m256i in) { - __m128i out; - __asm__ volatile(".ifnc %[out], %[in] \n\t" - ".irp i," __ALL_REGS "\n\t" - " .ifc %[out], " VREGS_PREFIX "\\i \n\t" - " .irp j," __ALL_REGS "\n\t" - " .ifc %[in], " XREGS_PREFIX "\\j \n\t" - " vori.b $vr\\i, $vr\\j, 0 \n\t" - " .endif \n\t" - " .endr \n\t" - " .endif \n\t" - ".endr \n\t" - ".endif \n\t" - : [out] "=f"(out) - : [in] "f"(in)); - return out; -} -// Convert __m256i high part to __m128i -static inline __m128i lasx_extracti128_hi(__m256i in) { - __m128i out; - __asm__ volatile(".irp i," __ALL_REGS "\n\t" - " .ifc %[out], " VREGS_PREFIX "\\i \n\t" - " .irp j," __ALL_REGS "\n\t" - " .ifc %[in], " XREGS_PREFIX "\\j \n\t" - " xvpermi.q $xr\\i, $xr\\j, 0x11 \n\t" - " .endif \n\t" - " .endr \n\t" - " .endif \n\t" - ".endr \n\t" - : [out] "=f"(out) - : [in] "f"(in)); - return out; -} -#endif + +// This should be the correct header whether +// you use visual studio or other compilers. +#include /* Encoding of argument for LoongArch64 xvldi instruction. See: @@ -9474,7 +8863,7 @@ all lanes the result as 64-bit elements to all lanes */ -namespace lasx_vldi { +namespace vldi { template class const_u16 { constexpr static const uint8_t b0 = ((v >> 0 * 8) & 0xff); @@ -9591,412 +8980,235 @@ template class const_u64 { constexpr static int value = int((operation << 8) | byte) - 8192; constexpr static bool valid = operation != 0xffff; }; +} // namespace vldi -} // namespace lasx_vldi - -// Uncomment when running under QEMU affected -// by bug https://gitlab.com/qemu-project/qemu/-/issues/2865 -// Versions <= 9.2.2 are affected, likely anything newer is correct. -#ifndef QEMU_VLDI_BUG -// #define QEMU_VLDI_BUG 1 -#endif - -#ifdef QEMU_VLDI_BUG - #define lasx_splat_u16(v) __lasx_xvreplgr2vr_h(v) - #define lasx_splat_u32(v) __lasx_xvreplgr2vr_w(v) -#else -template constexpr __m256i lasx_splat_u16_aux() { - constexpr bool is_imm10 = (int16_t(x) < 512) && (int16_t(x) > -512); - constexpr uint16_t imm10 = is_imm10 ? x : 0; - constexpr bool is_vldi = lasx_vldi::const_u16::valid; - constexpr int vldi_imm = is_vldi ? lasx_vldi::const_u16::value : 0; - - return is_imm10 ? __lasx_xvrepli_h(int16_t(imm10)) - : is_vldi ? __lasx_xvldi(vldi_imm) - : __lasx_xvreplgr2vr_h(x); -} - -template constexpr __m256i lasx_splat_u32_aux() { - constexpr bool is_imm10 = (int32_t(x) < 512) && (int32_t(x) > -512); - constexpr uint32_t imm10 = is_imm10 ? x : 0; - constexpr bool is_vldi = lasx_vldi::const_u32::valid; - constexpr int vldi_imm = is_vldi ? lasx_vldi::const_u32::value : 0; - - return is_imm10 ? __lasx_xvrepli_w(int32_t(imm10)) - : is_vldi ? __lasx_xvldi(vldi_imm) - : __lasx_xvreplgr2vr_w(x); -} - - #define lasx_splat_u16(v) lasx_splat_u16_aux<(v)>() - #define lasx_splat_u32(v) lasx_splat_u32_aux<(v)>() -#endif // QEMU_VLDI_BUG - -#endif // SIMDUTF_LASX_INTRINSICS_H -/* end file src/simdutf/lasx/intrinsics.h */ -/* begin file src/simdutf/lasx/bitmanipulation.h */ -#ifndef SIMDUTF_LASX_BITMANIPULATION_H -#define SIMDUTF_LASX_BITMANIPULATION_H - -#include - -namespace simdutf { -namespace lasx { -namespace { - -simdutf_really_inline int count_ones(uint64_t input_num) { - return __lsx_vpickve2gr_w(__lsx_vpcnt_d(__lsx_vreplgr2vr_d(input_num)), 0); -} - -#if SIMDUTF_NEED_TRAILING_ZEROES -simdutf_really_inline int trailing_zeroes(uint64_t input_num) { - return __builtin_ctzll(input_num); -} -#endif - -} // unnamed namespace -} // namespace lasx -} // namespace simdutf - -#endif // SIMDUTF_LASX_BITMANIPULATION_H -/* end file src/simdutf/lasx/bitmanipulation.h */ -/* begin file src/simdutf/lasx/simd.h */ -#ifndef SIMDUTF_LASX_SIMD_H -#define SIMDUTF_LASX_SIMD_H - - -namespace simdutf { -namespace lasx { -namespace { -namespace simd { - -__attribute__((aligned(32))) static const uint8_t prev_shuf_table[32][32] = { - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, - {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, - {0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, - 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}, - {0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, - 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}, - {0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, - 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, - {0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, - 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, - {0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, - 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, - {0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, - 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8}, - {0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, - 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7}, - {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, - 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6}, - {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, - 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5}, - {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, - 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4}, - {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, - 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3}, - {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, - 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2}, - {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, - 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1}, - {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0}, - {15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, - 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - {14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, - 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - {13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, - 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - {12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, - 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - {11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, - 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - {10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, - 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - {9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, - 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - {8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, - 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0}, - {7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, - 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0}, - {6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, - 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0}, - {5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, - 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0}, - {4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, - 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0}, - {3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, - 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0}, - {2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, - 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0}, - {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, -}; - -__attribute__((aligned(32))) static const uint8_t bitsel_mask_table[32][32] = { - {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0}}; +// Uncomment when running under QEMU affected +// by bug https://gitlab.com/qemu-project/qemu/-/issues/2865 +// Versions <= 9.2.2 are affected, likely anything newer is correct. +#ifndef QEMU_VLDI_BUG +// #define QEMU_VLDI_BUG 1 +#endif -// Forward-declared so they can be used by splat and friends. -template struct base { - __m256i value; +#ifndef lsx_splat_u16 + #ifdef QEMU_VLDI_BUG + #define lsx_splat_u16(v) __lsx_vreplgr2vr_h(v) + #define lsx_splat_u32(v) __lsx_vreplgr2vr_w(v) + #else +namespace { +template constexpr __m128i lsx_splat_u16_aux() { + return ((int16_t(x) < 512) && (int16_t(x) > -512)) + ? __lsx_vrepli_h( + ((int16_t(x) < 512) && (int16_t(x) > -512)) ? int16_t(x) : 0) + : (vldi::const_u16::valid + ? __lsx_vldi(vldi::const_u16::valid + ? vldi::const_u16::value + : 0) + : __lsx_vreplgr2vr_h(x)); +} - // Zero constructor - simdutf_really_inline base() : value{__m256i()} {} +template constexpr __m128i lsx_splat_u32_aux() { + return ((int32_t(x) < 512) && (int32_t(x) > -512)) + ? __lsx_vrepli_w( + ((int32_t(x) < 512) && (int32_t(x) > -512)) ? int32_t(x) : 0) + : (vldi::const_u32::valid + ? __lsx_vldi(vldi::const_u32::valid + ? vldi::const_u32::value + : 0) + : __lsx_vreplgr2vr_w(x)); +} +} // namespace + #define lsx_splat_u16(v) lsx_splat_u16_aux<(v)>() + #define lsx_splat_u32(v) lsx_splat_u32_aux<(v)>() + #endif // QEMU_VLDI_BUG +#endif // lsx_splat_u16 +#endif // SIMDUTF_LSX_INTRINSICS_H +/* end file src/simdutf/lsx/intrinsics.h */ +/* begin file src/simdutf/lsx/bitmanipulation.h */ +#ifndef SIMDUTF_LSX_BITMANIPULATION_H +#define SIMDUTF_LSX_BITMANIPULATION_H - // Conversion from SIMD register - simdutf_really_inline base(const __m256i _value) : value(_value) {} - // Conversion to SIMD register - simdutf_really_inline operator const __m256i &() const { return this->value; } - simdutf_really_inline operator __m256i &() { return this->value; } - template - simdutf_really_inline void store_ascii_as_utf16(char16_t *ptr) const { - if (big_endian) { - __m256i zero = __lasx_xvldi(0); - __m256i in8 = __lasx_xvpermi_d(this->value, 0b11011000); - __m256i inlow = __lasx_xvilvl_b(in8, zero); - __m256i inhigh = __lasx_xvilvh_b(in8, zero); - __lasx_xvst(inlow, reinterpret_cast(ptr), 0); - __lasx_xvst(inhigh, reinterpret_cast(ptr), 32); - } else { - __m256i inlow = __lasx_vext2xv_hu_bu(this->value); - __m256i inhigh = __lasx_vext2xv_hu_bu( - __lasx_xvpermi_q(this->value, this->value, 0b00000001)); - __lasx_xvst(inlow, reinterpret_cast<__m256i *>(ptr), 0); - __lasx_xvst(inhigh, reinterpret_cast<__m256i *>(ptr), 32); - } - } - simdutf_really_inline void store_ascii_as_utf32(char32_t *ptr) const { - __m256i in32_0 = __lasx_vext2xv_wu_bu(this->value); - __lasx_xvst(in32_0, reinterpret_cast(ptr), 0); +#include - __m256i in8_1 = __lasx_xvpermi_d(this->value, 0b00000001); - __m256i in32_1 = __lasx_vext2xv_wu_bu(in8_1); - __lasx_xvst(in32_1, reinterpret_cast(ptr), 32); +namespace simdutf { +namespace lsx { +namespace { - __m256i in8_2 = __lasx_xvpermi_d(this->value, 0b00000010); - __m256i in32_2 = __lasx_vext2xv_wu_bu(in8_2); - __lasx_xvst(in32_2, reinterpret_cast(ptr), 64); +simdutf_really_inline int count_ones(uint64_t input_num) { + return __lsx_vpickve2gr_w(__lsx_vpcnt_d(__lsx_vreplgr2vr_d(input_num)), 0); +} + +#if SIMDUTF_NEED_TRAILING_ZEROES +simdutf_really_inline int trailing_zeroes(uint64_t input_num) { + return __builtin_ctzll(input_num); +} +#endif + +} // unnamed namespace +} // namespace lsx +} // namespace simdutf + +#endif // SIMDUTF_LSX_BITMANIPULATION_H +/* end file src/simdutf/lsx/bitmanipulation.h */ +/* begin file src/simdutf/lsx/simd.h */ +#ifndef SIMDUTF_LSX_SIMD_H +#define SIMDUTF_LSX_SIMD_H + + +namespace simdutf { +namespace lsx { +namespace { +namespace simd { + +template struct simd8; + +// +// Base class of simd8 and simd8, both of which use __m128i +// internally. +// +template > struct base_u8 { + __m128i value; + static const int SIZE = sizeof(value); + + // Conversion from/to SIMD register + simdutf_really_inline base_u8(const __m128i _value) : value(_value) {} + simdutf_really_inline operator const __m128i &() const { return this->value; } + simdutf_really_inline operator __m128i &() { return this->value; } - __m256i in8_3 = __lasx_xvpermi_d(this->value, 0b00000011); - __m256i in32_3 = __lasx_vext2xv_wu_bu(in8_3); - __lasx_xvst(in32_3, reinterpret_cast(ptr), 96); - } // Bit operations - simdutf_really_inline Child operator|(const Child other) const { - return __lasx_xvor_v(this->value, other); + simdutf_really_inline simd8 operator|(const simd8 other) const { + return __lsx_vor_v(this->value, other); } - simdutf_really_inline Child operator&(const Child other) const { - return __lasx_xvand_v(this->value, other); + simdutf_really_inline simd8 operator&(const simd8 other) const { + return __lsx_vand_v(this->value, other); } - simdutf_really_inline Child operator^(const Child other) const { - return __lasx_xvxor_v(this->value, other); + simdutf_really_inline simd8 operator^(const simd8 other) const { + return __lsx_vxor_v(this->value, other); } - simdutf_really_inline Child &operator|=(const Child other) { - auto this_cast = static_cast(this); + simdutf_really_inline simd8 operator~() const { return *this ^ 0xFFu; } + simdutf_really_inline simd8 &operator|=(const simd8 other) { + auto this_cast = static_cast *>(this); *this_cast = *this_cast | other; return *this_cast; } -}; - -template struct simd8; -template > -struct base8 : base> { - simdutf_really_inline base8() : base>() {} - simdutf_really_inline base8(const __m256i _value) : base>(_value) {} friend simdutf_really_inline Mask operator==(const simd8 lhs, const simd8 rhs) { - return __lasx_xvseq_b(lhs, rhs); + return __lsx_vseq_b(lhs, rhs); } - static const int SIZE = sizeof(base::value); - - template + template simdutf_really_inline simd8 prev(const simd8 prev_chunk) const { - static_assert(N <= 16, "unsupported shift value"); - - if (!N) - return this->value; - - __m256i zero = __lasx_xvldi(0); - __m256i result, shuf; - if (N < 16) { - shuf = __lasx_xvld(prev_shuf_table[N], 0); - - result = __lasx_xvshuf_b( - __lasx_xvpermi_q(this->value, this->value, 0b00000001), this->value, - shuf); - __m256i srl_prev = __lasx_xvbsrl_v( - __lasx_xvpermi_q(zero, prev_chunk.value, 0b00110001), (16 - N)); - __m256i mask = __lasx_xvld(bitsel_mask_table[N], 0); - result = __lasx_xvbitsel_v(result, srl_prev, mask); - - return result; - } else if (N == 16) { - return __lasx_xvpermi_q(this->value, prev_chunk.value, 0b00100001); - } + return __lsx_vor_v(__lsx_vbsll_v(this->value, N), + __lsx_vbsrl_v(prev_chunk.value, 16 - N)); } }; // SIMD byte mask type (returned by things like eq and gt) -template <> struct simd8 : base8 { +template <> struct simd8 : base_u8 { + typedef uint16_t bitmask_t; + typedef uint32_t bitmask2_t; + static simdutf_really_inline simd8 splat(bool _value) { - return __lasx_xvreplgr2vr_b(uint8_t(-(!!_value))); + return __lsx_vreplgr2vr_b(uint8_t(-(!!_value))); } - simdutf_really_inline simd8() : base8() {} - simdutf_really_inline simd8(const __m256i _value) : base8(_value) {} + simdutf_really_inline simd8(const __m128i _value) : base_u8(_value) {} + // False constructor + simdutf_really_inline simd8() : simd8(__lsx_vldi(0)) {} // Splat constructor - simdutf_really_inline simd8(bool _value) : base8(splat(_value)) {} + simdutf_really_inline simd8(bool _value) : simd8(splat(_value)) {} + simdutf_really_inline void store(uint8_t dst[16]) const { + return __lsx_vst(this->value, dst, 0); + } simdutf_really_inline uint32_t to_bitmask() const { - __m256i mask = __lasx_xvmsknz_b(this->value); - uint32_t mask0 = __lasx_xvpickve2gr_wu(mask, 0); - uint32_t mask1 = __lasx_xvpickve2gr_wu(mask, 4); - return (mask0 | (mask1 << 16)); - } - simdutf_really_inline bool any() const { - if (__lasx_xbz_b(this->value)) - return false; - return true; + return __lsx_vpickve2gr_wu(__lsx_vmsknz_b(*this), 0); } - simdutf_really_inline simd8 operator~() const { return *this ^ true; } }; -template struct base8_numeric : base8 { - static simdutf_really_inline simd8 splat(T _value) { - return __lasx_xvreplgr2vr_b(_value); +// Unsigned bytes +template <> struct simd8 : base_u8 { + static simdutf_really_inline simd8 splat(uint8_t _value) { + return __lsx_vreplgr2vr_b(_value); } - static simdutf_really_inline simd8 zero() { return __lasx_xvldi(0); } - static simdutf_really_inline simd8 load(const T values[32]) { - return __lasx_xvld(reinterpret_cast(values), 0); + static simdutf_really_inline simd8 zero() { return __lsx_vldi(0); } + static simdutf_really_inline simd8 load(const uint8_t *values) { + return __lsx_vld(values, 0); } + simdutf_really_inline simd8(const __m128i _value) + : base_u8(_value) {} + // Zero constructor + simdutf_really_inline simd8() : simd8(zero()) {} + // Array constructor + simdutf_really_inline simd8(const uint8_t values[16]) : simd8(load(values)) {} + // Splat constructor + simdutf_really_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + // Member-by-member initialization + simdutf_really_inline + simd8(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, + uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, + uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15) + : simd8((__m128i)v16u8{v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, + v12, v13, v14, v15}) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdutf_really_inline simd8 repeat_16(T v0, T v1, T v2, T v3, T v4, - T v5, T v6, T v7, T v8, T v9, - T v10, T v11, T v12, T v13, - T v14, T v15) { - return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, - v14, v15, v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, - v12, v13, v14, v15); + simdutf_really_inline static simd8 + repeat_16(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, + uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, + uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, + uint8_t v15) { + return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, + v13, v14, v15); } - simdutf_really_inline base8_numeric() : base8() {} - simdutf_really_inline base8_numeric(const __m256i _value) - : base8(_value) {} - // Store to array - simdutf_really_inline void store(T dst[32]) const { - return __lasx_xvst(this->value, reinterpret_cast<__m256i *>(dst), 0); + simdutf_really_inline void store(uint8_t dst[16]) const { + return __lsx_vst(this->value, dst, 0); + } + + // Order-specific operations + simdutf_really_inline simd8 + operator>=(const simd8 other) const { + return __lsx_vsle_bu(other, *this); + } + simdutf_really_inline simd8 + operator>(const simd8 other) const { + return __lsx_vslt_bu(other, *this); + } + simdutf_really_inline simd8 &operator-=(const simd8 other) { + value = __lsx_vsub_b(value, other.value); + return *this; + } + // Same as >, but instead of guaranteeing all 1's == true, false = 0 and true + // = nonzero. For ARM, returns all 1's. + simdutf_really_inline simd8 + gt_bits(const simd8 other) const { + return simd8(*this > other); + } + + // Bit-specific operations + simdutf_really_inline simd8 any_bits_set(simd8 bits) const { + return __lsx_vslt_bu(__lsx_vldi(0), __lsx_vand_v(this->value, bits)); + } + simdutf_really_inline bool is_ascii() const { + return __lsx_vpickve2gr_hu(__lsx_vmskgez_b(this->value), 0) == 0xFFFF; } - // Override to distinguish from bool version - simdutf_really_inline simd8 operator~() const { return *this ^ 0xFFu; } + simdutf_really_inline bool any_bits_set_anywhere() const { + return __lsx_vpickve2gr_hu(__lsx_vmsknz_b(this->value), 0) > 0; + } + template simdutf_really_inline simd8 shr() const { + return __lsx_vsrli_b(this->value, N); + } + template simdutf_really_inline simd8 shl() const { + return __lsx_vslli_b(this->value, N); + } // Perform a lookup assuming the value is between 0 and 16 (undefined behavior // for out of range values) template simdutf_really_inline simd8 lookup_16(simd8 lookup_table) const { - __m256i origin = __lasx_xvand_v(this->value, __lasx_xvldi(0x1f)); - return __lasx_xvshuf_b(__lasx_xvldi(0), lookup_table, origin); + return lookup_table.apply_lookup_16_to(*this); } template @@ -10010,115 +9222,150 @@ template struct base8_numeric : base8 { replace7, replace8, replace9, replace10, replace11, replace12, replace13, replace14, replace15)); } + + template + simdutf_really_inline simd8 + apply_lookup_16_to(const simd8 original) const { + __m128i original_tmp = __lsx_vand_v(original, __lsx_vldi(0x1f)); + return __lsx_vshuf_b(__lsx_vldi(0), *this, simd8(original_tmp)); + } + + simdutf_really_inline uint64_t sum_bytes() const { + const auto sum_u16 = __lsx_vhaddw_hu_bu(value, value); + const auto sum_u32 = __lsx_vhaddw_wu_hu(sum_u16, sum_u16); + const auto sum_u64 = __lsx_vhaddw_du_wu(sum_u32, sum_u32); + + return uint64_t(__lsx_vpickve2gr_du(sum_u64, 0)) + + uint64_t(__lsx_vpickve2gr_du(sum_u64, 1)); + } }; // Signed bytes -template <> struct simd8 : base8_numeric { - simdutf_really_inline simd8() : base8_numeric() {} - simdutf_really_inline simd8(const __m256i _value) - : base8_numeric(_value) {} +template <> struct simd8 { + __m128i value; - // Splat constructor - simdutf_really_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdutf_really_inline simd8(const int8_t values[32]) : simd8(load(values)) {} - simdutf_really_inline operator simd8() const; - simdutf_really_inline bool is_ascii() const { - __m256i ascii_mask = __lasx_xvslti_b(this->value, 0); - if (__lasx_xbnz_v(ascii_mask)) - return false; - return true; + static const int SIZE = sizeof(value); + + static simdutf_really_inline simd8 splat(int8_t _value) { + return __lsx_vreplgr2vr_b(_value); } - // Order-sensitive comparisons - simdutf_really_inline simd8 operator>(const simd8 other) const { - return __lasx_xvslt_b(other, this->value); + static simdutf_really_inline simd8 zero() { return __lsx_vldi(0); } + static simdutf_really_inline simd8 load(const int8_t values[16]) { + return __lsx_vld(values, 0); } - simdutf_really_inline simd8 operator<(const simd8 other) const { - return __lasx_xvslt_b(this->value, other); + + template + simdutf_really_inline void store_ascii_as_utf16(char16_t *p) const { + __m128i zero = __lsx_vldi(0); + if constexpr (match_system(big_endian)) { + __lsx_vst(__lsx_vilvl_b(zero, (__m128i)this->value), + reinterpret_cast(p), 0); + __lsx_vst(__lsx_vilvh_b(zero, (__m128i)this->value), + reinterpret_cast(p + 8), 0); + } else { + __lsx_vst(__lsx_vilvl_b((__m128i)this->value, zero), + reinterpret_cast(p), 0); + __lsx_vst(__lsx_vilvh_b((__m128i)this->value, zero), + reinterpret_cast(p + 8), 0); + } } -}; -// Unsigned bytes -template <> struct simd8 : base8_numeric { - simdutf_really_inline simd8() : base8_numeric() {} - simdutf_really_inline simd8(const __m256i _value) - : base8_numeric(_value) {} + simdutf_really_inline void store_ascii_as_utf32(char32_t *p) const { + __m128i zero = __lsx_vldi(0); + __m128i in16low = __lsx_vilvl_b(zero, (__m128i)this->value); + __m128i in16high = __lsx_vilvh_b(zero, (__m128i)this->value); + __m128i in32_0 = __lsx_vilvl_h(zero, in16low); + __m128i in32_1 = __lsx_vilvh_h(zero, in16low); + __m128i in32_2 = __lsx_vilvl_h(zero, in16high); + __m128i in32_3 = __lsx_vilvh_h(zero, in16high); + __lsx_vst(in32_0, reinterpret_cast(p), 0); + __lsx_vst(in32_1, reinterpret_cast(p + 4), 0); + __lsx_vst(in32_2, reinterpret_cast(p + 8), 0); + __lsx_vst(in32_3, reinterpret_cast(p + 12), 0); + } + + // In places where the table can be reused, which is most uses in simdutf, it + // is worth it to do 4 table lookups, as there is no direct zero extension + // from u8 to u32. + simdutf_really_inline void store_ascii_as_utf32_tbl(char32_t *p) const { + const simd8 tb1{0, 255, 255, 255, 1, 255, 255, 255, + 2, 255, 255, 255, 3, 255, 255, 255}; + const simd8 tb2{4, 255, 255, 255, 5, 255, 255, 255, + 6, 255, 255, 255, 7, 255, 255, 255}; + const simd8 tb3{8, 255, 255, 255, 9, 255, 255, 255, + 10, 255, 255, 255, 11, 255, 255, 255}; + const simd8 tb4{12, 255, 255, 255, 13, 255, 255, 255, + 14, 255, 255, 255, 15, 255, 255, 255}; + + // encourage store pairing and interleaving + const auto shuf1 = this->apply_lookup_16_to(tb1); + const auto shuf2 = this->apply_lookup_16_to(tb2); + shuf1.store(reinterpret_cast(p)); + shuf2.store(reinterpret_cast(p + 4)); + + const auto shuf3 = this->apply_lookup_16_to(tb3); + const auto shuf4 = this->apply_lookup_16_to(tb4); + shuf3.store(reinterpret_cast(p + 8)); + shuf4.store(reinterpret_cast(p + 12)); + } + // Conversion from/to SIMD register + simdutf_really_inline simd8(const __m128i _value) : value(_value) {} + + // Zero constructor + simdutf_really_inline simd8() : simd8(zero()) {} // Splat constructor - simdutf_really_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + simdutf_really_inline simd8(int8_t _value) : simd8(splat(_value)) {} // Array constructor - simdutf_really_inline simd8(const uint8_t values[32]) : simd8(load(values)) {} - // Member-by-member initialization - simdutf_really_inline - simd8(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, - uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, - uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15, - uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20, - uint8_t v21, uint8_t v22, uint8_t v23, uint8_t v24, uint8_t v25, - uint8_t v26, uint8_t v27, uint8_t v28, uint8_t v29, uint8_t v30, - uint8_t v31) - : simd8((__m256i)v32u8{v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15, - v16, v17, v18, v19, v20, v21, v22, v23, - v24, v25, v26, v27, v28, v29, v30, v31}) {} + simdutf_really_inline simd8(const int8_t *values) : simd8(load(values)) {} - // Saturated math - simdutf_really_inline simd8 - saturating_sub(const simd8 other) const { - return __lasx_xvssub_bu(this->value, other); + // Store to array + simdutf_really_inline void store(int8_t dst[16]) const { + return __lsx_vst(value, dst, 0); } - // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdutf_really_inline simd8 - gt_bits(const simd8 other) const { - return this->saturating_sub(other); - } - simdutf_really_inline simd8 - operator>=(const simd8 other) const { - return __lasx_xvsle_bu(other, *this); + simdutf_really_inline operator simd8() const { + return ((__m128i)this->value); } - simdutf_really_inline simd8 &operator-=(const simd8 other) { - value = __lasx_xvsub_b(value, other.value); - return *this; + + simdutf_really_inline simd8 + operator|(const simd8 other) const { + return __lsx_vor_v((__m128i)value, (__m128i)other.value); } - // Bit-specific operations simdutf_really_inline bool is_ascii() const { - __m256i ascii_mask = __lasx_xvslti_b(this->value, 0); - if (__lasx_xbnz_v(ascii_mask)) - return false; - return true; - } - simdutf_really_inline bool any_bits_set_anywhere() const { - if (__lasx_xbnz_v(this->value)) - return true; - return false; + return (__lsx_vpickve2gr_hu(__lsx_vmskgez_b((__m128i)this->value), 0) == + 0xffff); } - template simdutf_really_inline simd8 shr() const { - return __lasx_xvsrli_b(this->value, N); + + // Order-sensitive comparisons + simdutf_really_inline simd8 operator>(const simd8 other) const { + return __lsx_vslt_b((__m128i)other.value, (__m128i)value); } - template simdutf_really_inline simd8 shl() const { - return __lasx_xvslli_b(this->value, N); + simdutf_really_inline simd8 operator<(const simd8 other) const { + return __lsx_vslt_b((__m128i)value, (__m128i)other.value); } - simdutf_really_inline uint64_t sum_bytes() const { - const auto sum_u16 = __lasx_xvhaddw_hu_bu(value, value); - const auto sum_u32 = __lasx_xvhaddw_wu_hu(sum_u16, sum_u16); - const auto sum_u64 = __lasx_xvhaddw_du_wu(sum_u32, sum_u32); + template + simdutf_really_inline simd8 + prev(const simd8 prev_chunk) const { + return __lsx_vor_v(__lsx_vbsll_v(this->value, N), + __lsx_vbsrl_v(prev_chunk.value, 16 - N)); + } - return uint64_t(__lasx_xvpickve2gr_du(sum_u64, 0)) + - uint64_t(__lasx_xvpickve2gr_du(sum_u64, 1)) + - uint64_t(__lasx_xvpickve2gr_du(sum_u64, 2)) + - uint64_t(__lasx_xvpickve2gr_du(sum_u64, 3)); + template + simdutf_really_inline simd8 + apply_lookup_16_to(const simd8 original) const { + __m128i original_tmp = __lsx_vand_v(original, __lsx_vldi(0x1f)); + return __lsx_vshuf_b(__lsx_vldi(0), (__m128i)this->value, + simd8(original_tmp)); } }; -simdutf_really_inline simd8::operator simd8() const { - return this->value; -} template struct simd8x64 { static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 2, - "LASX kernel should use two registers per 64-byte block."); + static_assert( + NUM_CHUNKS == 4, + "LoongArch kernel should use four registers per 64-byte block."); simd8 chunks[NUM_CHUNKS]; simd8x64(const simd8x64 &o) = delete; // no copy allowed @@ -10126,36 +9373,36 @@ template struct simd8x64 { operator=(const simd8 other) = delete; // no assignment allowed simd8x64() = delete; // no default constructor allowed - simdutf_really_inline simd8x64(const simd8 chunk0, const simd8 chunk1) - : chunks{chunk0, chunk1} {} + simdutf_really_inline simd8x64(const simd8 chunk0, const simd8 chunk1, + const simd8 chunk2, const simd8 chunk3) + : chunks{chunk0, chunk1, chunk2, chunk3} {} simdutf_really_inline simd8x64(const T *ptr) : chunks{simd8::load(ptr), - simd8::load(ptr + sizeof(simd8) / sizeof(T))} {} + simd8::load(ptr + sizeof(simd8) / sizeof(T)), + simd8::load(ptr + 2 * sizeof(simd8) / sizeof(T)), + simd8::load(ptr + 3 * sizeof(simd8) / sizeof(T))} {} simdutf_really_inline void store(T *ptr) const { this->chunks[0].store(ptr + sizeof(simd8) * 0 / sizeof(T)); this->chunks[1].store(ptr + sizeof(simd8) * 1 / sizeof(T)); - } - - simdutf_really_inline uint64_t to_bitmask() const { - uint64_t r_lo = uint32_t(this->chunks[0].to_bitmask()); - uint64_t r_hi = this->chunks[1].to_bitmask(); - return r_lo | (r_hi << 32); + this->chunks[2].store(ptr + sizeof(simd8) * 2 / sizeof(T)); + this->chunks[3].store(ptr + sizeof(simd8) * 3 / sizeof(T)); } simdutf_really_inline simd8x64 &operator|=(const simd8x64 &other) { this->chunks[0] |= other.chunks[0]; this->chunks[1] |= other.chunks[1]; + this->chunks[2] |= other.chunks[2]; + this->chunks[3] |= other.chunks[3]; return *this; } simdutf_really_inline simd8 reduce_or() const { - return this->chunks[0] | this->chunks[1]; + return (this->chunks[0] | this->chunks[1]) | + (this->chunks[2] | this->chunks[3]); } - simdutf_really_inline bool is_ascii() const { - return this->reduce_or().is_ascii(); - } + simdutf_really_inline bool is_ascii() const { return reduce_or().is_ascii(); } template simdutf_really_inline void store_ascii_as_utf16(char16_t *ptr) const { @@ -10163,117 +9410,158 @@ template struct simd8x64 { sizeof(simd8) * 0); this->chunks[1].template store_ascii_as_utf16(ptr + sizeof(simd8) * 1); + this->chunks[2].template store_ascii_as_utf16(ptr + + sizeof(simd8) * 2); + this->chunks[3].template store_ascii_as_utf16(ptr + + sizeof(simd8) * 3); } simdutf_really_inline void store_ascii_as_utf32(char32_t *ptr) const { - this->chunks[0].store_ascii_as_utf32(ptr + sizeof(simd8) * 0); - this->chunks[1].store_ascii_as_utf32(ptr + sizeof(simd8) * 1); + this->chunks[0].store_ascii_as_utf32_tbl(ptr + sizeof(simd8) * 0); + this->chunks[1].store_ascii_as_utf32_tbl(ptr + sizeof(simd8) * 1); + this->chunks[2].store_ascii_as_utf32_tbl(ptr + sizeof(simd8) * 2); + this->chunks[3].store_ascii_as_utf32_tbl(ptr + sizeof(simd8) * 3); + } + + simdutf_really_inline uint64_t to_bitmask() const { + __m128i mask = __lsx_vbsll_v(__lsx_vmsknz_b(this->chunks[3]), 6); + mask = __lsx_vor_v(mask, __lsx_vbsll_v(__lsx_vmsknz_b(this->chunks[2]), 4)); + mask = __lsx_vor_v(mask, __lsx_vbsll_v(__lsx_vmsknz_b(this->chunks[1]), 2)); + mask = __lsx_vor_v(mask, __lsx_vmsknz_b(this->chunks[0])); + return __lsx_vpickve2gr_du(mask, 0); } simdutf_really_inline uint64_t lt(const T m) const { const simd8 mask = simd8::splat(m); - return simd8x64(this->chunks[0] < mask, this->chunks[1] < mask) + return simd8x64(this->chunks[0] < mask, this->chunks[1] < mask, + this->chunks[2] < mask, this->chunks[3] < mask) .to_bitmask(); } - simdutf_really_inline uint64_t gt(const T m) const { const simd8 mask = simd8::splat(m); - return simd8x64(this->chunks[0] > mask, this->chunks[1] > mask) + return simd8x64(this->chunks[0] > mask, this->chunks[1] > mask, + this->chunks[2] > mask, this->chunks[3] > mask) + .to_bitmask(); + } + simdutf_really_inline uint64_t gteq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64(this->chunks[0] >= mask, this->chunks[1] >= mask, + this->chunks[2] >= mask, this->chunks[3] >= mask) .to_bitmask(); } simdutf_really_inline uint64_t gteq_unsigned(const uint8_t m) const { const simd8 mask = simd8::splat(m); - return simd8x64((simd8(__m256i(this->chunks[0])) >= mask), - (simd8(__m256i(this->chunks[1])) >= mask)) + return simd8x64(simd8(this->chunks[0].value) >= mask, + simd8(this->chunks[1].value) >= mask, + simd8(this->chunks[2].value) >= mask, + simd8(this->chunks[3].value) >= mask) .to_bitmask(); } }; // struct simd8x64 -/* begin file src/simdutf/lasx/simd16-inl.h */ +/* begin file src/simdutf/lsx/simd16-inl.h */ template struct simd16; +template > struct base_u16 { + __m128i value; + static const size_t SIZE = sizeof(value); + static const size_t ELEMENTS = sizeof(value) / sizeof(T); + + // Conversion from/to SIMD register + simdutf_really_inline base_u16() = default; + simdutf_really_inline base_u16(const __m128i _value) : value(_value) {} + // Bit operations + simdutf_really_inline simd16 operator|(const simd16 other) const { + return __lsx_vor_v(this->value, other.value); + } + simdutf_really_inline simd16 operator&(const simd16 other) const { + return __lsx_vand_v(this->value, other.value); + } + simdutf_really_inline simd16 operator~() const { + return __lsx_vxori_b(this->value, 0xFF); + } + + friend simdutf_really_inline Mask operator==(const simd16 lhs, + const simd16 rhs) { + return __lsx_vseq_h(lhs.value, rhs.value); + } + + template + simdutf_really_inline simd16 byte_right_shift() const { + return __lsx_vbsrl_v(this->value, N); + } + + simdutf_really_inline uint16_t first() const { + return uint16_t(__lsx_vpickve2gr_w(value, 0)); + } +}; + template > -struct base16 : base> { - using bitmask_type = uint32_t; +struct base16 : base_u16 { + using bitmask_type = uint16_t; - simdutf_really_inline base16() : base>() {} - simdutf_really_inline base16(const __m256i _value) - : base>(_value) {} + simdutf_really_inline base16() : base_u16() {} + simdutf_really_inline base16(const __m128i _value) : base_u16(_value) {} template simdutf_really_inline base16(const Pointer *ptr) - : base16(__lasx_xvld(reinterpret_cast(ptr), 0)) {} + : base16(__lsx_vld(ptr, 0)) {} - /// the size of vector in bytes - static const int SIZE = sizeof(base>::value); + static const int SIZE = sizeof(base_u16::value); - /// the number of elements of type T a vector can hold - static const int ELEMENTS = SIZE / sizeof(T); + template + simdutf_really_inline simd16 prev(const simd16 prev_chunk) const { + return __lsx_vor_v(__lsx_vbsll_v(*this, N * 2), + __lsx_vbsrl_v(prev_chunk, 16 - N * 2)); + } }; // SIMD byte mask type (returned by things like eq and gt) template <> struct simd16 : base16 { static simdutf_really_inline simd16 splat(bool _value) { - return __lasx_xvreplgr2vr_h(uint16_t(-(!!_value))); + return __lsx_vreplgr2vr_h(uint16_t(-(!!_value))); } simdutf_really_inline simd16() : base16() {} - simdutf_really_inline simd16(const __m256i _value) : base16(_value) {} - // Splat constructor - simdutf_really_inline simd16(bool _value) : base16(splat(_value)) {} + simdutf_really_inline simd16(const __m128i _value) : base16(_value) {} simdutf_really_inline bitmask_type to_bitmask() const { - __m256i mask = __lasx_xvmsknz_b(this->value); - bitmask_type mask0 = __lasx_xvpickve2gr_wu(mask, 0); - bitmask_type mask1 = __lasx_xvpickve2gr_wu(mask, 4); - return (mask0 | (mask1 << 16)); - } - simdutf_really_inline simd16 operator~() const { return *this ^ true; } - - simdutf_really_inline bool is_zero() const { - return __lasx_xbz_v(this->value); - } - - template simdutf_really_inline simd16 byte_right_shift() const { - const auto t0 = __lasx_xvbsrl_v(this->value, N); - const auto t1 = __lasx_xvpermi_q(this->value, __lasx_xvldi(0), 0b00000011); - const auto t2 = __lasx_xvbsll_v(t1, 16 - N); - const auto t3 = __lasx_xvor_v(t0, t2); - return t3; + __m128i mask = __lsx_vmsknz_b(this->value); + bitmask_type mask0 = bitmask_type(__lsx_vpickve2gr_wu(mask, 0)); + return mask0; } - simdutf_really_inline uint16_t first() const { - return uint16_t(__lasx_xvpickve2gr_w(value, 0)); - } + simdutf_really_inline bool is_zero() const { return __lsx_bz_v(this->value); } }; template struct base16_numeric : base16 { static simdutf_really_inline simd16 splat(T _value) { - return __lasx_xvreplgr2vr_h((uint16_t)_value); + return __lsx_vreplgr2vr_h(_value); } - static simdutf_really_inline simd16 zero() { return __lasx_xvldi(0); } + static simdutf_really_inline simd16 zero() { return __lsx_vldi(0); } + template static simdutf_really_inline simd16 load(const Pointer values) { - return __lasx_xvld(values, 0); + return __lsx_vld(values, 0); } - simdutf_really_inline base16_numeric() : base16() {} - simdutf_really_inline base16_numeric(const __m256i _value) + simdutf_really_inline base16_numeric(const __m128i _value) : base16(_value) {} // Store to array simdutf_really_inline void store(T dst[8]) const { - return __lasx_xvst(this->value, reinterpret_cast<__m256i *>(dst), 0); + return __lsx_vst(this->value, dst, 0); } // Override to distinguish from bool version - simdutf_really_inline simd16 operator~() const { return *this ^ 0xFFFFu; } + simdutf_really_inline simd16 operator~() const { + return __lsx_vxori_b(this->value, 0xFF); + } }; -// Unsigned code units +// Unsigned code unitstemplate<> template <> struct simd16 : base16_numeric { - simdutf_really_inline simd16() : base16_numeric() {} - simdutf_really_inline simd16(const __m256i _value) - : base16_numeric(_value) {} + simdutf_really_inline simd16(const __m128i _value) + : base16_numeric((__m128i)_value) {} // Splat constructor simdutf_really_inline simd16(uint16_t _value) : simd16(splat(_value)) {} @@ -10283,71 +9571,67 @@ template <> struct simd16 : base16_numeric { simdutf_really_inline simd16(const char16_t *values) : simd16(load(reinterpret_cast(values))) {} + // Copy constructor + simdutf_really_inline simd16(const simd16 mask) : simd16(mask.value) {} + // Order-specific operations simdutf_really_inline simd16 &operator+=(const simd16 other) { - value = __lasx_xvadd_h(value, other.value); + value = __lsx_vadd_h(value, other.value); return *this; } - // Change the endianness - simdutf_really_inline simd16 swap_bytes() const { - return __lasx_xvshuf4i_b(this->value, 0b10110001); - } - template static simdutf_really_inline simd8 pack_shifted_right(const simd16 &v0, const simd16 &v1) { - return __lasx_xvpermi_d(__lasx_xvssrlni_bu_h(v1.value, v0.value, N), - 0b11011000); + return __lsx_vssrlni_bu_h(v1.value, v0.value, N); } // Pack with the unsigned saturation of two uint16_t code units into single // uint8_t vector static simdutf_really_inline simd8 pack(const simd16 &v0, const simd16 &v1) { - return pack_shifted_right<0>(v0, v1); } - simdutf_really_inline uint64_t sum() const { - const auto sum_u32 = __lasx_xvhaddw_wu_hu(value, value); - const auto sum_u64 = __lasx_xvhaddw_du_wu(sum_u32, sum_u32); - - return uint64_t(__lasx_xvpickve2gr_du(sum_u64, 0)) + - uint64_t(__lasx_xvpickve2gr_du(sum_u64, 1)) + - uint64_t(__lasx_xvpickve2gr_du(sum_u64, 2)) + - uint64_t(__lasx_xvpickve2gr_du(sum_u64, 3)); + // Change the endianness + simdutf_really_inline simd16 swap_bytes() const { + return __lsx_vshuf4i_b(this->value, 0b10110001); } - template simdutf_really_inline simd16 byte_right_shift() const { - return __lasx_xvbsrl_v(this->value, N); + simdutf_really_inline uint64_t sum() const { + const auto sum_u32 = __lsx_vhaddw_wu_hu(value, value); + const auto sum_u64 = __lsx_vhaddw_du_wu(sum_u32, sum_u32); + + return uint64_t(__lsx_vpickve2gr_du(sum_u64, 0)) + + uint64_t(__lsx_vpickve2gr_du(sum_u64, 1)); } }; simdutf_really_inline simd16 operator<(const simd16 a, const simd16 b) { - return __lasx_xvslt_hu(a.value, b.value); + return __lsx_vslt_hu(a.value, b.value); } simdutf_really_inline simd16 operator>(const simd16 a, const simd16 b) { - return __lasx_xvslt_hu(b.value, a.value); + return __lsx_vslt_hu(b.value, a.value); } simdutf_really_inline simd16 operator<=(const simd16 a, const simd16 b) { - return __lasx_xvsle_hu(a.value, b.value); + return __lsx_vsle_hu(a.value, b.value); } simdutf_really_inline simd16 operator>=(const simd16 a, const simd16 b) { - return __lasx_xvsle_hu(b.value, a.value); + return __lsx_vsle_hu(b.value, a.value); } template struct simd16x32 { static constexpr int NUM_CHUNKS = 64 / sizeof(simd16); - static_assert(NUM_CHUNKS == 2, - "LASX kernel should use two registers per 64-byte block."); + static_assert( + NUM_CHUNKS == 4, + "LOONGARCH kernel should use four registers per 64-byte block."); simd16 chunks[NUM_CHUNKS]; simd16x32(const simd16x32 &o) = delete; // no copy allowed @@ -10355,139 +9639,133 @@ template struct simd16x32 { operator=(const simd16 other) = delete; // no assignment allowed simd16x32() = delete; // no default constructor allowed - simdutf_really_inline simd16x32(const simd16 chunk0, - const simd16 chunk1) - : chunks{chunk0, chunk1} {} + simdutf_really_inline + simd16x32(const simd16 chunk0, const simd16 chunk1, + const simd16 chunk2, const simd16 chunk3) + : chunks{chunk0, chunk1, chunk2, chunk3} {} simdutf_really_inline simd16x32(const T *ptr) : chunks{simd16::load(ptr), - simd16::load(ptr + sizeof(simd16) / sizeof(T))} {} + simd16::load(ptr + sizeof(simd16) / sizeof(T)), + simd16::load(ptr + 2 * sizeof(simd16) / sizeof(T)), + simd16::load(ptr + 3 * sizeof(simd16) / sizeof(T))} {} simdutf_really_inline void store(T *ptr) const { this->chunks[0].store(ptr + sizeof(simd16) * 0 / sizeof(T)); this->chunks[1].store(ptr + sizeof(simd16) * 1 / sizeof(T)); + this->chunks[2].store(ptr + sizeof(simd16) * 2 / sizeof(T)); + this->chunks[3].store(ptr + sizeof(simd16) * 3 / sizeof(T)); } simdutf_really_inline void swap_bytes() { this->chunks[0] = this->chunks[0].swap_bytes(); this->chunks[1] = this->chunks[1].swap_bytes(); + this->chunks[2] = this->chunks[2].swap_bytes(); + this->chunks[3] = this->chunks[3].swap_bytes(); } simdutf_really_inline uint64_t to_bitmask() const { - uint64_t r_lo = uint32_t(this->chunks[0].to_bitmask()); - uint64_t r_hi = this->chunks[1].to_bitmask(); - return r_lo | (r_hi << 32); + uint64_t r0 = uint32_t(this->chunks[0].to_bitmask()); + uint64_t r1 = this->chunks[1].to_bitmask(); + uint64_t r2 = this->chunks[2].to_bitmask(); + uint64_t r3 = this->chunks[3].to_bitmask(); + return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); + } + simdutf_really_inline uint64_t gteq(const T m) const { + const simd16 mask = simd16::splat(m); + return simd16x32(this->chunks[0] >= mask, this->chunks[1] >= mask, + this->chunks[2] >= mask, this->chunks[3] >= mask) + .to_bitmask(); } simdutf_really_inline uint64_t lteq(const T m) const { const simd16 mask = simd16::splat(m); - return simd16x32(this->chunks[0] <= mask, this->chunks[1] <= mask) + return simd16x32(this->chunks[0] <= mask, this->chunks[1] <= mask, + this->chunks[2] <= mask, this->chunks[3] <= mask) .to_bitmask(); } }; // struct simd16x32 -simdutf_really_inline simd16 min(const simd16 a, - const simd16 b) { - return __lasx_xvmin_hu(a.value, b.value); -} - -simdutf_really_inline simd16 operator==(const simd16 a, - uint16_t b) { - const auto bv = __lasx_xvreplgr2vr_h(b); - return __lasx_xvseq_h(a.value, bv); -} - -simdutf_really_inline simd16 as_vector_u16(const simd16 x) { - return x.value; -} - -simdutf_really_inline simd16 operator&(const simd16 a, +simdutf_really_inline simd16 operator^(const simd16 a, uint16_t b) { - const auto bv = __lasx_xvreplgr2vr_h(b); - return __lasx_xvand_v(a.value, bv); + const auto bv = __lsx_vreplgr2vr_h(b); + return __lsx_vxor_v(a.value, bv); } -simdutf_really_inline simd16 operator&(const simd16 a, - const simd16 b) { - return __lasx_xvand_v(a.value, b.value); +simdutf_really_inline simd16 operator^(const simd16 a, + const simd16 b) { + return __lsx_vxor_v(a.value, b.value); } -simdutf_really_inline simd16 operator^(const simd16 a, - uint16_t b) { - const auto bv = __lasx_xvreplgr2vr_h(b); - return __lasx_xvxor_v(a.value, bv); +simdutf_really_inline simd16 min(const simd16 a, + const simd16 b) { + return __lsx_vmin_hu(a.value, b.value); } -simdutf_really_inline simd16 operator^(const simd16 a, - const simd16 b) { - return __lasx_xvxor_v(a.value, b.value); +simdutf_really_inline simd16 as_vector_u16(const simd16 x) { + return x.value; } -/* end file src/simdutf/lasx/simd16-inl.h */ -/* begin file src/simdutf/lasx/simd32-inl.h */ +/* end file src/simdutf/lsx/simd16-inl.h */ +/* begin file src/simdutf/lsx/simd32-inl.h */ template struct simd32; template <> struct simd32 { - __m256i value; + __m128i value; static const int SIZE = sizeof(value); static const int ELEMENTS = SIZE / sizeof(uint32_t); // constructors - simdutf_really_inline simd32(__m256i v) : value(v) {} + simdutf_really_inline simd32(__m128i v) : value(v) {} template - simdutf_really_inline simd32(Ptr *ptr) : value(__lasx_xvld(ptr, 0)) {} + simdutf_really_inline simd32(Ptr *ptr) : value(__lsx_vld(ptr, 0)) {} // in-place operators simdutf_really_inline simd32 &operator-=(const simd32 other) { - value = __lasx_xvsub_w(value, other.value); + value = __lsx_vsub_w(value, other.value); return *this; } // members simdutf_really_inline uint64_t sum() const { - const auto odd = __lasx_xvsrli_d(value, 32); - const auto even = __lasx_xvand_v(value, __lasx_xvreplgr2vr_d(0xffffffff)); - - const auto sum64 = __lasx_xvadd_d(odd, even); - - return uint64_t(__lasx_xvpickve2gr_du(sum64, 0)) + - uint64_t(__lasx_xvpickve2gr_du(sum64, 1)) + - uint64_t(__lasx_xvpickve2gr_du(sum64, 2)) + - uint64_t(__lasx_xvpickve2gr_du(sum64, 3)); + return uint64_t(__lsx_vpickve2gr_wu(value, 0)) + + uint64_t(__lsx_vpickve2gr_wu(value, 1)) + + uint64_t(__lsx_vpickve2gr_wu(value, 2)) + + uint64_t(__lsx_vpickve2gr_wu(value, 3)); } // static members static simdutf_really_inline simd32 splat(uint32_t x) { - return __lasx_xvreplgr2vr_w(x); + return __lsx_vreplgr2vr_w(x); } static simdutf_really_inline simd32 zero() { - return __lasx_xvrepli_w(0); + return __lsx_vrepli_w(0); } }; // ------------------------------------------------------------ template <> struct simd32 { - __m256i value; + __m128i value; static const int SIZE = sizeof(value); // constructors - simdutf_really_inline simd32(__m256i v) : value(v) {} + simdutf_really_inline simd32(__m128i v) : value(v) {} }; // ------------------------------------------------------------ simdutf_really_inline simd32 operator&(const simd32 a, const simd32 b) { - return __lasx_xvor_v(a.value, b.value); + return __lsx_vor_v(a.value, b.value); } simdutf_really_inline simd32 operator<(const simd32 a, const simd32 b) { - return __lasx_xvslt_wu(a.value, b.value); + return __lsx_vslt_wu(a.value, b.value); } simdutf_really_inline simd32 operator>(const simd32 a, const simd32 b) { - return __lasx_xvslt_wu(b.value, a.value); + return __lsx_vslt_wu(b.value, a.value); } // ------------------------------------------------------------ @@ -10495,78 +9773,76 @@ simdutf_really_inline simd32 operator>(const simd32 a, simdutf_really_inline simd32 as_vector_u32(const simd32 v) { return v.value; } -/* end file src/simdutf/lasx/simd32-inl.h */ -/* begin file src/simdutf/lasx/simd64-inl.h */ +/* end file src/simdutf/lsx/simd32-inl.h */ +/* begin file src/simdutf/lsx/simd64-inl.h */ template struct simd64; template <> struct simd64 { - __m256i value; + __m128i value; static const int SIZE = sizeof(value); static const int ELEMENTS = SIZE / sizeof(uint64_t); // constructors - simdutf_really_inline simd64(__m256i v) : value(v) {} + simdutf_really_inline simd64(__m128i v) : value(v) {} template - simdutf_really_inline simd64(Ptr *ptr) : value(__lasx_xvld(ptr, 0)) {} + simdutf_really_inline simd64(Ptr *ptr) : value(__lsx_vld(ptr, 0)) {} // in-place operators simdutf_really_inline simd64 &operator+=(const simd64 other) { - value = __lasx_xvadd_d(value, other.value); + value = __lsx_vadd_d(value, other.value); return *this; } // members simdutf_really_inline uint64_t sum() const { - return uint64_t(__lasx_xvpickve2gr_du(value, 0)) + - uint64_t(__lasx_xvpickve2gr_du(value, 1)) + - uint64_t(__lasx_xvpickve2gr_du(value, 2)) + - uint64_t(__lasx_xvpickve2gr_du(value, 3)); + return uint64_t(__lsx_vpickve2gr_du(value, 0)) + + uint64_t(__lsx_vpickve2gr_du(value, 1)); } // static members static simdutf_really_inline simd64 zero() { - return __lasx_xvrepli_d(0); + return __lsx_vrepli_d(0); } }; // ------------------------------------------------------------ template <> struct simd64 { - __m256i value; + __m128i value; static const int SIZE = sizeof(value); // constructors - simdutf_really_inline simd64(__m256i v) : value(v) {} + simdutf_really_inline simd64(__m128i v) : value(v) {} }; // ------------------------------------------------------------ simd64 sum_8bytes(const simd8 v) { - const auto sum_u16 = __lasx_xvhaddw_hu_bu(v, v); - const auto sum_u32 = __lasx_xvhaddw_wu_hu(sum_u16, sum_u16); - const auto sum_u64 = __lasx_xvhaddw_du_wu(sum_u32, sum_u32); + const auto sum_u16 = __lsx_vhaddw_hu_bu(v, v); + const auto sum_u32 = __lsx_vhaddw_wu_hu(sum_u16, sum_u16); + const auto sum_u64 = __lsx_vhaddw_du_wu(sum_u32, sum_u32); return simd64(sum_u64); } -/* end file src/simdutf/lasx/simd64-inl.h */ +/* end file src/simdutf/lsx/simd64-inl.h */ } // namespace simd } // unnamed namespace -} // namespace lasx +} // namespace lsx } // namespace simdutf -#endif // SIMDUTF_LASX_SIMD_H -/* end file src/simdutf/lasx/simd.h */ +#endif // SIMDUTF_LSX_SIMD_H +/* end file src/simdutf/lsx/simd.h */ -/* begin file src/simdutf/lasx/end.h */ +/* begin file src/simdutf/lsx/end.h */ #undef SIMDUTF_SIMD_HAS_UNSIGNED_CMP -/* end file src/simdutf/lasx/end.h */ +/* end file src/simdutf/lsx/end.h */ -#endif // SIMDUTF_IMPLEMENTATION_LASX +#endif // SIMDUTF_IMPLEMENTATION_LSX -#endif // SIMDUTF_LASX_H -/* end file src/simdutf/lasx.h */ +#endif // SIMDUTF_LSX_H +/* end file src/simdutf/lsx.h */ /* begin file src/simdutf/fallback.h */ #ifndef SIMDUTF_FALLBACK_H #define SIMDUTF_FALLBACK_H @@ -10619,890 +9895,123 @@ class implementation final : public simdutf::implementation { simdutf_warn_unused result base64_to_binary( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; simdutf_warn_unused full_result base64_to_binary_details( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; - simdutf_warn_unused result - base64_to_binary(const char16_t *input, size_t length, char *output, - base64_options options, - last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; - simdutf_warn_unused full_result base64_to_binary_details( - const char16_t *input, size_t length, char *output, - base64_options options, - last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; - size_t binary_to_base64(const char *input, size_t length, char *output, - base64_options options) const noexcept; - const char *find(const char *start, const char *end, - char character) const noexcept; - const char16_t *find(const char16_t *start, const char16_t *end, - char16_t character) const noexcept; - -}; -} // namespace fallback -} // namespace simdutf - -#endif // SIMDUTF_FALLBACK_IMPLEMENTATION_H -/* end file src/simdutf/fallback/implementation.h */ - -/* begin file src/simdutf/fallback/begin.h */ -// redefining SIMDUTF_IMPLEMENTATION to "fallback" -// #define SIMDUTF_IMPLEMENTATION fallback -/* end file src/simdutf/fallback/begin.h */ - - // Declarations -/* begin file src/simdutf/fallback/bitmanipulation.h */ -#ifndef SIMDUTF_FALLBACK_BITMANIPULATION_H -#define SIMDUTF_FALLBACK_BITMANIPULATION_H - -#include - -namespace simdutf { -namespace fallback { -namespace {} // unnamed namespace -} // namespace fallback -} // namespace simdutf - -#endif // SIMDUTF_FALLBACK_BITMANIPULATION_H -/* end file src/simdutf/fallback/bitmanipulation.h */ - -/* begin file src/simdutf/fallback/end.h */ -/* end file src/simdutf/fallback/end.h */ - -#endif // SIMDUTF_IMPLEMENTATION_FALLBACK -#endif // SIMDUTF_FALLBACK_H -/* end file src/simdutf/fallback.h */ -#ifndef SIMDUTF_REGULAR_VISUAL_STUDIO -SIMDUTF_POP_DISABLE_WARNINGS -#endif - -// The scalar routines should be included once. -/* begin file src/scalar/swap_bytes.h */ -#ifndef SIMDUTF_SWAP_BYTES_H -#define SIMDUTF_SWAP_BYTES_H - -namespace simdutf { -namespace scalar { - -inline simdutf_warn_unused uint16_t u16_swap_bytes(const uint16_t word) { - return uint16_t((word >> 8) | (word << 8)); -} - -inline simdutf_warn_unused uint32_t u32_swap_bytes(const uint32_t word) { - return ((word >> 24) & 0xff) | // move byte 3 to byte 0 - ((word << 8) & 0xff0000) | // move byte 1 to byte 2 - ((word >> 8) & 0xff00) | // move byte 2 to byte 1 - ((word << 24) & 0xff000000); // byte 0 to byte 3 -} - -} // namespace scalar -} // namespace simdutf - -#endif -/* end file src/scalar/swap_bytes.h */ -/* begin file src/scalar/base64.h */ -#ifndef SIMDUTF_BASE64_H -#define SIMDUTF_BASE64_H - -#include -#include -#include -#include -#include - -namespace simdutf { -namespace scalar { -namespace { -namespace base64 { - -// This function is not expected to be fast. Do not use in long loops. -// In most instances you should be using is_ignorable. -template bool is_ascii_white_space(char_type c) { - return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f'; -} - -template bool is_eight_byte(char_type c) { - if (sizeof(char_type) == 1) { - return true; - } - return uint8_t(c) == c; -} - -template -bool is_ignorable(char_type c, simdutf::base64_options options) { - const uint8_t *to_base64 = - (options & base64_default_or_url) - ? tables::base64::to_base64_default_or_url_value - : ((options & base64_url) ? tables::base64::to_base64_url_value - : tables::base64::to_base64_value); - const bool ignore_garbage = - (options == base64_options::base64_url_accept_garbage) || - (options == base64_options::base64_default_accept_garbage) || - (options == base64_options::base64_default_or_url_accept_garbage); - uint8_t code = to_base64[uint8_t(c)]; - if (is_eight_byte(c) && code <= 63) { - return false; - } - if (is_eight_byte(c) && code == 64) { - return true; - } - return ignore_garbage; -} -template -bool is_base64(char_type c, simdutf::base64_options options) { - const uint8_t *to_base64 = - (options & base64_default_or_url) - ? tables::base64::to_base64_default_or_url_value - : ((options & base64_url) ? tables::base64::to_base64_url_value - : tables::base64::to_base64_value); - uint8_t code = to_base64[uint8_t(c)]; - if (is_eight_byte(c) && code <= 63) { - return true; - } - return false; -} - -template -bool is_base64_or_padding(char_type c, simdutf::base64_options options) { - const uint8_t *to_base64 = - (options & base64_default_or_url) - ? tables::base64::to_base64_default_or_url_value - : ((options & base64_url) ? tables::base64::to_base64_url_value - : tables::base64::to_base64_value); - if (c == '=') { - return true; - } - uint8_t code = to_base64[uint8_t(c)]; - if (is_eight_byte(c) && code <= 63) { - return true; - } - return false; -} - -template -bool is_ignorable_or_padding(char_type c, simdutf::base64_options options) { - return is_ignorable(c, options) || c == '='; -} - -struct reduced_input { - size_t equalsigns; // number of padding characters '=', typically 0, 1, 2. - size_t equallocation; // location of the first padding character if any - size_t srclen; // length of the input buffer before padding - size_t full_input_length; // length of the input buffer with padding but - // without ignorable characters -}; - -// find the end of the base64 input buffer -// It returns the number of padding characters, the location of the first -// padding character if any, the length of the input buffer before padding -// and the length of the input buffer with padding. The input buffer is not -// modified. The function assumes that there are at most two padding characters. -template -reduced_input find_end(const char_type *src, size_t srclen, - simdutf::base64_options options) { - const uint8_t *to_base64 = - (options & base64_default_or_url) - ? tables::base64::to_base64_default_or_url_value - : ((options & base64_url) ? tables::base64::to_base64_url_value - : tables::base64::to_base64_value); - const bool ignore_garbage = - (options == base64_options::base64_url_accept_garbage) || - (options == base64_options::base64_default_accept_garbage) || - (options == base64_options::base64_default_or_url_accept_garbage); - - size_t equalsigns = 0; - // We intentionally include trailing spaces in the full input length. - // See https://github.com/simdutf/simdutf/issues/824 - size_t full_input_length = srclen; - // skip trailing spaces - while (!ignore_garbage && srclen > 0 && - scalar::base64::is_eight_byte(src[srclen - 1]) && - to_base64[uint8_t(src[srclen - 1])] == 64) { - srclen--; - } - size_t equallocation = - srclen; // location of the first padding character if any - if (ignore_garbage) { - // Technically, we don't need to find the first padding character, we can - // just change our algorithms, but it adds substantial complexity. - auto it = simdutf::find(src, src + srclen, '='); - if (it != src + srclen) { - equallocation = it - src; - equalsigns = 1; - srclen = equallocation; - full_input_length = equallocation + 1; - } - return {equalsigns, equallocation, srclen, full_input_length}; - } - if (!ignore_garbage && srclen > 0 && src[srclen - 1] == '=') { - // This is the last '=' sign. - equallocation = srclen - 1; - srclen--; - equalsigns = 1; - // skip trailing spaces - while (srclen > 0 && scalar::base64::is_eight_byte(src[srclen - 1]) && - to_base64[uint8_t(src[srclen - 1])] == 64) { - srclen--; - } - if (srclen > 0 && src[srclen - 1] == '=') { - // This is the second '=' sign. - equallocation = srclen - 1; - srclen--; - equalsigns = 2; - } - } - return {equalsigns, equallocation, srclen, full_input_length}; -} - -// Returns true upon success. The destination buffer must be large enough. -// This functions assumes that the padding (=) has been removed. -// if check_capacity is true, it will check that the destination buffer is -// large enough. If it is not, it will return OUTPUT_BUFFER_TOO_SMALL. -template -full_result base64_tail_decode_impl( - char *dst, size_t outlen, const char_type *src, size_t length, - size_t padding_characters, // number of padding characters - // '=', typically 0, 1, 2. - base64_options options, last_chunk_handling_options last_chunk_options) { - char *dstend = dst + outlen; - (void)dstend; - // This looks like 10 branches, but we expect the compiler to resolve this to - // two branches (easily predicted): - const uint8_t *to_base64 = - (options & base64_default_or_url) - ? tables::base64::to_base64_default_or_url_value - : ((options & base64_url) ? tables::base64::to_base64_url_value - : tables::base64::to_base64_value); - const uint32_t *d0 = - (options & base64_default_or_url) - ? tables::base64::base64_default_or_url::d0 - : ((options & base64_url) ? tables::base64::base64_url::d0 - : tables::base64::base64_default::d0); - const uint32_t *d1 = - (options & base64_default_or_url) - ? tables::base64::base64_default_or_url::d1 - : ((options & base64_url) ? tables::base64::base64_url::d1 - : tables::base64::base64_default::d1); - const uint32_t *d2 = - (options & base64_default_or_url) - ? tables::base64::base64_default_or_url::d2 - : ((options & base64_url) ? tables::base64::base64_url::d2 - : tables::base64::base64_default::d2); - const uint32_t *d3 = - (options & base64_default_or_url) - ? tables::base64::base64_default_or_url::d3 - : ((options & base64_url) ? tables::base64::base64_url::d3 - : tables::base64::base64_default::d3); - const bool ignore_garbage = - (options == base64_options::base64_url_accept_garbage) || - (options == base64_options::base64_default_accept_garbage) || - (options == base64_options::base64_default_or_url_accept_garbage); - - const char_type *srcend = src + length; - const char_type *srcinit = src; - const char *dstinit = dst; - - uint32_t x; - size_t idx; - uint8_t buffer[4]; - while (true) { - while (src + 4 <= srcend && is_eight_byte(src[0]) && - is_eight_byte(src[1]) && is_eight_byte(src[2]) && - is_eight_byte(src[3]) && - (x = d0[uint8_t(src[0])] | d1[uint8_t(src[1])] | - d2[uint8_t(src[2])] | d3[uint8_t(src[3])]) < 0x01FFFFFF) { - if (match_system(endianness::BIG)) { - x = scalar::u32_swap_bytes(x); - } - if (check_capacity && dstend - dst < 3) { - return {OUTPUT_BUFFER_TOO_SMALL, size_t(src - srcinit), - size_t(dst - dstinit)}; - } - std::memcpy(dst, &x, 3); // optimization opportunity: copy 4 bytes - dst += 3; - src += 4; - } - const char_type *srccur = src; - idx = 0; - // we need at least four characters. -#ifdef __clang__ - // If possible, we read four characters at a time. (It is an optimization.) - if (ignore_garbage && src + 4 <= srcend) { - char_type c0 = src[0]; - char_type c1 = src[1]; - char_type c2 = src[2]; - char_type c3 = src[3]; - - uint8_t code0 = to_base64[uint8_t(c0)]; - uint8_t code1 = to_base64[uint8_t(c1)]; - uint8_t code2 = to_base64[uint8_t(c2)]; - uint8_t code3 = to_base64[uint8_t(c3)]; - - buffer[idx] = code0; - idx += (is_eight_byte(c0) && code0 <= 63); - buffer[idx] = code1; - idx += (is_eight_byte(c1) && code1 <= 63); - buffer[idx] = code2; - idx += (is_eight_byte(c2) && code2 <= 63); - buffer[idx] = code3; - idx += (is_eight_byte(c3) && code3 <= 63); - src += 4; - } -#endif - while ((idx < 4) && (src < srcend)) { - char_type c = *src; - - uint8_t code = to_base64[uint8_t(c)]; - buffer[idx] = uint8_t(code); - if (is_eight_byte(c) && code <= 63) { - idx++; - } else if (!ignore_garbage && - (code > 64 || !scalar::base64::is_eight_byte(c))) { - return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), - size_t(dst - dstinit)}; - } else { - // We have a space or a newline or garbage. We ignore it. - } - src++; - } - if (idx != 4) { - simdutf_log_assert(idx < 4, "idx should be less than 4"); - // We never should have that the number of base64 characters + the - // number of padding characters is more than 4. - if (!ignore_garbage && (idx + padding_characters > 4)) { - return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), - size_t(dst - dstinit), true}; - } - - // The idea here is that in loose mode, - // if there is padding at all, it must be used - // to form 4-wise chunk. However, in loose mode, - // we do accept no padding at all. - if (!ignore_garbage && - last_chunk_options == last_chunk_handling_options::loose && - (idx >= 2) && padding_characters > 0 && - ((idx + padding_characters) & 3) != 0) { - return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), - size_t(dst - dstinit), true}; - } else - - // The idea here is that in strict mode, we do not want to accept - // incomplete base64 chunks. So if the chunk was otherwise valid, we - // return BASE64_INPUT_REMAINDER. - if (!ignore_garbage && - last_chunk_options == last_chunk_handling_options::strict && - (idx >= 2) && ((idx + padding_characters) & 3) != 0) { - // The partial chunk was at src - idx - return {BASE64_INPUT_REMAINDER, size_t(src - srcinit), - size_t(dst - dstinit), true}; - } else - // If there is a partial chunk with insufficient padding, with - // stop_before_partial, we need to just ignore it. In "only full" - // mode, skip the minute there are padding characters. - if ((last_chunk_options == - last_chunk_handling_options::stop_before_partial && - (padding_characters + idx < 4) && (idx != 0) && - (idx >= 2 || padding_characters == 0)) || - (last_chunk_options == - last_chunk_handling_options::only_full_chunks && - (idx >= 2 || padding_characters == 0))) { - // partial means that we are *not* going to consume the read - // characters. We need to rewind the src pointer. - src = srccur; - return {SUCCESS, size_t(src - srcinit), size_t(dst - dstinit)}; - } else { - if (idx == 2) { - uint32_t triple = (uint32_t(buffer[0]) << 3 * 6) + - (uint32_t(buffer[1]) << 2 * 6); - if (!ignore_garbage && - (last_chunk_options == last_chunk_handling_options::strict) && - (triple & 0xffff)) { - return {BASE64_EXTRA_BITS, size_t(src - srcinit), - size_t(dst - dstinit)}; - } - if (check_capacity && dstend - dst < 1) { - return {OUTPUT_BUFFER_TOO_SMALL, size_t(srccur - srcinit), - size_t(dst - dstinit)}; - } - if (match_system(endianness::BIG)) { - triple <<= 8; - std::memcpy(dst, &triple, 1); - } else { - triple = scalar::u32_swap_bytes(triple); - triple >>= 8; - std::memcpy(dst, &triple, 1); - } - dst += 1; - } else if (idx == 3) { - uint32_t triple = (uint32_t(buffer[0]) << 3 * 6) + - (uint32_t(buffer[1]) << 2 * 6) + - (uint32_t(buffer[2]) << 1 * 6); - if (!ignore_garbage && - (last_chunk_options == last_chunk_handling_options::strict) && - (triple & 0xff)) { - return {BASE64_EXTRA_BITS, size_t(src - srcinit), - size_t(dst - dstinit)}; - } - if (check_capacity && dstend - dst < 2) { - return {OUTPUT_BUFFER_TOO_SMALL, size_t(srccur - srcinit), - size_t(dst - dstinit)}; - } - if (match_system(endianness::BIG)) { - triple <<= 8; - std::memcpy(dst, &triple, 2); - } else { - triple = scalar::u32_swap_bytes(triple); - triple >>= 8; - std::memcpy(dst, &triple, 2); - } - dst += 2; - } else if (!ignore_garbage && idx == 1 && - (!is_partial(last_chunk_options) || - (is_partial(last_chunk_options) && - padding_characters > 0))) { - return {BASE64_INPUT_REMAINDER, size_t(src - srcinit), - size_t(dst - dstinit)}; - } else if (!ignore_garbage && idx == 0 && padding_characters > 0) { - return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), - size_t(dst - dstinit), true}; - } - return {SUCCESS, size_t(src - srcinit), size_t(dst - dstinit)}; - } - } - if (check_capacity && dstend - dst < 3) { - return {OUTPUT_BUFFER_TOO_SMALL, size_t(srccur - srcinit), - size_t(dst - dstinit)}; - } - uint32_t triple = - (uint32_t(buffer[0]) << 3 * 6) + (uint32_t(buffer[1]) << 2 * 6) + - (uint32_t(buffer[2]) << 1 * 6) + (uint32_t(buffer[3]) << 0 * 6); - if (match_system(endianness::BIG)) { - triple <<= 8; - std::memcpy(dst, &triple, 3); - } else { - triple = scalar::u32_swap_bytes(triple); - triple >>= 8; - std::memcpy(dst, &triple, 3); - } - dst += 3; - } -} - -template -full_result -base64_tail_decode(char *dst, const char_type *src, size_t length, - size_t padding_characters, // number of padding characters - // '=', typically 0, 1, 2. - base64_options options, - last_chunk_handling_options last_chunk_options) { - return base64_tail_decode_impl(dst, 0, src, length, padding_characters, - options, last_chunk_options); -} - -// like base64_tail_decode, but it will not write past the end of the output -// buffer. The outlen parameter is modified to reflect the number of bytes -// written. This functions assumes that the padding (=) has been removed. -// -template -full_result base64_tail_decode_safe( - char *dst, size_t outlen, const char_type *src, size_t length, - size_t padding_characters, // number of padding characters - // '=', typically 0, 1, 2. - base64_options options, last_chunk_handling_options last_chunk_options) { - return base64_tail_decode_impl(dst, outlen, src, length, - padding_characters, options, - last_chunk_options); -} - -inline full_result -patch_tail_result(full_result r, size_t previous_input, size_t previous_output, - size_t equallocation, size_t full_input_length, - last_chunk_handling_options last_chunk_options) { - r.input_count += previous_input; - r.output_count += previous_output; - if (r.padding_error) { - r.input_count = equallocation; - } - - if (r.error == error_code::SUCCESS) { - if (!is_partial(last_chunk_options)) { - // A success when we are not in stop_before_partial mode. - // means that we have consumed the whole input buffer. - r.input_count = full_input_length; - } else if (r.output_count % 3 != 0) { - r.input_count = full_input_length; - } - } - return r; -} - -// Returns the number of bytes written. The destination buffer must be large -// enough. It will add padding (=) if needed. -size_t tail_encode_base64(char *dst, const char *src, size_t srclen, - base64_options options) { - // By default, we use padding if we are not using the URL variant. - // This is check with ((options & base64_url) == 0) which returns true if we - // are not using the URL variant. However, we also allow 'inversion' of the - // convention with the base64_reverse_padding option. If the - // base64_reverse_padding option is set, we use padding if we are using the - // URL variant, and we omit it if we are not using the URL variant. This is - // checked with - // ((options & base64_reverse_padding) == base64_reverse_padding). - bool use_padding = - ((options & base64_url) == 0) ^ - ((options & base64_reverse_padding) == base64_reverse_padding); - // This looks like 3 branches, but we expect the compiler to resolve this to - // a single branch: - const char *e0 = (options & base64_url) ? tables::base64::base64_url::e0 - : tables::base64::base64_default::e0; - const char *e1 = (options & base64_url) ? tables::base64::base64_url::e1 - : tables::base64::base64_default::e1; - const char *e2 = (options & base64_url) ? tables::base64::base64_url::e2 - : tables::base64::base64_default::e2; - char *out = dst; - size_t i = 0; - uint8_t t1, t2, t3; - for (; i + 2 < srclen; i += 3) { - t1 = uint8_t(src[i]); - t2 = uint8_t(src[i + 1]); - t3 = uint8_t(src[i + 2]); - *out++ = e0[t1]; - *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; - *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; - *out++ = e2[t3]; - } - switch (srclen - i) { - case 0: - break; - case 1: - t1 = uint8_t(src[i]); - *out++ = e0[t1]; - *out++ = e1[(t1 & 0x03) << 4]; - if (use_padding) { - *out++ = '='; - *out++ = '='; - } - break; - default: /* case 2 */ - t1 = uint8_t(src[i]); - t2 = uint8_t(src[i + 1]); - *out++ = e0[t1]; - *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; - *out++ = e2[(t2 & 0x0F) << 2]; - if (use_padding) { - *out++ = '='; - } - } - return (size_t)(out - dst); -} - -template -simdutf_warn_unused size_t maximal_binary_length_from_base64( - const char_type *input, size_t length) noexcept { - // We follow https://infra.spec.whatwg.org/#forgiving-base64-decode - size_t padding = 0; - if (length > 0) { - if (input[length - 1] == '=') { - padding++; - if (length > 1 && input[length - 2] == '=') { - padding++; - } - } - } - size_t actual_length = length - padding; - if (actual_length % 4 <= 1) { - return actual_length / 4 * 3; - } - // if we have a valid input, then the remainder must be 2 or 3 adding one or - // two extra bytes. - return actual_length / 4 * 3 + (actual_length % 4) - 1; -} - -template -simdutf_warn_unused full_result base64_to_binary_details_impl( - const char_type *input, size_t length, char *output, base64_options options, - last_chunk_handling_options last_chunk_options) noexcept { - const bool ignore_garbage = - (options == base64_options::base64_url_accept_garbage) || - (options == base64_options::base64_default_accept_garbage) || - (options == base64_options::base64_default_or_url_accept_garbage); - auto ri = simdutf::scalar::base64::find_end(input, length, options); - size_t equallocation = ri.equallocation; - size_t equalsigns = ri.equalsigns; - length = ri.srclen; - size_t full_input_length = ri.full_input_length; - if (length == 0) { - if (!ignore_garbage && equalsigns > 0) { - return {INVALID_BASE64_CHARACTER, equallocation, 0}; - } - return {SUCCESS, full_input_length, 0}; - } - full_result r = scalar::base64::base64_tail_decode( - output, input, length, equalsigns, options, last_chunk_options); - r = scalar::base64::patch_tail_result(r, 0, 0, equallocation, - full_input_length, last_chunk_options); - if (!is_partial(last_chunk_options) && r.error == error_code::SUCCESS && - equalsigns > 0 && !ignore_garbage) { - // additional checks - if ((r.output_count % 3 == 0) || - ((r.output_count % 3) + 1 + equalsigns != 4)) { - return {INVALID_BASE64_CHARACTER, equallocation, r.output_count}; - } - } - // When is_partial(last_chunk_options) is true, we must either end with - // the end of the stream (beyond whitespace) or right after a non-ignorable - // character or at the very beginning of the stream. - // See https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 - if (is_partial(last_chunk_options) && r.error == error_code::SUCCESS && - r.input_count < full_input_length) { - // First check if we can extend the input to the end of the stream - while (r.input_count < full_input_length && - base64_ignorable(*(input + r.input_count), options)) { - r.input_count++; - } - // If we are still not at the end of the stream, then we must backtrack - // to the last non-ignorable character. - if (r.input_count < full_input_length) { - while (r.input_count > 0 && - base64_ignorable(*(input + r.input_count - 1), options)) { - r.input_count--; - } - } - } - return r; -} + last_chunk_handling_options::loose) const noexcept override; + simdutf_warn_unused result base64_to_binary( + const char16_t *input, size_t length, char *output, + base64_options options, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept override; + simdutf_warn_unused full_result base64_to_binary_details( + const char16_t *input, size_t length, char *output, + base64_options options, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept override; + size_t binary_to_base64(const char *input, size_t length, char *output, + base64_options options) const noexcept override; + size_t + binary_to_base64_with_lines(const char *input, size_t length, char *output, + size_t line_length, + base64_options options) const noexcept override; + const char *find(const char *start, const char *end, + char character) const noexcept override; + const char16_t *find(const char16_t *start, const char16_t *end, + char16_t character) const noexcept override; -template -simdutf_warn_unused full_result base64_to_binary_details_safe_impl( - const char_type *input, size_t length, char *output, size_t outlen, - base64_options options, - last_chunk_handling_options last_chunk_options) noexcept { - const bool ignore_garbage = - (options == base64_options::base64_url_accept_garbage) || - (options == base64_options::base64_default_accept_garbage) || - (options == base64_options::base64_default_or_url_accept_garbage); - auto ri = simdutf::scalar::base64::find_end(input, length, options); - size_t equallocation = ri.equallocation; - size_t equalsigns = ri.equalsigns; - length = ri.srclen; - size_t full_input_length = ri.full_input_length; - if (length == 0) { - if (!ignore_garbage && equalsigns > 0) { - return {INVALID_BASE64_CHARACTER, equallocation, 0}; - } - return {SUCCESS, full_input_length, 0}; - } - full_result r = scalar::base64::base64_tail_decode_safe( - output, outlen, input, length, equalsigns, options, last_chunk_options); - r = scalar::base64::patch_tail_result(r, 0, 0, equallocation, - full_input_length, last_chunk_options); - if (!is_partial(last_chunk_options) && r.error == error_code::SUCCESS && - equalsigns > 0 && !ignore_garbage) { - // additional checks - if ((r.output_count % 3 == 0) || - ((r.output_count % 3) + 1 + equalsigns != 4)) { - return {INVALID_BASE64_CHARACTER, equallocation, r.output_count}; - } - } +}; +} // namespace fallback +} // namespace simdutf - // When is_partial(last_chunk_options) is true, we must either end with - // the end of the stream (beyond whitespace) or right after a non-ignorable - // character or at the very beginning of the stream. - // See https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 - if (is_partial(last_chunk_options) && r.error == error_code::SUCCESS && - r.input_count < full_input_length) { - // First check if we can extend the input to the end of the stream - while (r.input_count < full_input_length && - base64_ignorable(*(input + r.input_count), options)) { - r.input_count++; - } - // If we are still not at the end of the stream, then we must backtrack - // to the last non-ignorable character. - if (r.input_count < full_input_length) { - while (r.input_count > 0 && - base64_ignorable(*(input + r.input_count - 1), options)) { - r.input_count--; - } - } - } - return r; -} +#endif // SIMDUTF_FALLBACK_IMPLEMENTATION_H +/* end file src/simdutf/fallback/implementation.h */ -simdutf_warn_unused size_t -base64_length_from_binary(size_t length, base64_options options) noexcept { - // By default, we use padding if we are not using the URL variant. - // This is check with ((options & base64_url) == 0) which returns true if we - // are not using the URL variant. However, we also allow 'inversion' of the - // convention with the base64_reverse_padding option. If the - // base64_reverse_padding option is set, we use padding if we are using the - // URL variant, and we omit it if we are not using the URL variant. This is - // checked with - // ((options & base64_reverse_padding) == base64_reverse_padding). - bool use_padding = - ((options & base64_url) == 0) ^ - ((options & base64_reverse_padding) == base64_reverse_padding); - if (!use_padding) { - return length / 3 * 4 + ((length % 3) ? (length % 3) + 1 : 0); - } - return (length + 2) / 3 * - 4; // We use padding to make the length a multiple of 4. -} - -// Return the length of the prefix that contains count base64 characters. -// Thus, if count is 3, the function returns the length of the prefix -// that contains 3 base64 characters. -// The function returns (size_t)-1 if there is not enough base64 characters in -// the input. -template -simdutf_warn_unused size_t prefix_length(size_t count, - simdutf::base64_options options, - const char_type *input, - size_t length) noexcept { - size_t i = 0; - while (i < length && is_ignorable(input[i], options)) { - i++; - } - if (count == 0) { - return i; // duh! - } - for (; i < length; i++) { - if (is_ignorable(input[i], options)) { - continue; - } - // We have a base64 character or a padding character. - count--; - if (count == 0) { - return i + 1; - } - } - simdutf_log_assert(false, "You never get here"); +/* begin file src/simdutf/fallback/begin.h */ +// redefining SIMDUTF_IMPLEMENTATION to "fallback" +// #define SIMDUTF_IMPLEMENTATION fallback +/* end file src/simdutf/fallback/begin.h */ - return -1; // should never happen -} + // Declarations +/* begin file src/simdutf/fallback/bitmanipulation.h */ +#ifndef SIMDUTF_FALLBACK_BITMANIPULATION_H +#define SIMDUTF_FALLBACK_BITMANIPULATION_H -} // namespace base64 -} // unnamed namespace -} // namespace scalar +#include + +namespace simdutf { +namespace fallback { +namespace {} // unnamed namespace +} // namespace fallback } // namespace simdutf +#endif // SIMDUTF_FALLBACK_BITMANIPULATION_H +/* end file src/simdutf/fallback/bitmanipulation.h */ + +/* begin file src/simdutf/fallback/end.h */ +/* end file src/simdutf/fallback/end.h */ + +#endif // SIMDUTF_IMPLEMENTATION_FALLBACK +#endif // SIMDUTF_FALLBACK_H +/* end file src/simdutf/fallback.h */ +#ifndef SIMDUTF_REGULAR_VISUAL_STUDIO +SIMDUTF_POP_DISABLE_WARNINGS #endif -/* end file src/scalar/base64.h */ + +// The scalar routines should be included once. /* begin file src/implementation.cpp */ -#include +#include #include #include +#include #if SIMDUTF_ATOMIC_REF -/* begin file src/scalar/atomic_util.h */ -#ifndef SIMDUTF_ATOMIC_UTIL_H -#define SIMDUTF_ATOMIC_UTIL_H -#if SIMDUTF_ATOMIC_REF - #include -namespace simdutf { -namespace scalar { - -// This function is a memcpy that uses atomic operations to read from the -// source. -inline void memcpy_atomic_read(char *dst, const char *src, size_t len) { - static_assert(std::atomic_ref::required_alignment == sizeof(char), - "std::atomic_ref requires the same alignment as char_type"); - // We expect all 64-bit systems to be able to read 64-bit words from an - // aligned memory region atomically. You might be able to do better on - // specific systems, e.g., x64 systems can read 128-bit words atomically. - constexpr size_t alignment = sizeof(uint64_t); - - // Lambda for atomic byte-by-byte copy - auto bbb_memcpy_atomic_read = [](char *bytedst, const char *bytesrc, - size_t bytelen) noexcept { - char *mutable_src = const_cast(bytesrc); - for (size_t j = 0; j < bytelen; ++j) { - bytedst[j] = - std::atomic_ref(mutable_src[j]).load(std::memory_order_relaxed); - } - }; - - // Handle unaligned start - size_t offset = reinterpret_cast(src) % alignment; - if (offset) { - size_t to_align = std::min(len, alignment - offset); - bbb_memcpy_atomic_read(dst, src, to_align); - src += to_align; - dst += to_align; - len -= to_align; - } - - // Process aligned 64-bit chunks - while (len >= alignment) { - auto *src_aligned = reinterpret_cast(const_cast(src)); - const auto dst_value = - std::atomic_ref(*src_aligned).load(std::memory_order_relaxed); - std::memcpy(dst, &dst_value, sizeof(uint64_t)); - src += alignment; - dst += alignment; - len -= alignment; - } - - // Handle remaining bytes - if (len) { - bbb_memcpy_atomic_read(dst, src, len); - } -} - -// This function is a memcpy that uses atomic operations to write to the -// destination. -inline void memcpy_atomic_write(char *dst, const char *src, size_t len) { - static_assert(std::atomic_ref::required_alignment == sizeof(char), - "std::atomic_ref requires the same alignment as char"); - // We expect all 64-bit systems to be able to write 64-bit words to an aligned - // memory region atomically. - // You might be able to do better on specific systems, e.g., x64 systems can - // write 128-bit words atomically. - constexpr size_t alignment = sizeof(uint64_t); - - // Lambda for atomic byte-by-byte write - auto bbb_memcpy_atomic_write = [](char *bytedst, const char *bytesrc, - size_t bytelen) noexcept { - for (size_t j = 0; j < bytelen; ++j) { - std::atomic_ref(bytedst[j]) - .store(bytesrc[j], std::memory_order_relaxed); - } - }; - - // Handle unaligned start - size_t offset = reinterpret_cast(dst) % alignment; - if (offset) { - size_t to_align = std::min(len, alignment - offset); - bbb_memcpy_atomic_write(dst, src, to_align); - dst += to_align; - src += to_align; - len -= to_align; - } - - // Process aligned 64-bit chunks - while (len >= alignment) { - auto *dst_aligned = reinterpret_cast(dst); - uint64_t src_val; - std::memcpy(&src_val, src, sizeof(uint64_t)); // Non-atomic read from src - std::atomic_ref(*dst_aligned) - .store(src_val, std::memory_order_relaxed); - dst += alignment; - src += alignment; - len -= alignment; - } + #include +#endif - // Handle remaining bytes - if (len) { - bbb_memcpy_atomic_write(dst, src, len); - } -} -} // namespace scalar -} // namespace simdutf -#endif // SIMDUTF_ATOMIC_REF -#endif // SIMDUTF_ATOMIC_UTIL_H -/* end file src/scalar/atomic_util.h */ +// The macro SIMDUTF_USE_STATIC_INITIALIZATION, when set to 1, means that we +// will use translation-unit-scope variables to hold our implementations. +// +// The downside of a translation-unit-scope variable is that the initialization +// order is not well defined, thus if someone uses simdutf before main() starts, +// they might get a crash. Thus setting SIMDUTF_USE_STATIC_INITIALIZATION to 1 +// is not recommended if you are using simdutf in a library that might be used +// by other code before main() starts. However, the upside is that there is no +// synchronization overhead on every call to get_active_implementation(). When +// compiling without the c++ standard library, we use static initialization, +// because C++ relies on the standard library for thread-safe initialization of +// function-scope static variables. +// +// By default, we avoid translation-unit-scope static initialization, so we set +// SIMDUTF_USE_STATIC_INITIALIZATION to 0. It comes with a small performance +// cost on the first call to get_active_implementation(), and a smaller cost on +// subsequent calls but it is then safe to use the simdutf library in static +// initialization. +// +// Further reading: https://en.cppreference.com/cpp/language/siof +#ifndef SIMDUTF_USE_STATIC_INITIALIZATION + #if SIMDUTF_NO_LIBCXX + #define SIMDUTF_USE_STATIC_INITIALIZATION 1 + #else // SIMDUTF_NO_LIBCXX + #define SIMDUTF_USE_STATIC_INITIALIZATION 0 + #endif // SIMDUTF_NO_LIBCXX +#endif // SIMDUTF_USE_STATIC_INITIALIZATION + +// When building without libc++abi (SIMDUTF_NO_LIBCXX=1) on GCC/Clang, provide +// a weak stub for __cxa_pure_virtual so the abstract implementation vtable +// does not drag in libc++abi just for this unreachable hook. Kept weak so a +// real libc++abi definition wins if one is linked in anyway. +#if SIMDUTF_NO_LIBCXX +extern "C" __attribute__((weak, noreturn)) void __cxa_pure_virtual() { + __builtin_trap(); +} +namespace std { +__attribute__((weak, noreturn)) void +__glibcxx_assert_fail(const char *, int, const char *, const char *) noexcept { + __builtin_trap(); +} +} // namespace std #endif static_assert(sizeof(uint8_t) == sizeof(char), @@ -11514,22 +10023,6 @@ static_assert(sizeof(uint32_t) == sizeof(char32_t), // next line is redundant, but it is kept to catch defective systems. static_assert(CHAR_BIT == 8, "simdutf requires 8-bit bytes"); -// Useful for debugging purposes -namespace simdutf { -namespace { - -template std::string toBinaryString(T b) { - std::string binary = ""; - T mask = T(1) << (sizeof(T) * CHAR_BIT - 1); - while (mask > 0) { - binary += ((b & mask) == 0) ? '0' : '1'; - mask >>= 1; - } - return binary; -} -} // namespace -} // namespace simdutf - namespace simdutf { bool implementation::supported_by_runtime_system() const { uint32_t required_instruction_sets = this->required_instruction_sets(); @@ -11549,6 +10042,16 @@ simdutf_warn_unused size_t implementation::maximal_binary_length_from_base64( return scalar::base64::maximal_binary_length_from_base64(input, length); } +simdutf_warn_unused size_t implementation::binary_length_from_base64( + const char *input, size_t length) const noexcept { + return scalar::base64::binary_length_from_base64(input, length); +} + +simdutf_warn_unused size_t implementation::binary_length_from_base64( + const char16_t *input, size_t length) const noexcept { + return scalar::base64::binary_length_from_base64(input, length); +} + simdutf_warn_unused size_t implementation::base64_length_from_binary( size_t length, base64_options options) const noexcept { return scalar::base64::base64_length_from_binary(length, options); @@ -11565,66 +10068,108 @@ namespace internal { SIMDUTF_IMPLEMENTATION_LASX + SIMDUTF_IMPLEMENTATION_FALLBACK == \ 1) -// Static array of known implementations. We are hoping these get baked into the -// executable without requiring a static initializer. - #if SIMDUTF_IMPLEMENTATION_ICELAKE + #if SIMDUTF_USE_STATIC_INITIALIZATION +static const icelake::implementation icelake_singleton{}; + #endif static const icelake::implementation *get_icelake_singleton() { + #if !SIMDUTF_USE_STATIC_INITIALIZATION static const icelake::implementation icelake_singleton{}; + #endif return &icelake_singleton; } #endif #if SIMDUTF_IMPLEMENTATION_HASWELL + #if SIMDUTF_USE_STATIC_INITIALIZATION +static const haswell::implementation haswell_singleton{}; + #endif static const haswell::implementation *get_haswell_singleton() { + #if !SIMDUTF_USE_STATIC_INITIALIZATION static const haswell::implementation haswell_singleton{}; + #endif return &haswell_singleton; } #endif #if SIMDUTF_IMPLEMENTATION_WESTMERE + #if SIMDUTF_USE_STATIC_INITIALIZATION +static const westmere::implementation westmere_singleton{}; + #endif static const westmere::implementation *get_westmere_singleton() { + #if !SIMDUTF_USE_STATIC_INITIALIZATION static const westmere::implementation westmere_singleton{}; + #endif return &westmere_singleton; } #endif #if SIMDUTF_IMPLEMENTATION_ARM64 + #if SIMDUTF_USE_STATIC_INITIALIZATION +static const arm64::implementation arm64_singleton{}; + #endif static const arm64::implementation *get_arm64_singleton() { + #if !SIMDUTF_USE_STATIC_INITIALIZATION static const arm64::implementation arm64_singleton{}; + #endif return &arm64_singleton; } #endif #if SIMDUTF_IMPLEMENTATION_PPC64 + #if SIMDUTF_USE_STATIC_INITIALIZATION +static const ppc64::implementation ppc64_singleton{}; + #endif static const ppc64::implementation *get_ppc64_singleton() { + #if !SIMDUTF_USE_STATIC_INITIALIZATION static const ppc64::implementation ppc64_singleton{}; + #endif return &ppc64_singleton; } #endif #if SIMDUTF_IMPLEMENTATION_RVV + #if SIMDUTF_USE_STATIC_INITIALIZATION +static const rvv::implementation rvv_singleton{}; + #endif static const rvv::implementation *get_rvv_singleton() { + #if !SIMDUTF_USE_STATIC_INITIALIZATION static const rvv::implementation rvv_singleton{}; + #endif return &rvv_singleton; } #endif -#if SIMDUTF_IMPLEMENTATION_LSX -static const lsx::implementation *get_lsx_singleton() { - static const lsx::implementation lsx_singleton{}; - return &lsx_singleton; -} -#endif #if SIMDUTF_IMPLEMENTATION_LASX + #if SIMDUTF_USE_STATIC_INITIALIZATION +static const lasx::implementation lasx_singleton{}; + #endif static const lasx::implementation *get_lasx_singleton() { + #if !SIMDUTF_USE_STATIC_INITIALIZATION static const lasx::implementation lasx_singleton{}; + #endif return &lasx_singleton; } #endif +#if SIMDUTF_IMPLEMENTATION_LSX + #if SIMDUTF_USE_STATIC_INITIALIZATION +static const lsx::implementation lsx_singleton{}; + #endif +static const lsx::implementation *get_lsx_singleton() { + #if !SIMDUTF_USE_STATIC_INITIALIZATION + static const lsx::implementation lsx_singleton{}; + #endif + return &lsx_singleton; +} +#endif #if SIMDUTF_IMPLEMENTATION_FALLBACK + #if SIMDUTF_USE_STATIC_INITIALIZATION +static const fallback::implementation fallback_singleton{}; + #endif static const fallback::implementation *get_fallback_singleton() { + #if !SIMDUTF_USE_STATIC_INITIALIZATION static const fallback::implementation fallback_singleton{}; + #endif return &fallback_singleton; } #endif #if SIMDUTF_SINGLE_IMPLEMENTATION -static const implementation *get_single_implementation() { +simdutf_really_inline static const implementation *get_single_implementation() { return #if SIMDUTF_IMPLEMENTATION_ICELAKE get_icelake_singleton(); @@ -11641,12 +10186,12 @@ static const implementation *get_single_implementation() { #if SIMDUTF_IMPLEMENTATION_PPC64 get_ppc64_singleton(); #endif - #if SIMDUTF_IMPLEMENTATION_LSX - get_lsx_singleton(); - #endif #if SIMDUTF_IMPLEMENTATION_LASX get_lasx_singleton(); #endif + #if SIMDUTF_IMPLEMENTATION_LSX + get_lsx_singleton(); + #endif #if SIMDUTF_IMPLEMENTATION_FALLBACK get_fallback_singleton(); #endif @@ -11659,8 +10204,8 @@ static const implementation *get_single_implementation() { class detect_best_supported_implementation_on_first_use final : public implementation { public: - std::string name() const noexcept final { return set_best()->name(); } - std::string description() const noexcept final { + std::string_view name() const noexcept final { return set_best()->name(); } + std::string_view description() const noexcept final { return set_best()->description(); } uint32_t required_instruction_sets() const noexcept final { @@ -11706,6 +10251,14 @@ class detect_best_supported_implementation_on_first_use final return set_best()->binary_to_base64(input, length, output, options); } + size_t + binary_to_base64_with_lines(const char *input, size_t length, char *output, + size_t line_length, + base64_options options) const noexcept override { + return set_best()->binary_to_base64_with_lines(input, length, output, + line_length, options); + } + const char *find(const char *start, const char *end, char character) const noexcept override { return set_best()->find(start, end, character); @@ -11716,6 +10269,16 @@ class detect_best_supported_implementation_on_first_use final return set_best()->find(start, end, character); } + simdutf_warn_unused size_t binary_length_from_base64( + const char *input, size_t length) const noexcept override { + return set_best()->binary_length_from_base64(input, length); + } + + simdutf_warn_unused size_t binary_length_from_base64( + const char16_t *input, size_t length) const noexcept override { + return set_best()->binary_length_from_base64(input, length); + } + simdutf_really_inline detect_best_supported_implementation_on_first_use() noexcept : implementation("best_supported_detector", @@ -11731,38 +10294,72 @@ static_assert(std::is_trivially_destructible< "detect_best_supported_implementation_on_first_use should be " "trivially destructible"); +#if SIMDUTF_USE_STATIC_INITIALIZATION +static const std::initializer_list + available_implementation_pointers{ + #if SIMDUTF_IMPLEMENTATION_ICELAKE + get_icelake_singleton(), + #endif + #if SIMDUTF_IMPLEMENTATION_HASWELL + get_haswell_singleton(), + #endif + #if SIMDUTF_IMPLEMENTATION_WESTMERE + get_westmere_singleton(), + #endif + #if SIMDUTF_IMPLEMENTATION_ARM64 + get_arm64_singleton(), + #endif + #if SIMDUTF_IMPLEMENTATION_PPC64 + get_ppc64_singleton(), + #endif + #if SIMDUTF_IMPLEMENTATION_RVV + get_rvv_singleton(), + #endif + #if SIMDUTF_IMPLEMENTATION_LASX + get_lasx_singleton(), + #endif + #if SIMDUTF_IMPLEMENTATION_LSX + get_lsx_singleton(), + #endif + #if SIMDUTF_IMPLEMENTATION_FALLBACK + get_fallback_singleton(), + #endif + }; +#endif static const std::initializer_list & get_available_implementation_pointers() { +#if !SIMDUTF_USE_STATIC_INITIALIZATION static const std::initializer_list available_implementation_pointers{ -#if SIMDUTF_IMPLEMENTATION_ICELAKE + #if SIMDUTF_IMPLEMENTATION_ICELAKE get_icelake_singleton(), -#endif -#if SIMDUTF_IMPLEMENTATION_HASWELL + #endif + #if SIMDUTF_IMPLEMENTATION_HASWELL get_haswell_singleton(), -#endif -#if SIMDUTF_IMPLEMENTATION_WESTMERE + #endif + #if SIMDUTF_IMPLEMENTATION_WESTMERE get_westmere_singleton(), -#endif -#if SIMDUTF_IMPLEMENTATION_ARM64 + #endif + #if SIMDUTF_IMPLEMENTATION_ARM64 get_arm64_singleton(), -#endif -#if SIMDUTF_IMPLEMENTATION_PPC64 + #endif + #if SIMDUTF_IMPLEMENTATION_PPC64 get_ppc64_singleton(), -#endif -#if SIMDUTF_IMPLEMENTATION_RVV + #endif + #if SIMDUTF_IMPLEMENTATION_RVV get_rvv_singleton(), -#endif -#if SIMDUTF_IMPLEMENTATION_LSX - get_lsx_singleton(), -#endif -#if SIMDUTF_IMPLEMENTATION_LASX + #endif + #if SIMDUTF_IMPLEMENTATION_LASX get_lasx_singleton(), -#endif -#if SIMDUTF_IMPLEMENTATION_FALLBACK + #endif + #if SIMDUTF_IMPLEMENTATION_LSX + get_lsx_singleton(), + #endif + #if SIMDUTF_IMPLEMENTATION_FALLBACK get_fallback_singleton(), + #endif + }; #endif - }; // available_implementation_pointers return available_implementation_pointers; } @@ -11799,6 +10396,10 @@ class unsupported_implementation final : public implementation { base64_options) const noexcept override { return 0; } + size_t binary_to_base64_with_lines(const char *, size_t, char *, size_t, + base64_options) const noexcept override { + return 0; + } const char *find(const char *, const char *, char) const noexcept override { return nullptr; } @@ -11806,14 +10407,27 @@ class unsupported_implementation final : public implementation { char16_t) const noexcept override { return nullptr; } + simdutf_warn_unused size_t + binary_length_from_base64(const char *, size_t) const noexcept override { + return 0; + } + simdutf_warn_unused size_t + binary_length_from_base64(const char16_t *, size_t) const noexcept override { + return 0; + } unsupported_implementation() : implementation("unsupported", "Unsupported CPU (no detected SIMD instructions)", 0) {} }; +#if SIMDUTF_USE_STATIC_INITIALIZATION +static const unsupported_implementation unsupported_singleton{}; +#endif const unsupported_implementation *get_unsupported_singleton() { +#if !SIMDUTF_USE_STATIC_INITIALIZATION static const unsupported_implementation unsupported_singleton{}; +#endif return &unsupported_singleton; } static_assert(std::is_trivially_destructible::value, @@ -11873,49 +10487,174 @@ detect_best_supported_implementation_on_first_use::set_best() const noexcept { /** * The list of available implementations compiled into simdutf. */ +#if SIMDUTF_USE_STATIC_INITIALIZATION +static const internal::available_implementation_list + available_implementations_instance{}; +#endif SIMDUTF_DLLIMPORTEXPORT const internal::available_implementation_list & get_available_implementations() { +#if !SIMDUTF_USE_STATIC_INITIALIZATION static const internal::available_implementation_list - available_implementations{}; - return available_implementations; + available_implementations_instance{}; +#endif + return available_implementations_instance; } +#if SIMDUTF_USE_STATIC_INITIALIZATION && !SIMDUTF_SINGLE_IMPLEMENTATION +static const internal::detect_best_supported_implementation_on_first_use + detect_best_supported_implementation_on_first_use_singleton; +#endif + +#if SIMDUTF_USE_STATIC_INITIALIZATION +static internal::atomic_ptr + active_implementation_instance{ + #if SIMDUTF_SINGLE_IMPLEMENTATION + internal::get_single_implementation() + #else + &detect_best_supported_implementation_on_first_use_singleton + #endif + }; +#endif + /** * The active implementation. */ SIMDUTF_DLLIMPORTEXPORT internal::atomic_ptr & get_active_implementation() { -#if SIMDUTF_SINGLE_IMPLEMENTATION - // skip runtime detection - static internal::atomic_ptr active_implementation{ - internal::get_single_implementation()}; - return active_implementation; -#else +#if !SIMDUTF_USE_STATIC_INITIALIZATION + #if !SIMDUTF_SINGLE_IMPLEMENTATION static const internal::detect_best_supported_implementation_on_first_use detect_best_supported_implementation_on_first_use_singleton; - static internal::atomic_ptr active_implementation{ - &detect_best_supported_implementation_on_first_use_singleton}; - return active_implementation; + #endif + static internal::atomic_ptr + active_implementation_instance{ + #if SIMDUTF_SINGLE_IMPLEMENTATION + internal::get_single_implementation() + #else + &detect_best_supported_implementation_on_first_use_singleton + #endif + }; #endif + return active_implementation_instance; } #if SIMDUTF_SINGLE_IMPLEMENTATION -const implementation *get_default_implementation() { +simdutf_really_inline const implementation *get_default_implementation() { return internal::get_single_implementation(); } #else -internal::atomic_ptr &get_default_implementation() { +simdutf_really_inline internal::atomic_ptr & +get_default_implementation() { return get_active_implementation(); } #endif -#define SIMDUTF_GET_CURRENT_IMPLEMENTION +#define SIMDUTF_GET_CURRENT_IMPLEMENTATION + + #if SIMDUTF_ATOMIC_REF +template +simdutf_warn_unused result atomic_base64_to_binary_safe_impl( + const char_type *input, size_t length, char *output, size_t &outlen, + base64_options options, + last_chunk_handling_options last_chunk_handling_options, + bool decode_up_to_bad_char) noexcept { + #if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) + // We use a smaller buffer during fuzzing to more easily detect bugs. + constexpr size_t buffer_size = 128; + #else + // Arbitrary block sizes: 4KB for input. + constexpr size_t buffer_size = 4096; + #endif + std::array temp_buffer; + const char_type *const input_init = input; + size_t actual_out = 0; + bool last_chunk = false; + const size_t length_init = length; + result r; + while (!last_chunk) { + last_chunk |= (temp_buffer.size() >= outlen - actual_out); + size_t temp_outlen = (std::min)(temp_buffer.size(), outlen - actual_out); + r = base64_to_binary_safe(input, length, temp_buffer.data(), temp_outlen, + options, last_chunk_handling_options, + decode_up_to_bad_char); + // We processed r.count characters of input. + // We wrote temp_outlen bytes to temp_buffer. + // If there is no ignorable characters, + // we should expect that values/4.0*3 == temp_outlen, + // except maybe at the tail end of the string. + + // + // We are assuming that when r.error == error_code::OUTPUT_BUFFER_TOO_SMALL, + // we truncate the results so that a number of base64 characters divisible + // by four is processed. + // + + // + // We wrote temp_outlen bytes to temp_buffer. + // We need to copy them to output. + // Copy with relaxed atomic operations to the output + simdutf_log_assert(temp_outlen <= outlen - actual_out, + "Output buffer is too small"); + simdutf_log_assert(temp_outlen <= temp_buffer.size(), + "Output buffer is too small"); + + simdutf::scalar::memcpy_atomic_write(output + actual_out, + temp_buffer.data(), temp_outlen); + actual_out += temp_outlen; + length -= r.count; + input += r.count; + + if (r.error != error_code::OUTPUT_BUFFER_TOO_SMALL) { + break; + } + } + if (size_t(input - input_init) != length_init) { + // We did not process all input characters. In such case, we + // should not end with an ignorable character. See + // https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 + while (input > input_init && base64_ignorable(*(input - 1), options)) { + --input; + } + } + outlen = actual_out; + return {r.error, size_t(input - input_init)}; +} + +simdutf_warn_unused result atomic_base64_to_binary_safe( + const char *input, size_t length, char *output, size_t &outlen, + base64_options options, + last_chunk_handling_options last_chunk_handling_options, + bool decode_up_to_bad_char) noexcept { + return atomic_base64_to_binary_safe_impl( + input, length, output, outlen, options, last_chunk_handling_options, + decode_up_to_bad_char); +} +simdutf_warn_unused result atomic_base64_to_binary_safe( + const char16_t *input, size_t length, char *output, size_t &outlen, + base64_options options, + last_chunk_handling_options last_chunk_handling_options, + bool decode_up_to_bad_char) noexcept { + return atomic_base64_to_binary_safe_impl( + input, length, output, outlen, options, last_chunk_handling_options, + decode_up_to_bad_char); +} + #endif // SIMDUTF_ATOMIC_REF -simdutf_warn_unused const char *find(const char *start, const char *end, - char character) noexcept { +// this has been moved to implementation.h +// simdutf_warn_unused size_t +// base64_length_from_binary(size_t length, base64_options option) noexcept; + +// this has been moved to implementation.h +// simdutf_warn_unused size_t base64_length_from_binary_with_lines( +// size_t length, base64_options options, size_t line_length) noexcept; +// } + +simdutf_warn_unused const char *detail::find(const char *start, const char *end, + char character) noexcept { return get_default_implementation()->find(start, end, character); } -simdutf_warn_unused const char16_t * -find(const char16_t *start, const char16_t *end, char16_t character) noexcept { +simdutf_warn_unused const char16_t *detail::find(const char16_t *start, + const char16_t *end, + char16_t character) noexcept { return get_default_implementation()->find(start, end, character); } @@ -11938,6 +10677,16 @@ simdutf_warn_unused size_t maximal_binary_length_from_base64( input, length); } +simdutf_warn_unused size_t binary_length_from_base64(const char *input, + size_t length) noexcept { + return get_default_implementation()->binary_length_from_base64(input, length); +} + +simdutf_warn_unused size_t binary_length_from_base64(const char16_t *input, + size_t length) noexcept { + return get_default_implementation()->binary_length_from_base64(input, length); +} + simdutf_warn_unused result base64_to_binary( const char16_t *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_handling_options) noexcept { @@ -11945,167 +10694,37 @@ simdutf_warn_unused result base64_to_binary( input, length, output, options, last_chunk_handling_options); } -template -simdutf_warn_unused result slow_base64_to_binary_safe_impl( - const chartype *input, size_t length, char *output, size_t &outlen, - base64_options options, - last_chunk_handling_options last_chunk_options) noexcept { - const bool ignore_garbage = (options & base64_default_accept_garbage) != 0; - auto ri = simdutf::scalar::base64::find_end(input, length, options); - size_t equallocation = ri.equallocation; - size_t equalsigns = ri.equalsigns; - length = ri.srclen; - size_t full_input_length = ri.full_input_length; - (void)full_input_length; - if (length == 0) { - outlen = 0; - if (!ignore_garbage && equalsigns > 0) { - return {INVALID_BASE64_CHARACTER, equallocation}; - } - return {SUCCESS, 0}; - } - - // The parameters of base64_tail_decode_safe are: - // - dst: the output buffer - // - outlen: the size of the output buffer - // - srcr: the input buffer - // - length: the size of the input buffer - // - padded_characters: the number of padding characters - // - options: the options for the base64 decoder - // - last_chunk_options: the options for the last chunk - // The function will return the number of bytes written to the output buffer - // and the number of bytes read from the input buffer. - // The function will also return an error code if the input buffer is not - // valid base64. - full_result r = scalar::base64::base64_tail_decode_safe( - output, outlen, input, length, equalsigns, options, last_chunk_options); - r = scalar::base64::patch_tail_result(r, 0, 0, equallocation, - full_input_length, last_chunk_options); - outlen = r.output_count; - if (!is_partial(last_chunk_options) && r.error == error_code::SUCCESS && - equalsigns > 0) { - // additional checks - if ((outlen % 3 == 0) || ((outlen % 3) + 1 + equalsigns != 4)) { - r.error = error_code::INVALID_BASE64_CHARACTER; - } - } - return {r.error, r.input_count}; // we cannot return r itself because it gets - // converted to error/output_count -} -simdutf_warn_unused bool base64_ignorable(char input, - base64_options options) noexcept { - return scalar::base64::is_ignorable(input, options); -} -simdutf_warn_unused bool base64_ignorable(char16_t input, - base64_options options) noexcept { - return scalar::base64::is_ignorable(input, options); -} -simdutf_warn_unused bool base64_valid(char input, - base64_options options) noexcept { - return scalar::base64::is_base64(input, options); -} -simdutf_warn_unused bool base64_valid(char16_t input, - base64_options options) noexcept { - return scalar::base64::is_base64(input, options); -} -simdutf_warn_unused bool -base64_valid_or_padding(char input, base64_options options) noexcept { - return scalar::base64::is_base64_or_padding(input, options); -} -simdutf_warn_unused bool -base64_valid_or_padding(char16_t input, base64_options options) noexcept { - return scalar::base64::is_base64_or_padding(input, options); +simdutf_warn_unused full_result base64_to_binary_details( + const char *input, size_t length, char *output, base64_options options, + last_chunk_handling_options last_chunk_handling_options) noexcept { + return get_default_implementation()->base64_to_binary_details( + input, length, output, options, last_chunk_handling_options); } -template -simdutf_warn_unused result base64_to_binary_safe_impl( - const chartype *input, size_t length, char *output, size_t &outlen, - base64_options options, - last_chunk_handling_options last_chunk_handling_options, - bool decode_up_to_bad_char) noexcept { - static_assert(std::is_same::value || - std::is_same::value, - "Only char and char16_t are supported."); - size_t remaining_input_length = length; - size_t remaining_output_length = outlen; - size_t input_position = 0; - size_t output_position = 0; - - // We also do a first pass using the fast path to decode as much as possible - size_t safe_input = (std::min)( - remaining_input_length, - base64_length_from_binary(remaining_output_length / 3 * 3, options)); - bool done_with_partial = (safe_input == remaining_input_length); - simdutf::full_result r = - get_default_implementation()->base64_to_binary_details( - input + input_position, safe_input, output + output_position, options, - done_with_partial - ? last_chunk_handling_options - : simdutf::last_chunk_handling_options::only_full_chunks); - simdutf_log_assert(r.input_count <= safe_input, - "You should not read more than safe_input"); - simdutf_log_assert(r.output_count <= remaining_output_length, - "You should not write more than remaining_output_length"); - // Technically redundant, but we want to be explicit about it. - input_position += r.input_count; - output_position += r.output_count; - remaining_input_length -= r.input_count; - remaining_output_length -= r.output_count; - if (r.error != simdutf::error_code::SUCCESS) { - // There is an error. We return. - if (decode_up_to_bad_char && - r.error == error_code::INVALID_BASE64_CHARACTER) { - return slow_base64_to_binary_safe_impl( - input, length, output, outlen, options, last_chunk_handling_options); - } - outlen = output_position; - return {r.error, input_position}; - } - - if (done_with_partial) { - // We are done. We have decoded everything. - outlen = output_position; - return {simdutf::error_code::SUCCESS, input_position}; - } - // We have decoded some data, but we still have some data to decode. - // We need to decode the rest of the input buffer. - r = simdutf::scalar::base64::base64_to_binary_details_safe_impl( - input + input_position, remaining_input_length, output + output_position, - remaining_output_length, options, last_chunk_handling_options); - input_position += r.input_count; - output_position += r.output_count; - remaining_input_length -= r.input_count; - remaining_output_length -= r.output_count; - - if (r.error != simdutf::error_code::SUCCESS) { - // There is an error. We return. - if (decode_up_to_bad_char && - r.error == error_code::INVALID_BASE64_CHARACTER) { - return slow_base64_to_binary_safe_impl( - input, length, output, outlen, options, last_chunk_handling_options); - } - outlen = output_position; - return {r.error, input_position}; - } - if (input_position < length) { - // We cannot process the entire input in one go, so we need to - // process it in two steps: first the fast path, then the slow path. - // In some cases, the processing might 'eat up' trailing ignorable - // characters in the fast path, but that can be a problem. - // suppose we have just white space followed by a single base64 character. - // If we first process the white space with the fast path, it will - // eat all of it. But, by the JavaScript standard, we should consume - // no character. See - // https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 - while (input_position > 0 && - base64_ignorable(input[input_position - 1], options)) { - input_position--; - } - } - outlen = output_position; - return {simdutf::error_code::SUCCESS, input_position}; +simdutf_warn_unused full_result base64_to_binary_details( + const char16_t *input, size_t length, char *output, base64_options options, + last_chunk_handling_options last_chunk_handling_options) noexcept { + return get_default_implementation()->base64_to_binary_details( + input, length, output, options, last_chunk_handling_options); } +// moved to implementation.h +// simdutf_warn_unused bool base64_ignorable(char input, +// base64_options options) noexcept +// simdutf_warn_unused bool base64_ignorable(char16_t input, +// base64_options options) noexcept +// simdutf_warn_unused bool base64_valid(char input, +// base64_options options) noexcept +// simdutf_warn_unused bool base64_valid(char16_t input, +// base64_options options) noexcept +// simdutf_warn_unused bool +// base64_valid_or_padding(char input, base64_options options) noexcept +// simdutf_warn_unused bool +// base64_valid_or_padding(char16_t input, base64_options options) noexcept + +// base64_to_binary_safe_impl is moved to +// include/simdutf/base64_implementation.h + #if SIMDUTF_ATOMIC_REF size_t atomic_binary_to_base64(const char *input, size_t length, char *output, base64_options options) noexcept { @@ -12149,23 +10768,31 @@ base64_to_binary_safe(const char16_t *input, size_t length, char *output, decode_up_to_bad_char); } -simdutf_warn_unused size_t -base64_length_from_binary(size_t length, base64_options options) noexcept { - return get_default_implementation()->base64_length_from_binary(length, - options); -} - size_t binary_to_base64(const char *input, size_t length, char *output, base64_options options) noexcept { return get_default_implementation()->binary_to_base64(input, length, output, options); } +size_t binary_to_base64_with_lines(const char *input, size_t length, + char *output, size_t line_length, + base64_options options) noexcept { + return get_default_implementation()->binary_to_base64_with_lines( + input, length, output, line_length, options); +} + +#if SIMDUTF_USE_STATIC_INITIALIZATION +static const implementation *const builtin_impl_instance = + get_available_implementations()[SIMDUTF_STRINGIFY( + SIMDUTF_BUILTIN_IMPLEMENTATION)]; +#endif const implementation *builtin_implementation() { - static const implementation *builtin_impl = +#if !SIMDUTF_USE_STATIC_INITIALIZATION + static const implementation *const builtin_impl_instance = get_available_implementations()[SIMDUTF_STRINGIFY( SIMDUTF_BUILTIN_IMPLEMENTATION)]; - return builtin_impl; +#endif + return builtin_impl_instance; } } // namespace simdutf @@ -12179,7 +10806,6 @@ SIMDUTF_DISABLE_UNDESIRED_WARNINGS /* begin file src/simdutf/arm64/begin.h */ // redefining SIMDUTF_IMPLEMENTATION to "arm64" // #define SIMDUTF_IMPLEMENTATION arm64 -#define SIMDUTF_SIMD_HAS_BYTEMASK 1 /* end file src/simdutf/arm64/begin.h */ namespace simdutf { namespace arm64 { @@ -12218,8 +10844,101 @@ using namespace simd; * https://www.codeproject.com/Articles/276993/Base-Encoding-on-a-GPU. (2013). */ -size_t encode_base64(char *dst, const char *src, size_t srclen, - base64_options options) { +/** + * Insert a line feed character in the 16-byte input at index K in [0,16). + */ +inline uint8x16_t insert_line_feed16(uint8x16_t input, size_t K) { + static const uint8_t shuffle_masks[16][16] = { + {0x80, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 0x80, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 0x80, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 0x80, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 0x80, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 0x80, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 0x80, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 0x80, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 0x80, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 0x80, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0x80, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0x80, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0x80, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 0x80, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0x80, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0x80}}; + // Prepare a vector with '\n' (0x0A) + uint8x16_t line_feed_vector = vdupq_n_u8('\n'); + + // Load the precomputed shuffle mask for K + uint8x16_t mask = vld1q_u8(shuffle_masks[K]); + + // Create a mask where 0x80 indicates the line feed position + uint8x16_t lf_pos = vceqq_u8(mask, vdupq_n_u8(0x80)); + + uint8x16_t result = vqtbl1q_u8(input, mask); + + // Use vbsl to select '\n' where lf_pos is true, else keep input bytes + return vbslq_u8(lf_pos, line_feed_vector, result); +} + +// offset is the number of characters in the current line. +// It can range from 0 to line_length (inclusive). +// If offset == line_length, we need to insert a line feed before writing +// anything. +size_t write_output_with_line_feeds(uint8_t *dst, uint8x16_t src, + size_t line_length, size_t &offset) { + // Fast path: no need to insert line feeds + // If we are at offset, we would write from [offset, offset + 16). + // We need that line_length >= offset + 16. + if (offset + 16 <= line_length) { + // No need to insert line feeds + vst1q_u8(dst, src); + offset += 16; // offset could be line_length here. + return 16; + } + + // We have that offset + 16 >= line_length + // the common case is that line_length is greater than 16 + if (simdutf_likely(line_length >= 16)) { + // offset <= line_length. + // offset + 16 > line_length + // So line_length - offset < 16 + // and line_length - offset >= 0 + uint8x16_t chunk = insert_line_feed16(src, line_length - offset); + vst1q_u8(dst, chunk); + // Not ideal to pull the last element and write it separately but + // it simplifies the code. + *(dst + 16) = vgetq_lane_u8(src, 15); + offset += 16 - line_length; + return 16 + 1; // we wrote 16 bytes plus one line feed + } + // Uncommon case where line_length < 16 + // This is going to be SLOW. + else { + uint8_t buffer[16]; + vst1q_u8(buffer, src); + size_t out_pos = 0; + size_t local_offset = offset; + for (size_t i = 0; i < 16;) { + if (local_offset == line_length) { + dst[out_pos++] = '\n'; + local_offset = 0; + } + dst[out_pos++] = buffer[i++]; + local_offset++; + } + offset = local_offset; + return out_pos; + } +} + +template +size_t encode_base64_impl(char *dst, const char *src, size_t srclen, + base64_options options, + size_t line_length = simdutf::default_line_length) { + size_t offset = 0; + if (line_length < 4) { + line_length = 4; // We do not support line_length less than 4 + } // credit: Wojciech Muła uint8_t *out = (uint8_t *)dst; constexpr static uint8_t source_table[64] = { @@ -12262,8 +10981,40 @@ size_t encode_base64(char *dst, const char *src, size_t srclen, result.val[1] = vqtbl4q_u8(table, result.val[1]); result.val[2] = vqtbl4q_u8(table, result.val[2]); result.val[3] = vqtbl4q_u8(table, result.val[3]); - vst4q_u8(out, result); - out += 64; + if (insert_line_feeds) { + if (line_length >= 64) { // fast path + vst4q_u8(out, result); + if (offset + 64 > line_length) { + size_t location_end = line_length - offset; + size_t to_move = 64 - location_end; + std::memmove(out + location_end + 1, out + location_end, to_move); + out[location_end] = '\n'; + offset = to_move; + out += 64 + 1; + } else { + offset += 64; + out += 64; + } + } else { // slow path + uint8x16x2_t Z0 = vzipq_u8(result.val[0], result.val[1]); + uint8x16x2_t Z1 = vzipq_u8(result.val[2], result.val[3]); + uint16x8x2_t Z2 = vzipq_u16(vreinterpretq_u16_u8(Z0.val[0]), + vreinterpretq_u16_u8(Z1.val[0])); + uint16x8x2_t Z3 = vzipq_u16(vreinterpretq_u16_u8(Z0.val[1]), + vreinterpretq_u16_u8(Z1.val[1])); + uint8x16_t T0 = vreinterpretq_u8_u16(Z2.val[0]); + uint8x16_t T1 = vreinterpretq_u8_u16(Z2.val[1]); + uint8x16_t T2 = vreinterpretq_u8_u16(Z3.val[0]); + uint8x16_t T3 = vreinterpretq_u8_u16(Z3.val[1]); + out += write_output_with_line_feeds(out, T0, line_length, offset); + out += write_output_with_line_feeds(out, T1, line_length, offset); + out += write_output_with_line_feeds(out, T2, line_length, offset); + out += write_output_with_line_feeds(out, T3, line_length, offset); + } + } else { + vst4q_u8(out, result); + out += 64; + } } if (i + 24 <= srclen) { @@ -12280,17 +11031,52 @@ size_t encode_base64(char *dst, const char *src, size_t srclen, result.val[1] = vqtbl4_u8(table, result.val[1]); result.val[2] = vqtbl4_u8(table, result.val[2]); result.val[3] = vqtbl4_u8(table, result.val[3]); - vst4_u8(out, result); - out += 32; + if (insert_line_feeds) { + if (line_length >= 32) { // fast path + vst4_u8(out, result); + if (offset + 32 > line_length) { + size_t location_end = line_length - offset; + size_t to_move = 32 - location_end; + std::memmove(out + location_end + 1, out + location_end, to_move); + out[location_end] = '\n'; + offset = to_move; + out += 32 + 1; + } else { + offset += 32; + out += 32; + } + } else { // slow path + uint8x8x2_t Z0 = vzip_u8(result.val[0], result.val[1]); + uint8x8x2_t Z1 = vzip_u8(result.val[2], result.val[3]); + uint16x4x2_t Z2 = vzip_u16(vreinterpret_u16_u8(Z0.val[0]), + vreinterpret_u16_u8(Z1.val[0])); + uint16x4x2_t Z3 = vzip_u16(vreinterpret_u16_u8(Z0.val[1]), + vreinterpret_u16_u8(Z1.val[1])); + uint8x8_t T0 = vreinterpret_u8_u16(Z2.val[0]); + uint8x8_t T1 = vreinterpret_u8_u16(Z2.val[1]); + uint8x8_t T2 = vreinterpret_u8_u16(Z3.val[0]); + uint8x8_t T3 = vreinterpret_u8_u16(Z3.val[1]); + uint8x16_t TT0 = vcombine_u8(T0, T1); + uint8x16_t TT1 = vcombine_u8(T2, T3); + out += write_output_with_line_feeds(out, TT0, line_length, offset); + out += write_output_with_line_feeds(out, TT1, line_length, offset); + } + } else { + vst4_u8(out, result); + out += 32; + } i += 24; } - - out += scalar::base64::tail_encode_base64((char *)out, src + i, srclen - i, - options); - + out += scalar::base64::tail_encode_base64_impl( + (char *)out, src + i, srclen - i, options, line_length, offset); return size_t((char *)out - dst); } +size_t encode_base64(char *dst, const char *src, size_t srclen, + base64_options options) { + return encode_base64_impl(dst, src, srclen, options); +} + static inline void compress(uint8x16_t data, uint16_t mask, char *output) { if (mask == 0) { vst1q_u8((uint8_t *)output, data); @@ -12746,7 +11532,9 @@ compress_decode_base64(char *dst, const char_type *src, size_t srclen, (uint32_t(uint8_t(buffer_start[2])) << 1 * 6) + (uint32_t(uint8_t(buffer_start[3])) << 0 * 6)) << 8; +#if !SIMDUTF_IS_BIG_ENDIAN triple = scalar::u32_swap_bytes(triple); +#endif std::memcpy(dst, &triple, 4); dst += 3; @@ -12758,7 +11546,9 @@ compress_decode_base64(char *dst, const char_type *src, size_t srclen, (uint32_t(uint8_t(buffer_start[2])) << 1 * 6) + (uint32_t(uint8_t(buffer_start[3])) << 0 * 6)) << 8; +#if !SIMDUTF_IS_BIG_ENDIAN triple = scalar::u32_swap_bytes(triple); +#endif std::memcpy(dst, &triple, 3); dst += 3; @@ -13057,40 +11847,6 @@ template struct buf_block_reader { size_t idx; }; -// Routines to print masks and text for debugging bitmask operations -simdutf_unused static char *format_input_text_64(const uint8_t *text) { - static char *buf = - reinterpret_cast(malloc(sizeof(simd8x64) + 1)); - for (size_t i = 0; i < sizeof(simd8x64); i++) { - buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); - } - buf[sizeof(simd8x64)] = '\0'; - return buf; -} - -// Routines to print masks and text for debugging bitmask operations -simdutf_unused static char *format_input_text(const simd8x64 &in) { - static char *buf = - reinterpret_cast(malloc(sizeof(simd8x64) + 1)); - in.store(reinterpret_cast(buf)); - for (size_t i = 0; i < sizeof(simd8x64); i++) { - if (buf[i] < ' ') { - buf[i] = '_'; - } - } - buf[sizeof(simd8x64)] = '\0'; - return buf; -} - -simdutf_unused static char *format_mask(uint64_t mask) { - static char *buf = reinterpret_cast(malloc(64 + 1)); - for (size_t i = 0; i < 64; i++) { - buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; - } - buf[64] = '\0'; - return buf; -} - template simdutf_really_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) @@ -13137,6 +11893,71 @@ simdutf_really_inline void buf_block_reader::advance() { /* end file src/generic/buf_block_reader.h */ // other functions +/* begin file src/generic/base64lengths.h */ +namespace simdutf { +namespace arm64 { +namespace { +namespace base64_lengths { + +simdutf_warn_unused size_t binary_length_from_base64(const char *input, + size_t length) { + size_t pos = 0; + size_t count = 0; + for (; pos + 64 <= length; pos += 64) { + simd8x64 block(reinterpret_cast(input + pos)); + uint64_t maybe_base64 = block.gteq(33); // >= 33 which is '!' in ASCII + count += count_ones(maybe_base64); + } + while (pos < length) { + count += (input[pos] > 0x20) ? 1 : 0; + pos++; + } + // Count padding at the end. + size_t padding = 0; + pos = length; + while (pos > 0 && padding < 2) { + char c = input[--pos]; + if (c == '=') { + padding++; + } else if (c > ' ') { + break; + } + } + return ((count - padding) * 3) / 4; +} + +simdutf_warn_unused size_t binary_length_from_base64(const char16_t *input, + size_t length) { + size_t pos = 0; + size_t count = 0; + for (; pos + 32 <= length; pos += 32) { + simd16x32 block(reinterpret_cast(input + pos)); + uint64_t maybe_base64 = block.gteq(33); // >= 33 which is '!' in ASCII + count += count_ones(maybe_base64); + } + while (pos < length) { + count += (input[pos] > 0x20) ? 1 : 0; + pos++; + } + // Count padding at the end. + size_t padding = 0; + pos = length; + while (pos > 0 && padding < 2) { + char16_t c = input[--pos]; + if (c == '=') { + padding++; + } else if (c > ' ') { + break; + } + } + return ((count - padding) * 3) / 4; +} + +} // namespace base64_lengths +} // unnamed namespace +} // namespace arm64 +} // namespace simdutf +/* end file src/generic/base64lengths.h */ // // Implementation-specific overrides @@ -13270,6 +12091,12 @@ size_t implementation::binary_to_base64(const char *input, size_t length, return encode_base64(output, input, length, options); } +size_t implementation::binary_to_base64_with_lines( + const char *input, size_t length, char *output, size_t line_length, + base64_options options) const noexcept { + return encode_base64_impl(output, input, length, options, line_length); +} + const char *implementation::find(const char *start, const char *end, char character) const noexcept { return util_find(start, end, character); @@ -13280,11 +12107,20 @@ const char16_t *implementation::find(const char16_t *start, const char16_t *end, return util_find(start, end, character); } +simdutf_warn_unused size_t implementation::binary_length_from_base64( + const char *input, size_t length) const noexcept { + return base64_lengths::binary_length_from_base64(input, length); +} + +simdutf_warn_unused size_t implementation::binary_length_from_base64( + const char16_t *input, size_t length) const noexcept { + return base64_lengths::binary_length_from_base64(input, length); +} + } // namespace arm64 } // namespace simdutf /* begin file src/simdutf/arm64/end.h */ -#undef SIMDUTF_SIMD_HAS_BYTEMASK /* end file src/simdutf/arm64/end.h */ /* end file src/arm64/implementation.cpp */ #endif @@ -13332,6 +12168,13 @@ size_t implementation::binary_to_base64(const char *input, size_t length, return scalar::base64::tail_encode_base64(output, input, length, options); } +size_t implementation::binary_to_base64_with_lines( + const char *input, size_t length, char *output, size_t line_length, + base64_options options) const noexcept { + return scalar::base64::tail_encode_base64_impl(output, input, length, + options, line_length); +} + const char *implementation::find(const char *start, const char *end, char character) const noexcept { return std::find(start, end, character); @@ -13597,9 +12440,14 @@ struct block64 { __m512i chunks[1]; }; -template -size_t encode_base64(char *dst, const char *src, size_t srclen, - base64_options options) { +template +size_t encode_base64_impl(char *dst, const char *src, size_t srclen, + base64_options options, + size_t line_length = simdutf::default_line_length) { + size_t offset = 0; + if (line_length < 4) { + line_length = 4; // We do not support line_length less than 4 + } // credit: Wojciech Muła const uint8_t *input = (const uint8_t *)src; @@ -13608,7 +12456,6 @@ size_t encode_base64(char *dst, const char *src, size_t srclen, base64_url ? "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" : "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - const __m512i shuffle_input = _mm512_setr_epi32( 0x01020001, 0x04050304, 0x07080607, 0x0a0b090a, 0x0d0e0c0d, 0x10110f10, 0x13141213, 0x16171516, 0x191a1819, 0x1c1d1b1c, 0x1f201e1f, 0x22232122, @@ -13618,17 +12465,58 @@ size_t encode_base64(char *dst, const char *src, size_t srclen, const __m512i multi_shifts = _mm512_set1_epi64(UINT64_C(0x3036242a1016040a)); size_t size = srclen; __mmask64 input_mask = 0xffffffffffff; // (1 << 48) - 1 - while (size >= 48) { + // We want that input == end_input means that we must stop. + const uint8_t *end_input = input + (size - (size % 48)); + while (input != end_input) { const __m512i v = _mm512_maskz_loadu_epi8( input_mask, reinterpret_cast(input)); const __m512i in = _mm512_permutexvar_epi8(shuffle_input, v); const __m512i indices = _mm512_multishift_epi64_epi8(multi_shifts, in); const __m512i result = _mm512_permutexvar_epi8(indices, lookup); - _mm512_storeu_si512(reinterpret_cast<__m512i *>(out), result); - out += 64; + if (use_lines) { + if (offset + 64 > line_length) { + if (line_length >= 64) { + __m512i expanded = _mm512_mask_expand_epi8( + _mm512_set1_epi8('\n'), ~(1ULL << ((line_length - offset))), + result); + _mm512_storeu_si512(reinterpret_cast<__m512i *>(out), expanded); + __m128i last_lane = + _mm512_extracti32x4_epi32(result, 3); // Lane 3 (bytes 48-63) + uint8_t last_byte = + static_cast(_mm_extract_epi8(last_lane, 15)); + out[64] = last_byte; + out += 65; + offset = 64 - (line_length - offset); + } else { // slow path + alignas(64) uint8_t local_buffer[64]; + _mm512_storeu_si512(reinterpret_cast<__m512i *>(local_buffer), + result); + size_t out_pos = 0; + size_t local_offset = offset; + for (size_t j = 0; j < 64;) { + if (local_offset == line_length) { + out[out_pos++] = '\n'; + local_offset = 0; + } + out[out_pos++] = local_buffer[j++]; + local_offset++; + } + offset = local_offset; + out += out_pos; + } + } else { + _mm512_storeu_si512(reinterpret_cast<__m512i *>(out), result); + offset += 64; + out += 64; + } + } else { + _mm512_storeu_si512(reinterpret_cast<__m512i *>(out), result); + out += 64; + } input += 48; - size -= 48; } + size = size % 48; + input_mask = ((__mmask64)1 << size) - 1; const __m512i v = _mm512_maskz_loadu_epi8( input_mask, reinterpret_cast(input)); @@ -13643,14 +12531,65 @@ size_t encode_base64(char *dst, const char *src, size_t srclen, if (!padding_needed) { output_len = non_padded_output_len; } - __mmask64 output_mask = output_len == 64 ? (__mmask64)UINT64_MAX - : ((__mmask64)1 << output_len) - 1; + // If no output, we are done. + if (output_len == 0) { + return (size_t)(out - (uint8_t *)dst); + } + __mmask64 output_mask = 0xFFFFFFFFFFFFFFFF >> (64 - output_len); __m512i result = _mm512_mask_permutexvar_epi8( _mm512_set1_epi8('='), ((__mmask64)1 << non_padded_output_len) - 1, indices, lookup); - _mm512_mask_storeu_epi8(reinterpret_cast<__m512i *>(out), output_mask, - result); - return (size_t)(out - (uint8_t *)dst) + output_len; + if (use_lines) { + if (offset + output_len > line_length) { + if (line_length >= 64) { + __m512i expanded = _mm512_mask_expand_epi8( + _mm512_set1_epi8('\n'), ~(1ULL << ((line_length - offset))), + result); + if (output_len == 64) { + _mm512_storeu_si512(reinterpret_cast<__m512i *>(out), expanded); + out += 64; + _mm512_mask_storeu_epi8(reinterpret_cast<__m512i *>(out - 63), + 1ULL << 63, result); + out++; + } else { + output_mask = 0xFFFFFFFFFFFFFFFF >> (64 - output_len - 1); + _mm512_mask_storeu_epi8(reinterpret_cast<__m512i *>(out), output_mask, + expanded); + out += output_len + 1; + } + } else { + alignas(64) uint8_t local_buffer[64]; + _mm512_storeu_si512(reinterpret_cast<__m512i *>(local_buffer), result); + size_t out_pos = 0; + size_t local_offset = offset; + for (size_t j = 0; j < output_len;) { + if (local_offset == line_length) { + out[out_pos++] = '\n'; + local_offset = 0; + } + out[out_pos++] = local_buffer[j++]; + local_offset++; + } + offset = local_offset; + out += out_pos; + } + } else { + _mm512_mask_storeu_epi8(reinterpret_cast<__m512i *>(out), output_mask, + result); + out += output_len; + } + } else { + _mm512_mask_storeu_epi8(reinterpret_cast<__m512i *>(out), output_mask, + result); + out += output_len; + } + return (size_t)(out - (uint8_t *)dst); +} + +template +size_t encode_base64(char *dst, const char *src, size_t srclen, + base64_options options) { + return encode_base64_impl(dst, src, srclen, options); } template @@ -13807,7 +12746,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, size_t full_input_length = ri.full_input_length; if (srclen == 0) { if (!ignore_garbage && padding_characters > 0) { - return {INVALID_BASE64_CHARACTER, equallocation, 0}; + return {INVALID_BASE64_CHARACTER, equallocation, 0, true}; } return {SUCCESS, full_input_length, 0}; } @@ -13905,8 +12844,8 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, // We never should have that the number of base64 characters + the // number of padding characters is more than 4. if (!ignore_garbage && (idx + padding_characters > 4)) { - return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), - size_t(dst - dstinit), true}; + return {INVALID_BASE64_CHARACTER, equallocation, size_t(dst - dstinit), + true}; } // The idea here is that in loose mode, // if there is padding at all, it must be used @@ -13916,8 +12855,8 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, last_chunk_options == last_chunk_handling_options::loose && (idx >= 2) && padding_characters > 0 && ((idx + padding_characters) & 3) != 0) { - return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), - size_t(dst - dstinit), true}; + return {INVALID_BASE64_CHARACTER, equallocation, size_t(dst - dstinit), + true}; } else // The idea here is that in strict mode, we do not want to accept // incomplete base64 chunks. So if the chunk was otherwise valid, we @@ -13928,7 +12867,8 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, // The partial chunk was at src - idx _mm512_mask_storeu_epi8((__m512i *)dst, output_mask, shuffled); dst += output_len; - return {BASE64_INPUT_REMAINDER, equallocation, size_t(dst - dstinit)}; + return {BASE64_INPUT_REMAINDER, equallocation, size_t(dst - dstinit), + true}; } else // If there is a partial chunk with insufficient padding, with // stop_before_partial, we need to just ignore it. In "only full" mode, @@ -14009,7 +12949,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, _mm512_mask_storeu_epi8((__m512i *)dst, output_mask, shuffled); dst += output_len; return {INVALID_BASE64_CHARACTER, equallocation, - size_t(dst - dstinit)}; + size_t(dst - dstinit), true}; } else { _mm512_mask_storeu_epi8((__m512i *)dst, output_mask, shuffled); dst += output_len; @@ -14020,7 +12960,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, size_t output_count = size_t(dst - dstinit); if ((output_count % 3 == 0) || ((output_count % 3) + 1 + padding_characters != 4)) { - return {INVALID_BASE64_CHARACTER, equallocation, output_count}; + return {INVALID_BASE64_CHARACTER, equallocation, output_count, true}; } } return {SUCCESS, full_input_length, size_t(dst - dstinit)}; @@ -14029,11 +12969,76 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, if (!ignore_garbage && padding_characters > 0) { if ((size_t(dst - dstinit) % 3 == 0) || ((size_t(dst - dstinit) % 3) + 1 + padding_characters != 4)) { - return {INVALID_BASE64_CHARACTER, equallocation, size_t(dst - dstinit)}; + return {INVALID_BASE64_CHARACTER, equallocation, size_t(dst - dstinit), + true}; } } return {SUCCESS, srclen, size_t(dst - dstinit)}; } + +simdutf_warn_unused size_t icelake_binary_length_from_base64(const char *input, + size_t length) { + size_t count = 0; + const char *ptr = input; + const char *end = input + length; + + __m512i spaces = _mm512_set1_epi8(0x20); + while (ptr + 64 <= end) { + __m512i data = _mm512_loadu_si512(reinterpret_cast(ptr)); + uint64_t mask = _mm512_cmpgt_epi8_mask(data, spaces); + count += count_ones(mask); + ptr += 64; + } + + while (ptr < end) { + count += (*ptr > 0x20) ? 1 : 0; + ptr++; + } + + size_t padding = 0; + size_t pos = length; + while (pos > 0 && padding < 2) { + char c = input[--pos]; + if (c == '=') { + padding++; + } else if (c > ' ') { + break; + } + } + return ((count - padding) * 3) / 4; +} + +simdutf_warn_unused size_t +icelake_binary_length_from_base64(const char16_t *input, size_t length) { + size_t count = 0; + const char16_t *ptr = input; + const char16_t *end = input + length; + + __m512i spaces = _mm512_set1_epi16(0x20); + while (ptr + 32 <= end) { + __m512i data = _mm512_loadu_si512(reinterpret_cast(ptr)); + __mmask32 mask = _mm512_cmpgt_epi16_mask(data, spaces); + count += _mm_popcnt_u32(mask); + ptr += 32; + } + + while (ptr < end) { + count += (*ptr > 0x20) ? 1 : 0; + ptr++; + } + + size_t padding = 0; + size_t pos = length; + while (pos > 0 && padding < 2) { + char16_t c = input[--pos]; + if (c == '=') { + padding++; + } else if (c > ' ') { + break; + } + } + return ((count - padding) * 3) / 4; +} /* end file src/icelake/icelake_base64.inl.cpp */ /* begin file src/icelake/icelake_find.inl.cpp */ simdutf_really_inline const char *util_find(const char *start, const char *end, @@ -14055,6 +13060,8 @@ simdutf_really_inline const char *util_find(const char *start, const char *end, __m512i data = _mm512_maskz_loadu_epi8( load_mask, reinterpret_cast(start)); __mmask64 match_mask = _mm512_cmpeq_epi8_mask(data, char_vec); + match_mask &= load_mask; // When searching for null terminators, this + // prevents false positives if (match_mask != 0) { size_t index = _tzcnt_u64(match_mask); @@ -14144,6 +13151,8 @@ simdutf_really_inline const char16_t *util_find(const char16_t *start, __m512i data = _mm512_maskz_loadu_epi16( load_mask, reinterpret_cast(start)); __mmask32 match_mask = _mm512_cmpeq_epi16_mask(data, char_vec); + match_mask &= load_mask; // When searching for null terminators, this + // prevents false positives if (match_mask != 0) { size_t index = _tzcnt_u32(match_mask); @@ -14323,6 +13332,18 @@ size_t implementation::binary_to_base64(const char *input, size_t length, } } +size_t implementation::binary_to_base64_with_lines( + const char *input, size_t length, char *output, size_t line_length, + base64_options options) const noexcept { + if (options & base64_url) { + return encode_base64_impl(output, input, length, options, + line_length); + } else { + return encode_base64_impl(output, input, length, options, + line_length); + } +} + const char *implementation::find(const char *start, const char *end, char character) const noexcept { return util_find(start, end, character); @@ -14332,6 +13353,16 @@ const char16_t *implementation::find(const char16_t *start, const char16_t *end, return util_find(start, end, character); } +simdutf_warn_unused size_t implementation::binary_length_from_base64( + const char *input, size_t length) const noexcept { + return icelake_binary_length_from_base64(input, length); +} + +simdutf_warn_unused size_t implementation::binary_length_from_base64( + const char16_t *input, size_t length) const noexcept { + return icelake_binary_length_from_base64(input, length); +} + } // namespace icelake } // namespace simdutf @@ -14410,6 +13441,7 @@ using namespace simd; template simdutf_really_inline __m256i lookup_pshufb_improved(const __m256i input) { + // Precomputed shuffle masks for K = 1 to 16 // credit: Wojciech Muła __m256i result = _mm256_subs_epu8(input, _mm256_set1_epi8(51)); const __m256i less = _mm256_cmpgt_epi8(_mm256_set1_epi8(26), input); @@ -14436,9 +13468,110 @@ simdutf_really_inline __m256i lookup_pshufb_improved(const __m256i input) { return _mm256_add_epi8(result, input); } -template -size_t encode_base64(char *dst, const char *src, size_t srclen, - base64_options options) { +simdutf_really_inline __m256i insert_line_feed32(__m256i input, int K) { + + static const uint8_t low_table[16][32] = { + {0x80, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {0, 0x80, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {0, 1, 0x80, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {0, 1, 2, 0x80, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {0, 1, 2, 3, 0x80, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {0, 1, 2, 3, 4, 0x80, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {0, 1, 2, 3, 4, 5, 0x80, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {0, 1, 2, 3, 4, 5, 6, 0x80, 7, 8, 9, 10, 11, 12, 13, 14, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {0, 1, 2, 3, 4, 5, 6, 7, 0x80, 8, 9, 10, 11, 12, 13, 14, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 0x80, 9, 10, 11, 12, 13, 14, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0x80, 10, 11, 12, 13, 14, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0x80, 11, 12, 13, 14, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0x80, 12, 13, 14, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 0x80, 13, 14, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0x80, 14, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0x80, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}}; + static const uint8_t high_table[16][32] = { + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0x80, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 0x80, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 0x80, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 0x80, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 0x80, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 0x80, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 0x80, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 0x80, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 0x80, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 0x80, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0x80, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0x80, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0x80, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 0x80, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0x80, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0x80}}; + + __m256i line_feed_vector = _mm256_set1_epi8('\n'); + if (K >= 16) { + __m256i mask = _mm256_loadu_si256((const __m256i *)high_table[K - 16]); + __m256i lf_pos = + _mm256_cmpeq_epi8(mask, _mm256_set1_epi8(static_cast(0x80))); + __m256i shuffled = _mm256_shuffle_epi8(input, mask); + __m256i result = _mm256_blendv_epi8(shuffled, line_feed_vector, lf_pos); + return result; + } + // Shift input right by 1 byte + __m256i shift = _mm256_alignr_epi8( + input, _mm256_permute2x128_si256(input, input, 0x21), 15); + + input = _mm256_blend_epi32(input, shift, 0xF0); + + __m256i mask = _mm256_loadu_si256((const __m256i *)low_table[K]); + + __m256i lf_pos = + _mm256_cmpeq_epi8(mask, _mm256_set1_epi8(static_cast(0x80))); + __m256i shuffled = _mm256_shuffle_epi8(input, mask); + + __m256i result = _mm256_blendv_epi8(shuffled, line_feed_vector, lf_pos); + return result; +} + +template +size_t +avx2_encode_base64_impl(char *dst, const char *src, size_t srclen, + base64_options options, + size_t line_length = simdutf::default_line_length) { + size_t offset = 0; + + if (line_length < 4) { + line_length = 4; // We do not support line_length less than 4 + } // credit: Wojciech Muła const uint8_t *input = (const uint8_t *)src; @@ -14504,20 +13637,123 @@ size_t encode_base64(char *dst, const char *src, size_t srclen, const __m256i input2 = _mm256_or_si256(t1_2, t3_2); const __m256i input3 = _mm256_or_si256(t1_3, t3_3); - _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), - lookup_pshufb_improved(input0)); - out += 32; + if (use_lines) { + if (line_length >= 32) { // fast path + __m256i result; + result = lookup_pshufb_improved(input0); + if (offset + 32 > line_length) { + size_t location_end = line_length - offset; + size_t to_move = 32 - location_end; + // We could do this, or extract instead. + _mm256_storeu_si256(reinterpret_cast<__m256i *>(out + 1), result); + _mm256_storeu_si256( + reinterpret_cast<__m256i *>(out), + insert_line_feed32(result, static_cast(location_end))); + offset = to_move; + out += 32 + 1; + } else { + _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), result); + offset += 32; + out += 32; + } + result = lookup_pshufb_improved(input1); + + if (offset + 32 > line_length) { + size_t location_end = line_length - offset; + size_t to_move = 32 - location_end; + + // We could do this, or extract instead. + _mm256_storeu_si256(reinterpret_cast<__m256i *>(out + 1), result); + _mm256_storeu_si256( + reinterpret_cast<__m256i *>(out), + insert_line_feed32(result, static_cast(location_end))); + // see above. + // out[32] = static_cast(_mm256_extract_epi8(result, 31)); + offset = to_move; + out += 32 + 1; + } else { - _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), - lookup_pshufb_improved(input1)); - out += 32; + _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), result); - _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), - lookup_pshufb_improved(input2)); - out += 32; - _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), - lookup_pshufb_improved(input3)); - out += 32; + offset += 32; + out += 32; + } + result = lookup_pshufb_improved(input2); + + if (offset + 32 > line_length) { + size_t location_end = line_length - offset; + size_t to_move = 32 - location_end; + + // We could do this, or extract instead. + _mm256_storeu_si256(reinterpret_cast<__m256i *>(out + 1), result); + _mm256_storeu_si256( + reinterpret_cast<__m256i *>(out), + insert_line_feed32(result, static_cast(location_end))); + // see above. + // out[32] = static_cast(_mm256_extract_epi8(result, 31)); + offset = to_move; + out += 32 + 1; + } else { + _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), result); + offset += 32; + out += 32; + } + result = lookup_pshufb_improved(input3); + + if (offset + 32 > line_length) { + size_t location_end = line_length - offset; + size_t to_move = 32 - location_end; + + // We could do this, or extract instead. + _mm256_storeu_si256(reinterpret_cast<__m256i *>(out + 1), result); + _mm256_storeu_si256( + reinterpret_cast<__m256i *>(out), + insert_line_feed32(result, static_cast(location_end))); + // see above. + // out[32] = static_cast(_mm256_extract_epi8(result, 31)); + offset = to_move; + out += 32 + 1; + } else { + _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), result); + offset += 32; + out += 32; + } + } else { // slow path + // could be optimized + uint8_t buffer[128]; + _mm256_storeu_si256(reinterpret_cast<__m256i *>(buffer), + lookup_pshufb_improved(input0)); + _mm256_storeu_si256(reinterpret_cast<__m256i *>(buffer + 32), + lookup_pshufb_improved(input1)); + _mm256_storeu_si256(reinterpret_cast<__m256i *>(buffer + 64), + lookup_pshufb_improved(input2)); + _mm256_storeu_si256(reinterpret_cast<__m256i *>(buffer + 96), + lookup_pshufb_improved(input3)); + size_t out_pos = 0; + size_t local_offset = offset; + for (size_t j = 0; j < 128;) { + if (local_offset == line_length) { + out[out_pos++] = '\n'; + local_offset = 0; + } + out[out_pos++] = buffer[j++]; + local_offset++; + } + offset = local_offset; + out += out_pos; + } + } else { + _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), + lookup_pshufb_improved(input0)); + _mm256_storeu_si256(reinterpret_cast<__m256i *>(out + 32), + lookup_pshufb_improved(input1)); + _mm256_storeu_si256(reinterpret_cast<__m256i *>(out + 64), + lookup_pshufb_improved(input2)); + _mm256_storeu_si256(reinterpret_cast<__m256i *>(out + 96), + lookup_pshufb_improved(input3)); + + out += 128; + } } for (; i + 28 <= srclen; i += 24) { // lo = [xxxx|DDDC|CCBB|BAAA] @@ -14539,12 +13775,57 @@ size_t encode_base64(char *dst, const char *src, size_t srclen, const __m256i t3 = _mm256_mullo_epi16(t2, _mm256_set1_epi32(0x01000010)); const __m256i indices = _mm256_or_si256(t1, t3); - _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), - lookup_pshufb_improved(indices)); - out += 32; + if (use_lines) { + if (line_length >= 32) { // fast path + _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), + lookup_pshufb_improved(indices)); + + if (offset + 32 > line_length) { + size_t location_end = line_length - offset; + size_t to_move = 32 - location_end; + std::memmove(out + location_end + 1, out + location_end, to_move); + out[location_end] = '\n'; + offset = to_move; + out += 32 + 1; + } else { + offset += 32; + out += 32; + } + } else { // slow path + // could be optimized + alignas(32) uint8_t buffer[32]; + _mm256_storeu_si256(reinterpret_cast<__m256i *>(buffer), + lookup_pshufb_improved(indices)); + std::memcpy(out, buffer, 32); + size_t out_pos = 0; + size_t local_offset = offset; + for (size_t j = 0; j < 32;) { + if (local_offset == line_length) { + out[out_pos++] = '\n'; + local_offset = 0; + } + out[out_pos++] = buffer[j++]; + local_offset++; + } + offset = local_offset; + out += out_pos; + } + } else { + _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), + lookup_pshufb_improved(indices)); + + out += 32; + } } - return i / 3 * 4 + scalar::base64::tail_encode_base64((char *)out, src + i, - srclen - i, options); + return ((char *)out - (char *)dst) + + scalar::base64::tail_encode_base64_impl( + (char *)out, src + i, srclen - i, options, line_length, offset); +} + +template +size_t encode_base64(char *dst, const char *src, size_t srclen, + base64_options options) { + return avx2_encode_base64_impl(dst, src, srclen, options); } static inline void compress(__m128i data, uint16_t mask, char *output) { @@ -14622,7 +13903,7 @@ simdutf_really_inline void base64_decode_block_safe(char *out, const char *src) { base64_decode(out, _mm256_loadu_si256(reinterpret_cast(src))); - char buffer[32]; // We enforce safety with a buffer. + alignas(32) char buffer[32]; // We enforce safety with a buffer. base64_decode( buffer, _mm256_loadu_si256(reinterpret_cast(src + 32))); std::memcpy(out + 24, buffer, 24); @@ -14674,7 +13955,7 @@ class block64 { simdutf_really_inline void base64_decode_block_safe(char *out) { base64_decode(out, chunks[0]); - char buffer[32]; // We enforce safety with a buffer. + alignas(32) char buffer[32]; // We enforce safety with a buffer. base64_decode(buffer, chunks[1]); std::memcpy(out + 24, buffer, 24); } @@ -14900,6 +14181,73 @@ class block64 { return 63; } }; + +simdutf_warn_unused size_t avx2_binary_length_from_base64(const char *input, + size_t length) { + size_t count = 0; + const char *ptr = input; + const char *end = input + length; + + __m256i spaces = _mm256_set1_epi8(0x20); + while (ptr + 32 <= end) { + __m256i data = _mm256_loadu_si256(reinterpret_cast(ptr)); + __m256i gt_space = _mm256_cmpgt_epi8(data, spaces); + uint32_t mask = static_cast(_mm256_movemask_epi8(gt_space)); + count += count_ones(mask); + ptr += 32; + } + + while (ptr < end) { + count += (*ptr > 0x20) ? 1 : 0; + ptr++; + } + + size_t padding = 0; + size_t pos = length; + while (pos > 0 && padding < 2) { + char c = input[--pos]; + if (c == '=') { + padding++; + } else if (c > ' ') { + break; + } + } + return ((count - padding) * 3) / 4; +} + +simdutf_warn_unused size_t avx2_binary_length_from_base64(const char16_t *input, + size_t length) { + size_t count = 0; + const char16_t *ptr = input; + const char16_t *end = input + length; + + __m256i spaces = _mm256_set1_epi16(0x20); + while (ptr + 16 <= end) { + __m256i data = _mm256_loadu_si256(reinterpret_cast(ptr)); + __m256i gt_space = _mm256_cmpgt_epi16(data, spaces); + uint32_t mask = static_cast(_mm256_movemask_epi8(gt_space)); + count += count_ones(mask); + ptr += 16; + } + count /= 2; + + while (ptr < end) { + count += (*ptr > 0x20) ? 1 : 0; + ptr++; + } + + size_t padding = 0; + size_t pos = length; + while (pos > 0 && padding < 2) { + char16_t c = input[--pos]; + if (c == '=') { + padding++; + } else if (c > ' ') { + break; + } + } + return ((count - padding) * 3) / 4; +} /* end file src/haswell/avx2_base64.cpp */ } // unnamed namespace @@ -14939,40 +14287,6 @@ template struct buf_block_reader { size_t idx; }; -// Routines to print masks and text for debugging bitmask operations -simdutf_unused static char *format_input_text_64(const uint8_t *text) { - static char *buf = - reinterpret_cast(malloc(sizeof(simd8x64) + 1)); - for (size_t i = 0; i < sizeof(simd8x64); i++) { - buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); - } - buf[sizeof(simd8x64)] = '\0'; - return buf; -} - -// Routines to print masks and text for debugging bitmask operations -simdutf_unused static char *format_input_text(const simd8x64 &in) { - static char *buf = - reinterpret_cast(malloc(sizeof(simd8x64) + 1)); - in.store(reinterpret_cast(buf)); - for (size_t i = 0; i < sizeof(simd8x64); i++) { - if (buf[i] < ' ') { - buf[i] = '_'; - } - } - buf[sizeof(simd8x64)] = '\0'; - return buf; -} - -simdutf_unused static char *format_mask(uint64_t mask) { - static char *buf = reinterpret_cast(malloc(64 + 1)); - for (size_t i = 0; i < 64; i++) { - buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; - } - buf[64] = '\0'; - return buf; -} - template simdutf_really_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) @@ -15077,7 +14391,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, size_t full_input_length = ri.full_input_length; if (srclen == 0) { if (!ignore_garbage && equalsigns > 0) { - return {INVALID_BASE64_CHARACTER, equallocation, 0}; + return {INVALID_BASE64_CHARACTER, equallocation, 0, true}; } return {SUCCESS, full_input_length, 0}; } @@ -15244,7 +14558,8 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, if (!ignore_garbage && equalsigns > 0) { if ((size_t(dst - dstinit) % 3 == 0) || ((size_t(dst - dstinit) % 3) + 1 + equalsigns != 4)) { - return {INVALID_BASE64_CHARACTER, equallocation, size_t(dst - dstinit)}; + return {INVALID_BASE64_CHARACTER, equallocation, size_t(dst - dstinit), + true}; } } return {SUCCESS, srclen, size_t(dst - dstinit)}; @@ -15460,9 +14775,21 @@ size_t implementation::binary_to_base64(const char *input, size_t length, char *output, base64_options options) const noexcept { if (options & base64_url) { - return encode_base64(output, input, length, options); + return encode_base64(output, input, length, options); + } else { + return encode_base64(output, input, length, options); + } +} + +size_t implementation::binary_to_base64_with_lines( + const char *input, size_t length, char *output, size_t line_length, + base64_options options) const noexcept { + if (options & base64_url) { + return avx2_encode_base64_impl(output, input, length, options, + line_length); } else { - return encode_base64(output, input, length, options); + return avx2_encode_base64_impl(output, input, length, options, + line_length); } } @@ -15476,6 +14803,16 @@ const char16_t *implementation::find(const char16_t *start, const char16_t *end, return util::find(start, end, character); } +simdutf_warn_unused size_t implementation::binary_length_from_base64( + const char *input, size_t length) const noexcept { + return avx2_binary_length_from_base64(input, length); +} + +simdutf_warn_unused size_t implementation::binary_length_from_base64( + const char16_t *input, size_t length) const noexcept { + return avx2_binary_length_from_base64(input, length); +} + } // namespace haswell } // namespace simdutf @@ -16551,6 +15888,307 @@ enum class ErrorReporting { none, // no error checking is done, we assume valid inputs }; +#if (SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_UTF32) && SIMDUTF_FEATURE_UTF8 +/* begin file src/ppc64/ppc64_convert_utf16_to_utf8.cpp */ +/* + The vectorized algorithm works on single SSE register i.e., it + loads eight 16-bit code units. + + We consider three cases: + 1. an input register contains no surrogates and each value + is in range 0x0000 .. 0x07ff. + 2. an input register contains no surrogates and values are + is in range 0x0000 .. 0xffff. + 3. an input register contains surrogates --- i.e. codepoints + can have 16 or 32 bits. + + Ad 1. + + When values are less than 0x0800, it means that a 16-bit code unit + can be converted into: 1) single UTF8 byte (when it is an ASCII + char) or 2) two UTF8 bytes. + + For this case we do only some shuffle to obtain these 2-byte + codes and finally compress the whole SSE register with a single + shuffle. + + We need 256-entry lookup table to get a compression pattern + and the number of output bytes in the compressed vector register. + Each entry occupies 17 bytes. + + Ad 2. + + When values fit in 16-bit code units, but are above 0x07ff, then + a single word may produce one, two or three UTF8 bytes. + + We prepare data for all these three cases in two registers. + The first register contains lower two UTF8 bytes (used in all + cases), while the second one contains just the third byte for + the three-UTF8-bytes case. + + Finally these two registers are interleaved forming eight-element + array of 32-bit values. The array spans two SSE registers. + The bytes from the registers are compressed using two shuffles. + + We need 256-entry lookup table to get a compression pattern + and the number of output bytes in the compressed vector register. + Each entry occupies 17 bytes. + + To summarize: + - We need two 256-entry tables that have 8704 bytes in total. +*/ + +// Auxiliary procedure used by UTF-16 and UTF-32 into UTF-8. +// Note the pointer is passed by reference, it is updated by the procedure. +template +simdutf_really_inline void ppc64_convert_utf16_to_1_2_3_bytes_of_utf8( + const vector_u16 in, uint16_t one_byte_bitmask, + const T one_or_two_bytes_bytemask, uint16_t one_or_two_bytes_bitmask, + char *&utf8_output) { + // case: code units from register produce either 1, 2 or 3 UTF-8 bytes +#if SIMDUTF_IS_BIG_ENDIAN + const auto dup_lsb = + vector_u8(1, 1, 3, 3, 5, 5, 7, 7, 9, 9, 11, 11, 13, 13, 15, 15); +#else + const auto dup_lsb = + vector_u8(0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14); +#endif // SIMDUTF_IS_BIG_ENDIAN + + /* In this branch we handle three cases: + 1. [0000|0000|0ccc|cccc] => [0ccc|cccc] - + single UFT-8 byte + 2. [0000|0bbb|bbcc|cccc] => [110b|bbbb], [10cc|cccc] - two + UTF-8 bytes + 3. [aaaa|bbbb|bbcc|cccc] => [1110|aaaa], [10bb|bbbb], [10cc|cccc] - + three UTF-8 bytes + + We expand the input word (16-bit) into two code units (32-bit), thus + we have room for four bytes. However, we need five distinct bit + layouts. Note that the last byte in cases #2 and #3 is the same. + + We precompute byte 1 for case #1 and the common byte for cases #2 & #3 + in register t2. + + We precompute byte 1 for case #3 and -- **conditionally** -- precompute + either byte 1 for case #2 or byte 2 for case #3. Note that they + differ by exactly one bit. + + Finally from these two code units we build proper UTF-8 sequence, taking + into account the case (i.e, the number of bytes to write). + */ + /** + * Given [aaaa|bbbb|bbcc|cccc] our goal is to produce: + * t2 => [0ccc|cccc] [10cc|cccc] + * s4 => [1110|aaaa] ([110b|bbbb] OR [10bb|bbbb]) + */ + // [aaaa|bbbb|bbcc|cccc] => [bbcc|cccc|bbcc|cccc] + const auto t0 = as_vector_u16(dup_lsb.lookup_16(as_vector_u8(in))); + + // [bbcc|cccc|bbcc|cccc] => [00cc|cccc|0bcc|cccc] + const auto t1 = t0 & uint16_t(0b0011111101111111); + // [00cc|cccc|0bcc|cccc] => [10cc|cccc|0bcc|cccc] + const auto t2 = t1 | uint16_t(0b1000000000000000); + + // in = [aaaa|bbbb|bbcc|cccc] + // a0 = [0000|0000|0000|aaaa] + const auto a0 = in.shr<12>(); + // b0 = [aabb|bbbb|cccc|cc00] + const auto b0 = in.shl<2>(); + // s0 = [00bb|bbbb|00cc|cccc] + const auto s0 = select(uint16_t(0x3f00), b0, a0); + + // s3 = [11bb|bbbb|1110|aaaa] + const auto s3 = s0 | uint16_t(0b1100000011100000); + + const auto m0 = + ~as_vector_u16(one_or_two_bytes_bytemask) & uint16_t(0b0100000000000000); + const auto s4 = s3 ^ m0; + + // 4. compress 32-bit code units into 1, 2 or 3 bytes -- 2 x shuffle + const uint16_t mask = + (one_byte_bitmask & 0x5555) | (one_or_two_bytes_bitmask & 0xaaaa); + if (mask == 0) { + // We only have three-byte code units. Use fast path. +#if SIMDUTF_IS_BIG_ENDIAN + // Lookups produced by scripts/ppc64_convert_utf16_to_utf8.py + const auto shuffle0 = + vector_u8(1, 0, 16, 3, 2, 18, 5, 4, 20, 7, 6, 22, 9, 8, 24, 11); + const auto shuffle1 = vector_u8(10, 26, 13, 12, 28, 15, 14, 30, -1, -1, -1, + -1, -1, -1, -1, -1); +#else + const auto shuffle0 = + vector_u8(0, 1, 17, 2, 3, 19, 4, 5, 21, 6, 7, 23, 8, 9, 25, 10); + const auto shuffle1 = vector_u8(11, 27, 12, 13, 29, 14, 15, 31, -1, -1, -1, + -1, -1, -1, -1, -1); +#endif // SIMDUTF_IS_BIG_ENDIAN + const auto utf8_0 = shuffle0.lookup_32(as_vector_u8(s4), as_vector_u8(t2)); + const auto utf8_1 = shuffle1.lookup_32(as_vector_u8(s4), as_vector_u8(t2)); + + utf8_0.store(utf8_output); + utf8_output += 16; + utf8_1.store(utf8_output); + utf8_output += 8; + return; + } + + const uint8_t mask0 = uint8_t(mask); + + const uint8_t *row0 = + &simdutf::tables::ppc64_utf16_to_utf8::pack_1_2_3_utf8_bytes[mask0][0]; + const auto shuffle0 = vector_u8::load(row0 + 1); + + const auto utf8_0 = shuffle0.lookup_32(as_vector_u8(s4), as_vector_u8(t2)); + const uint8_t mask1 = static_cast(mask >> 8); + + const uint8_t *row1 = + &simdutf::tables::ppc64_utf16_to_utf8::pack_1_2_3_utf8_bytes[mask1][0]; + const auto shuffle1 = vector_u8::load(row1 + 1) + uint8_t(8); + const auto utf8_1 = shuffle1.lookup_32(as_vector_u8(s4), as_vector_u8(t2)); + + utf8_0.store(utf8_output); + utf8_output += row0[0]; + utf8_1.store(utf8_output); + utf8_output += row1[0]; +} + +struct utf16_to_utf8_t { + error_code err; + const char16_t *input; + char *output; +}; + +/* + Returns utf16_to_utf8_t value + A scalar routine should carry on the conversion of the tail, + iff there was no error. +*/ +template +utf16_to_utf8_t ppc64_convert_utf16_to_utf8(const char16_t *buf, size_t len, + char *utf8_output) { + + const char16_t *end = buf + len; + + const auto v_f800 = vector_u16(0xf800); + const auto v_d800 = vector_u16(0xd800); + const size_t safety_margin = + 12; // to avoid overruns, see issue + // https://github.com/simdutf/simdutf/issues/92 + + while (end - buf >= std::ptrdiff_t(16 + safety_margin)) { + auto in = vector_u16::load(buf); + if (not match_system(big_endian)) { + in = in.swap_bytes(); + } + // a single 16-bit UTF-16 word can yield 1, 2 or 3 UTF-8 bytes + if (in.is_ascii()) { + auto nextin = vector_u16::load(buf + vector_u16::ELEMENTS); + if (not match_system(big_endian)) { + nextin = nextin.swap_bytes(); + } + + if (nextin.is_ascii()) { + // 1. pack the bytes + const auto utf8_packed = vector_u16::pack(in, nextin); + // 2. store (16 bytes) + utf8_packed.store(utf8_output); + // 3. adjust pointers + buf += 16; + utf8_output += 16; + continue; // we are done for this round! + } + + // next block is not ASCII + const auto utf8_packed = vector_u16::pack(in, in); + // 2. store (16 bytes) + utf8_packed.store(utf8_output); + // 3. adjust pointers + buf += 8; + utf8_output += 8; + in = nextin; + // fallback + } + + // no bits set above 7th bit + const auto one_byte_bytemask = in < uint16_t(1 << 7); + const uint16_t one_byte_bitmask = one_byte_bytemask.to_bitmask(); + + // no bits set above 11th bit + const auto one_or_two_bytes_bytemask = in < uint16_t(1 << 11); + const uint16_t one_or_two_bytes_bitmask = + one_or_two_bytes_bytemask.to_bitmask(); + + if (one_or_two_bytes_bitmask == 0xffff) { + write_v_u16_11bits_to_utf8( + in, utf8_output, as_vector_u8(one_byte_bytemask), one_byte_bitmask); + buf += 8; + continue; + } + + // 1. Check if there are any surrogate word in the input chunk. + // We have also to deal with situation when there is a surrogate word + // at the end of a chunk. + const auto surrogates_bytemask = (in & v_f800) == v_d800; + + // bitmask = 0x0000 if there are no surrogates + // = 0xc000 if the last word is a surrogate + const uint16_t surrogates_bitmask = surrogates_bytemask.to_bitmask(); + // It might seem like checking for surrogates_bitmask == 0xc000 could help. + // However, it is likely an uncommon occurrence. + if (surrogates_bitmask == 0x0000) { + ppc64_convert_utf16_to_1_2_3_bytes_of_utf8( + in, one_byte_bitmask, one_or_two_bytes_bytemask, + one_or_two_bytes_bitmask, utf8_output); + + buf += 8; + // surrogate pair(s) in a register + } else { + // Let us do a scalar fallback. + // It may seem wasteful to use scalar code, but being efficient with SIMD + // in the presence of surrogate pairs may require non-trivial tables. + size_t forward = 15; + size_t k = 0; + if (size_t(end - buf) < forward + 1) { + forward = size_t(end - buf - 1); + } + for (; k < forward; k++) { + uint16_t word = scalar::utf16::swap_if_needed(buf[k]); + if ((word & 0xFF80) == 0) { + *utf8_output++ = uint8_t(word); + } else if ((word & 0xF800) == 0) { + *utf8_output++ = uint8_t((word >> 6) | 0b11000000); + *utf8_output++ = uint8_t((word & 0b111111) | 0b10000000); + } else if ((word & 0xF800) != 0xD800) { + *utf8_output++ = uint8_t((word >> 12) | 0b11100000); + *utf8_output++ = uint8_t(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = uint8_t((word & 0b111111) | 0b10000000); + } else { + // must be a surrogate pair + uint16_t diff = uint16_t(word - 0xD800); + uint16_t next_word = + scalar::utf16::swap_if_needed(buf[k + 1]); + k++; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if ((diff | diff2) > 0x3FF) { + return utf16_to_utf8_t{error_code::SURROGATE, buf + k - 1, + utf8_output}; + } + uint32_t value = (diff << 10) + diff2 + 0x10000; + *utf8_output++ = uint8_t((value >> 18) | 0b11110000); + *utf8_output++ = uint8_t(((value >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = uint8_t(((value >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = uint8_t((value & 0b111111) | 0b10000000); + } + } + buf += k; + } + } // while + + return utf16_to_utf8_t{error_code::SUCCESS, buf, utf8_output}; +} +/* end file src/ppc64/ppc64_convert_utf16_to_utf8.cpp */ +#endif // (SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_UTF32) && + // SIMDUTF_FEATURE_UTF8 + /* begin file src/ppc64/ppc64_base64.cpp */ /* * References and further reading: @@ -16580,9 +16218,9 @@ enum class ErrorReporting { * https://www.codeproject.com/Articles/276993/Base-Encoding-on-a-GPU. (2013). * * AMD XOP specific: http://0x80.pl/notesen/2016-01-12-sse-base64-encoding.html - * Altivec has capabilites of AMD XOP (or vice versa): shuffle using 2 vectors + * Altivec has capabilities of AMD XOP (or vice versa): shuffle using 2 vectors * and variable shifts, thus this implementation shares some code solution - * (modulo intrisic function names). + * (modulo intrinsic function names). */ constexpr bool with_base64_std = false; @@ -17095,7 +16733,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, size_t full_input_length = ri.full_input_length; if (srclen == 0) { if (!ignore_garbage && equalsigns > 0) { - return {INVALID_BASE64_CHARACTER, equallocation, 0}; + return {INVALID_BASE64_CHARACTER, equallocation, 0, true}; } return {SUCCESS, full_input_length, 0}; } @@ -17262,7 +16900,8 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, if (!ignore_garbage && equalsigns > 0) { if ((size_t(dst - dstinit) % 3 == 0) || ((size_t(dst - dstinit) % 3) + 1 + equalsigns != 4)) { - return {INVALID_BASE64_CHARACTER, equallocation, size_t(dst - dstinit)}; + return {INVALID_BASE64_CHARACTER, equallocation, size_t(dst - dstinit), + true}; } } return {SUCCESS, srclen, size_t(dst - dstinit)}; @@ -17590,6 +17229,14 @@ size_t implementation::binary_to_base64(const char *input, size_t length, return encode_base64(output, input, length, options); } } + +size_t implementation::binary_to_base64_with_lines( + const char *input, size_t length, char *output, size_t line_length, + base64_options options) const noexcept { + return scalar::base64::tail_encode_base64_impl(output, input, length, + options, line_length); +} + const char *implementation::find(const char *start, const char *end, char character) const noexcept { return util::find(start, end, character); @@ -17695,8 +17342,8 @@ rvv_utf32_store_utf16_m4(uint16_t *dst, vuint32m4_t utf32, size_t vl, /* end file src/rvv/rvv_utf8_to.inl.cpp */ /* begin file src/rvv/rvv_find.cpp */ -simdutf_really_inline const char *util_find(const char *start, const char *end, - char character) noexcept { +const char *implementation::find(const char *start, const char *end, + char character) const noexcept { const char *src = start; for (size_t len = end - start, vl; len > 0; len -= vl, src += vl) { vl = __riscv_vsetvl_e8m8(len); @@ -17709,9 +17356,8 @@ simdutf_really_inline const char *util_find(const char *start, const char *end, return end; } -simdutf_really_inline const char16_t *util_find(const char16_t *start, - const char16_t *end, - char16_t character) noexcept { +const char16_t *implementation::find(const char16_t *start, const char16_t *end, + char16_t character) const noexcept { const char16_t *src = start; for (size_t len = end - start, vl; len > 0; len -= vl, src += vl) { vl = __riscv_vsetvl_e16m8(len); @@ -17759,14 +17405,11 @@ size_t implementation::binary_to_base64(const char *input, size_t length, return scalar::base64::tail_encode_base64(output, input, length, options); } -const char *implementation::find(const char *start, const char *end, - char character) const noexcept { - return util_find(start, end, character); -} - -const char16_t *implementation::find(const char16_t *start, const char16_t *end, - char16_t character) const noexcept { - return util_find(start, end, character); +size_t implementation::binary_to_base64_with_lines( + const char *input, size_t length, char *output, size_t line_length, + base64_options options) const noexcept { + return scalar::base64::tail_encode_base64_impl(output, input, length, + options, line_length); } } // namespace rvv @@ -17834,7 +17477,6 @@ using namespace simd; */ // --- encoding ---------------------------------------------------- - template __m128i lookup_pshufb_improved(const __m128i input) { // credit: Wojciech Muła // reduce 0..51 -> 0 @@ -17866,9 +17508,47 @@ template __m128i lookup_pshufb_improved(const __m128i input) { return _mm_add_epi8(result, input); } -template -size_t encode_base64(char *dst, const char *src, size_t srclen, - base64_options options) { +inline __m128i insert_line_feed16(__m128i input, size_t K) { + static const uint8_t shuffle_masks[16][16] = { + {0x80, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 0x80, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 0x80, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 0x80, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 0x80, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 0x80, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 0x80, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 0x80, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 0x80, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 0x80, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0x80, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0x80, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0x80, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 0x80, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0x80, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0x80}}; + // Prepare a vector with '\n' (0x0A) + __m128i line_feed_vector = _mm_set1_epi8('\n'); + + // Load the precomputed shuffle mask for K (index K-1) + __m128i mask = _mm_loadu_si128((__m128i *)shuffle_masks[K]); + __m128i lf_pos = _mm_cmpeq_epi8(mask, _mm_set1_epi8(static_cast(0x80))); + + // Perform the shuffle to reposition the K bytes + __m128i shuffled = _mm_shuffle_epi8(input, mask); + + // Blend with line_feed_vector to insert '\n' at the appropriate positions + __m128i result = _mm_blendv_epi8(shuffled, line_feed_vector, lf_pos); + + return result; +} +template +size_t encode_base64_impl(char *dst, const char *src, size_t srclen, + base64_options options, + size_t line_length = simdutf::default_line_length) { + size_t offset = 0; + if (line_length < 4) { + line_length = 4; // We do not support line_length less than 4 + } // credit: Wojciech Muła // SSE (lookup: pshufb improved unrolled) const uint8_t *input = (const uint8_t *)src; @@ -17918,21 +17598,98 @@ size_t encode_base64(char *dst, const char *src, size_t srclen, const __m128i input2 = _mm_or_si128(t1_2, t3_2); const __m128i input3 = _mm_or_si128(t1_3, t3_3); - _mm_storeu_si128(reinterpret_cast<__m128i *>(out), - lookup_pshufb_improved(input0)); - out += 16; - - _mm_storeu_si128(reinterpret_cast<__m128i *>(out), - lookup_pshufb_improved(input1)); - out += 16; - - _mm_storeu_si128(reinterpret_cast<__m128i *>(out), - lookup_pshufb_improved(input2)); - out += 16; + const __m128i t0 = lookup_pshufb_improved(input0); + const __m128i t1 = lookup_pshufb_improved(input1); + const __m128i t2 = lookup_pshufb_improved(input2); + const __m128i t3 = lookup_pshufb_improved(input3); + + if (use_lines) { + if (line_length >= 64) { // fast path + if (offset + 64 > line_length) { + size_t location_end = line_length - offset; + size_t to_move = 64 - location_end; + if (location_end < 16) { + // We can store or extract store. See below. + //_mm_storeu_si128(reinterpret_cast<__m128i *>(out+1), t0); + _mm_storeu_si128(reinterpret_cast<__m128i *>(out), + insert_line_feed16(t0, location_end)); + out[16] = static_cast(_mm_extract_epi8(t0, 15)); + out += 17; + } else { + _mm_storeu_si128(reinterpret_cast<__m128i *>(out), t0); + out += 16; + } + if (location_end >= 16 && location_end < 32) { + // We can store or extract store. See below. + //_mm_storeu_si128(reinterpret_cast<__m128i *>(out+1), t1); + _mm_storeu_si128(reinterpret_cast<__m128i *>(out), + insert_line_feed16(t1, location_end - 16)); + out[16] = static_cast(_mm_extract_epi8(t1, 15)); + out += 17; + } else { + _mm_storeu_si128(reinterpret_cast<__m128i *>(out), t1); + out += 16; + } + if (location_end >= 32 && location_end < 48) { + // We can store or extract store. See below. + //_mm_storeu_si128(reinterpret_cast<__m128i *>(out+1), t2); + _mm_storeu_si128(reinterpret_cast<__m128i *>(out), + insert_line_feed16(t2, location_end - 32)); + out[16] = static_cast(_mm_extract_epi8(t2, 15)); + out += 17; + } else { + _mm_storeu_si128(reinterpret_cast<__m128i *>(out), t2); + out += 16; + } + if (location_end >= 48) { + // We can store or extract store. See below. + //_mm_storeu_si128(reinterpret_cast<__m128i *>(out+1), t3); + _mm_storeu_si128(reinterpret_cast<__m128i *>(out), + insert_line_feed16(t3, location_end - 48)); + out[16] = static_cast(_mm_extract_epi8(t3, 15)); + out += 17; + } else { + _mm_storeu_si128(reinterpret_cast<__m128i *>(out), t3); + out += 16; + } + offset = to_move; + } else { - _mm_storeu_si128(reinterpret_cast<__m128i *>(out), - lookup_pshufb_improved(input3)); - out += 16; + _mm_storeu_si128(reinterpret_cast<__m128i *>(out), t0); + _mm_storeu_si128(reinterpret_cast<__m128i *>(out + 16), t1); + _mm_storeu_si128(reinterpret_cast<__m128i *>(out + 32), t2); + _mm_storeu_si128(reinterpret_cast<__m128i *>(out + 48), t3); + offset += 64; + out += 64; + } + } else { // slow path + // could be optimized + alignas(64) uint8_t buffer[64]; + _mm_storeu_si128(reinterpret_cast<__m128i *>(buffer), t0); + _mm_storeu_si128(reinterpret_cast<__m128i *>(buffer + 16), t1); + _mm_storeu_si128(reinterpret_cast<__m128i *>(buffer + 32), t2); + _mm_storeu_si128(reinterpret_cast<__m128i *>(buffer + 48), t3); + std::memcpy(out, buffer, 64); + size_t out_pos = 0; + size_t local_offset = offset; + for (size_t j = 0; j < 64;) { + if (local_offset == line_length) { + out[out_pos++] = '\n'; + local_offset = 0; + } + out[out_pos++] = buffer[j++]; + local_offset++; + } + offset = local_offset; + out += out_pos; + } + } else { + _mm_storeu_si128(reinterpret_cast<__m128i *>(out), t0); + _mm_storeu_si128(reinterpret_cast<__m128i *>(out + 16), t1); + _mm_storeu_si128(reinterpret_cast<__m128i *>(out + 32), t2); + _mm_storeu_si128(reinterpret_cast<__m128i *>(out + 48), t3); + out += 64; + } } for (; i + 16 <= srclen; i += 12) { @@ -17970,13 +17727,55 @@ size_t encode_base64(char *dst, const char *src, size_t srclen, // res = [00dddddd|00cccccc|00bbbbbb|00aaaaaa] = t1 | t3 const __m128i indices = _mm_or_si128(t1, t3); - _mm_storeu_si128(reinterpret_cast<__m128i *>(out), - lookup_pshufb_improved(indices)); - out += 16; + const __m128i T0 = lookup_pshufb_improved(indices); + + _mm_storeu_si128(reinterpret_cast<__m128i *>(out), T0); + + if (use_lines) { + if (line_length >= 16) { // fast path + _mm_storeu_si128(reinterpret_cast<__m128i *>(out), T0); + if (offset + 16 > line_length) { + size_t location_end = line_length - offset; + size_t to_move = 16 - location_end; + std::memmove(out + location_end + 1, out + location_end, to_move); + out[location_end] = '\n'; + offset = to_move; + out += 16 + 1; + } else { + offset += 16; + out += 16; + } + } else { // slow path + // could be optimized + uint8_t buffer[16]; + _mm_storeu_si128(reinterpret_cast<__m128i *>(buffer), T0); + size_t out_pos = 0; + size_t local_offset = offset; + for (size_t j = 0; j < 16;) { + if (local_offset == line_length) { + out[out_pos++] = '\n'; + local_offset = 0; + } + out[out_pos++] = buffer[j++]; + local_offset++; + } + offset = local_offset; + out += out_pos; + } + } else { + _mm_storeu_si128(reinterpret_cast<__m128i *>(out), T0); + out += 16; + } } + return ((char *)out - (char *)dst) + + scalar::base64::tail_encode_base64_impl( + (char *)out, src + i, srclen - i, options, line_length, offset); +} - return i / 3 * 4 + scalar::base64::tail_encode_base64((char *)out, src + i, - srclen - i, options); +template +size_t encode_base64(char *dst, const char *src, size_t srclen, + base64_options options) { + return encode_base64_impl(dst, src, srclen, options); } // --- decoding ----------------------------------------------- @@ -18360,40 +18159,6 @@ template struct buf_block_reader { size_t idx; }; -// Routines to print masks and text for debugging bitmask operations -simdutf_unused static char *format_input_text_64(const uint8_t *text) { - static char *buf = - reinterpret_cast(malloc(sizeof(simd8x64) + 1)); - for (size_t i = 0; i < sizeof(simd8x64); i++) { - buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); - } - buf[sizeof(simd8x64)] = '\0'; - return buf; -} - -// Routines to print masks and text for debugging bitmask operations -simdutf_unused static char *format_input_text(const simd8x64 &in) { - static char *buf = - reinterpret_cast(malloc(sizeof(simd8x64) + 1)); - in.store(reinterpret_cast(buf)); - for (size_t i = 0; i < sizeof(simd8x64); i++) { - if (buf[i] < ' ') { - buf[i] = '_'; - } - } - buf[sizeof(simd8x64)] = '\0'; - return buf; -} - -simdutf_unused static char *format_mask(uint64_t mask) { - static char *buf = reinterpret_cast(malloc(64 + 1)); - for (size_t i = 0; i < 64; i++) { - buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; - } - buf[64] = '\0'; - return buf; -} - template simdutf_really_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) @@ -18496,7 +18261,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, size_t full_input_length = ri.full_input_length; if (srclen == 0) { if (!ignore_garbage && equalsigns > 0) { - return {INVALID_BASE64_CHARACTER, equallocation, 0}; + return {INVALID_BASE64_CHARACTER, equallocation, 0, true}; } return {SUCCESS, full_input_length, 0}; } @@ -18663,7 +18428,8 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, if (!ignore_garbage && equalsigns > 0) { if ((size_t(dst - dstinit) % 3 == 0) || ((size_t(dst - dstinit) % 3) + 1 + equalsigns != 4)) { - return {INVALID_BASE64_CHARACTER, equallocation, size_t(dst - dstinit)}; + return {INVALID_BASE64_CHARACTER, equallocation, size_t(dst - dstinit), + true}; } } return {SUCCESS, srclen, size_t(dst - dstinit)}; @@ -18751,6 +18517,71 @@ find(const char16_t *start, const char16_t *end, char16_t character) noexcept { } // namespace westmere } // namespace simdutf /* end file src/generic/find.h */ +/* begin file src/generic/base64lengths.h */ +namespace simdutf { +namespace westmere { +namespace { +namespace base64_lengths { + +simdutf_warn_unused size_t binary_length_from_base64(const char *input, + size_t length) { + size_t pos = 0; + size_t count = 0; + for (; pos + 64 <= length; pos += 64) { + simd8x64 block(reinterpret_cast(input + pos)); + uint64_t maybe_base64 = block.gteq(33); // >= 33 which is '!' in ASCII + count += count_ones(maybe_base64); + } + while (pos < length) { + count += (input[pos] > 0x20) ? 1 : 0; + pos++; + } + // Count padding at the end. + size_t padding = 0; + pos = length; + while (pos > 0 && padding < 2) { + char c = input[--pos]; + if (c == '=') { + padding++; + } else if (c > ' ') { + break; + } + } + return ((count - padding) * 3) / 4; +} + +simdutf_warn_unused size_t binary_length_from_base64(const char16_t *input, + size_t length) { + size_t pos = 0; + size_t count = 0; + for (; pos + 32 <= length; pos += 32) { + simd16x32 block(reinterpret_cast(input + pos)); + uint64_t maybe_base64 = block.gteq(33); // >= 33 which is '!' in ASCII + count += count_ones(maybe_base64); + } + while (pos < length) { + count += (input[pos] > 0x20) ? 1 : 0; + pos++; + } + // Count padding at the end. + size_t padding = 0; + pos = length; + while (pos > 0 && padding < 2) { + char16_t c = input[--pos]; + if (c == '=') { + padding++; + } else if (c > ' ') { + break; + } + } + return ((count - padding) * 3) / 4; +} + +} // namespace base64_lengths +} // unnamed namespace +} // namespace westmere +} // namespace simdutf +/* end file src/generic/base64lengths.h */ // // Implementation-specific overrides @@ -18889,6 +18720,19 @@ size_t implementation::binary_to_base64(const char *input, size_t length, } } +size_t implementation::binary_to_base64_with_lines( + const char *input, size_t length, char *output, size_t line_length, + base64_options options) const noexcept { + if (options & base64_url) { + return encode_base64_impl(output, input, length, options, + line_length); + + } else { + return encode_base64_impl(output, input, length, options, + line_length); + } +} + const char *implementation::find(const char *start, const char *end, char character) const noexcept { return util::find(start, end, character); @@ -18899,6 +18743,16 @@ const char16_t *implementation::find(const char16_t *start, const char16_t *end, return util::find(start, end, character); } +simdutf_warn_unused size_t implementation::binary_length_from_base64( + const char *input, size_t length) const noexcept { + return base64_lengths::binary_length_from_base64(input, length); +} + +simdutf_warn_unused size_t implementation::binary_length_from_base64( + const char16_t *input, size_t length) const noexcept { + return base64_lengths::binary_length_from_base64(input, length); +} + } // namespace westmere } // namespace simdutf @@ -18913,22 +18767,28 @@ SIMDUTF_UNTARGET_REGION /* end file src/simdutf/westmere/end.h */ /* end file src/westmere/implementation.cpp */ #endif -#if SIMDUTF_IMPLEMENTATION_LSX -/* begin file src/lsx/implementation.cpp */ -/* begin file src/simdutf/lsx/begin.h */ -// redefining SIMDUTF_IMPLEMENTATION to "lsx" -// #define SIMDUTF_IMPLEMENTATION lsx +#if SIMDUTF_IMPLEMENTATION_LASX +/* begin file src/lasx/implementation.cpp */ +/* begin file src/simdutf/lasx/begin.h */ +// redefining SIMDUTF_IMPLEMENTATION to "lasx" +// #define SIMDUTF_IMPLEMENTATION lasx #define SIMDUTF_SIMD_HAS_UNSIGNED_CMP 1 -/* end file src/simdutf/lsx/begin.h */ + +#if SIMDUTF_CAN_ALWAYS_RUN_LASX +// nothing needed. +#else +SIMDUTF_TARGET_LASX +#endif +/* end file src/simdutf/lasx/begin.h */ namespace simdutf { -namespace lsx { +namespace lasx { namespace { -#ifndef SIMDUTF_LSX_H - #error "lsx.h must be included" +#ifndef SIMDUTF_LASX_H + #error "lasx.h must be included" #endif using namespace simd; -/* begin file src/lsx/lsx_base64.cpp */ +/* begin file src/lasx/lasx_base64.cpp */ /** * References and further reading: * @@ -18969,100 +18829,118 @@ size_t encode_base64(char *dst, const char *src, size_t srclen, : "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; uint8_t *out = (uint8_t *)dst; - v16u8 shuf; - __m128i v_fc0fc00, v_3f03f0, shift_r, shift_l, base64_tbl0, base64_tbl1, + v32u8 shuf; + __m256i v_fc0fc00, v_3f03f0, shift_r, shift_l, base64_tbl0, base64_tbl1, base64_tbl2, base64_tbl3; - if (srclen >= 16) { - shuf = v16u8{1, 0, 2, 1, 4, 3, 5, 4, 7, 6, 8, 7, 10, 9, 11, 10}; - v_fc0fc00 = __lsx_vreplgr2vr_w(uint32_t(0x0fc0fc00)); - v_3f03f0 = __lsx_vreplgr2vr_w(uint32_t(0x003f03f0)); - shift_r = __lsx_vreplgr2vr_w(uint32_t(0x0006000a)); - shift_l = __lsx_vreplgr2vr_w(uint32_t(0x00080004)); - base64_tbl0 = __lsx_vld(lookup_tbl, 0); - base64_tbl1 = __lsx_vld(lookup_tbl, 16); - base64_tbl2 = __lsx_vld(lookup_tbl, 32); - base64_tbl3 = __lsx_vld(lookup_tbl, 48); - } + if (srclen >= 28) { + shuf = v32u8{1, 0, 2, 1, 4, 3, 5, 4, 7, 6, 8, 7, 10, 9, 11, 10, + 1, 0, 2, 1, 4, 3, 5, 4, 7, 6, 8, 7, 10, 9, 11, 10}; + v_fc0fc00 = __lasx_xvreplgr2vr_w(uint32_t(0x0fc0fc00)); + v_3f03f0 = __lasx_xvreplgr2vr_w(uint32_t(0x003f03f0)); + shift_r = __lasx_xvreplgr2vr_w(uint32_t(0x0006000a)); + shift_l = __lasx_xvreplgr2vr_w(uint32_t(0x00080004)); + base64_tbl0 = ____m256i(__lsx_vld(lookup_tbl, 0)); + base64_tbl1 = ____m256i(__lsx_vld(lookup_tbl, 16)); + base64_tbl2 = ____m256i(__lsx_vld(lookup_tbl, 32)); + base64_tbl3 = ____m256i(__lsx_vld(lookup_tbl, 48)); + } size_t i = 0; - for (; i + 52 <= srclen; i += 48) { - __m128i in0 = + for (; i + 100 <= srclen; i += 96) { + __m128i in0_lo = __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 0); - __m128i in1 = + __m128i in0_hi = __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 1); - __m128i in2 = + __m128i in1_lo = __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 2); - __m128i in3 = + __m128i in1_hi = __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 3); + __m128i in2_lo = + __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 4); + __m128i in2_hi = + __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 5); + __m128i in3_lo = + __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 6); + __m128i in3_hi = + __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 7); - in0 = __lsx_vshuf_b(in0, in0, (__m128i)shuf); - in1 = __lsx_vshuf_b(in1, in1, (__m128i)shuf); - in2 = __lsx_vshuf_b(in2, in2, (__m128i)shuf); - in3 = __lsx_vshuf_b(in3, in3, (__m128i)shuf); + __m256i in0 = lasx_set_q(in0_hi, in0_lo); + __m256i in1 = lasx_set_q(in1_hi, in1_lo); + __m256i in2 = lasx_set_q(in2_hi, in2_lo); + __m256i in3 = lasx_set_q(in3_hi, in3_lo); + + in0 = __lasx_xvshuf_b(in0, in0, (__m256i)shuf); + in1 = __lasx_xvshuf_b(in1, in1, (__m256i)shuf); + in2 = __lasx_xvshuf_b(in2, in2, (__m256i)shuf); + in3 = __lasx_xvshuf_b(in3, in3, (__m256i)shuf); - __m128i t0_0 = __lsx_vand_v(in0, v_fc0fc00); - __m128i t0_1 = __lsx_vand_v(in1, v_fc0fc00); - __m128i t0_2 = __lsx_vand_v(in2, v_fc0fc00); - __m128i t0_3 = __lsx_vand_v(in3, v_fc0fc00); + __m256i t0_0 = __lasx_xvand_v(in0, v_fc0fc00); + __m256i t0_1 = __lasx_xvand_v(in1, v_fc0fc00); + __m256i t0_2 = __lasx_xvand_v(in2, v_fc0fc00); + __m256i t0_3 = __lasx_xvand_v(in3, v_fc0fc00); - __m128i t1_0 = __lsx_vsrl_h(t0_0, shift_r); - __m128i t1_1 = __lsx_vsrl_h(t0_1, shift_r); - __m128i t1_2 = __lsx_vsrl_h(t0_2, shift_r); - __m128i t1_3 = __lsx_vsrl_h(t0_3, shift_r); + __m256i t1_0 = __lasx_xvsrl_h(t0_0, shift_r); + __m256i t1_1 = __lasx_xvsrl_h(t0_1, shift_r); + __m256i t1_2 = __lasx_xvsrl_h(t0_2, shift_r); + __m256i t1_3 = __lasx_xvsrl_h(t0_3, shift_r); - __m128i t2_0 = __lsx_vand_v(in0, v_3f03f0); - __m128i t2_1 = __lsx_vand_v(in1, v_3f03f0); - __m128i t2_2 = __lsx_vand_v(in2, v_3f03f0); - __m128i t2_3 = __lsx_vand_v(in3, v_3f03f0); + __m256i t2_0 = __lasx_xvand_v(in0, v_3f03f0); + __m256i t2_1 = __lasx_xvand_v(in1, v_3f03f0); + __m256i t2_2 = __lasx_xvand_v(in2, v_3f03f0); + __m256i t2_3 = __lasx_xvand_v(in3, v_3f03f0); - __m128i t3_0 = __lsx_vsll_h(t2_0, shift_l); - __m128i t3_1 = __lsx_vsll_h(t2_1, shift_l); - __m128i t3_2 = __lsx_vsll_h(t2_2, shift_l); - __m128i t3_3 = __lsx_vsll_h(t2_3, shift_l); + __m256i t3_0 = __lasx_xvsll_h(t2_0, shift_l); + __m256i t3_1 = __lasx_xvsll_h(t2_1, shift_l); + __m256i t3_2 = __lasx_xvsll_h(t2_2, shift_l); + __m256i t3_3 = __lasx_xvsll_h(t2_3, shift_l); - __m128i input0 = __lsx_vor_v(t1_0, t3_0); - __m128i input0_shuf0 = __lsx_vshuf_b(base64_tbl1, base64_tbl0, input0); - __m128i input0_shuf1 = __lsx_vshuf_b(base64_tbl3, base64_tbl2, - __lsx_vsub_b(input0, __lsx_vldi(32))); - __m128i input0_mask = __lsx_vslei_bu(input0, 31); - __m128i input0_result = - __lsx_vbitsel_v(input0_shuf1, input0_shuf0, input0_mask); - __lsx_vst(input0_result, reinterpret_cast<__m128i *>(out), 0); - out += 16; + __m256i input0 = __lasx_xvor_v(t1_0, t3_0); + __m256i input0_shuf0 = __lasx_xvshuf_b(base64_tbl1, base64_tbl0, input0); + __m256i input0_shuf1 = __lasx_xvshuf_b( + base64_tbl3, base64_tbl2, __lasx_xvsub_b(input0, __lasx_xvldi(32))); + __m256i input0_mask = __lasx_xvslei_bu(input0, 31); + __m256i input0_result = + __lasx_xvbitsel_v(input0_shuf1, input0_shuf0, input0_mask); + __lasx_xvst(input0_result, reinterpret_cast<__m256i *>(out), 0); + out += 32; - __m128i input1 = __lsx_vor_v(t1_1, t3_1); - __m128i input1_shuf0 = __lsx_vshuf_b(base64_tbl1, base64_tbl0, input1); - __m128i input1_shuf1 = __lsx_vshuf_b(base64_tbl3, base64_tbl2, - __lsx_vsub_b(input1, __lsx_vldi(32))); - __m128i input1_mask = __lsx_vslei_bu(input1, 31); - __m128i input1_result = - __lsx_vbitsel_v(input1_shuf1, input1_shuf0, input1_mask); - __lsx_vst(input1_result, reinterpret_cast<__m128i *>(out), 0); - out += 16; + __m256i input1 = __lasx_xvor_v(t1_1, t3_1); + __m256i input1_shuf0 = __lasx_xvshuf_b(base64_tbl1, base64_tbl0, input1); + __m256i input1_shuf1 = __lasx_xvshuf_b( + base64_tbl3, base64_tbl2, __lasx_xvsub_b(input1, __lasx_xvldi(32))); + __m256i input1_mask = __lasx_xvslei_bu(input1, 31); + __m256i input1_result = + __lasx_xvbitsel_v(input1_shuf1, input1_shuf0, input1_mask); + __lasx_xvst(input1_result, reinterpret_cast<__m256i *>(out), 0); + out += 32; - __m128i input2 = __lsx_vor_v(t1_2, t3_2); - __m128i input2_shuf0 = __lsx_vshuf_b(base64_tbl1, base64_tbl0, input2); - __m128i input2_shuf1 = __lsx_vshuf_b(base64_tbl3, base64_tbl2, - __lsx_vsub_b(input2, __lsx_vldi(32))); - __m128i input2_mask = __lsx_vslei_bu(input2, 31); - __m128i input2_result = - __lsx_vbitsel_v(input2_shuf1, input2_shuf0, input2_mask); - __lsx_vst(input2_result, reinterpret_cast<__m128i *>(out), 0); - out += 16; + __m256i input2 = __lasx_xvor_v(t1_2, t3_2); + __m256i input2_shuf0 = __lasx_xvshuf_b(base64_tbl1, base64_tbl0, input2); + __m256i input2_shuf1 = __lasx_xvshuf_b( + base64_tbl3, base64_tbl2, __lasx_xvsub_b(input2, __lasx_xvldi(32))); + __m256i input2_mask = __lasx_xvslei_bu(input2, 31); + __m256i input2_result = + __lasx_xvbitsel_v(input2_shuf1, input2_shuf0, input2_mask); + __lasx_xvst(input2_result, reinterpret_cast<__m256i *>(out), 0); + out += 32; - __m128i input3 = __lsx_vor_v(t1_3, t3_3); - __m128i input3_shuf0 = __lsx_vshuf_b(base64_tbl1, base64_tbl0, input3); - __m128i input3_shuf1 = __lsx_vshuf_b(base64_tbl3, base64_tbl2, - __lsx_vsub_b(input3, __lsx_vldi(32))); - __m128i input3_mask = __lsx_vslei_bu(input3, 31); - __m128i input3_result = - __lsx_vbitsel_v(input3_shuf1, input3_shuf0, input3_mask); - __lsx_vst(input3_result, reinterpret_cast<__m128i *>(out), 0); - out += 16; + __m256i input3 = __lasx_xvor_v(t1_3, t3_3); + __m256i input3_shuf0 = __lasx_xvshuf_b(base64_tbl1, base64_tbl0, input3); + __m256i input3_shuf1 = __lasx_xvshuf_b( + base64_tbl3, base64_tbl2, __lasx_xvsub_b(input3, __lasx_xvldi(32))); + __m256i input3_mask = __lasx_xvslei_bu(input3, 31); + __m256i input3_result = + __lasx_xvbitsel_v(input3_shuf1, input3_shuf0, input3_mask); + __lasx_xvst(input3_result, reinterpret_cast<__m256i *>(out), 0); + out += 32; } - for (; i + 16 <= srclen; i += 12) { + for (; i + 28 <= srclen; i += 24) { - __m128i in = __lsx_vld(reinterpret_cast(input + i), 0); + __m128i in_lo = __lsx_vld(reinterpret_cast(input + i), 0); + __m128i in_hi = + __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 1); + + __m256i in = lasx_set_q(in_hi, in_lo); // bytes from groups A, B and C are needed in separate 32-bit lanes // in = [DDDD|CCCC|BBBB|AAAA] @@ -19076,33 +18954,32 @@ size_t encode_base64(char *dst, const char *src, size_t srclen, // [bbbbcccc|ccdddddd|aaaaaabb|bbbbcccc] // ^^^^ ^^^^^^^^ ^^^^^^^^ ^^^^ // processed bits - in = __lsx_vshuf_b(in, in, (__m128i)shuf); + in = __lasx_xvshuf_b(in, in, (__m256i)shuf); // unpacking // t0 = [0000cccc|cc000000|aaaaaa00|00000000] - __m128i t0 = __lsx_vand_v(in, v_fc0fc00); + __m256i t0 = __lasx_xvand_v(in, v_fc0fc00); // t1 = [00000000|00cccccc|00000000|00aaaaaa] // ((c >> 6), (a >> 10)) - __m128i t1 = __lsx_vsrl_h(t0, shift_r); + __m256i t1 = __lasx_xvsrl_h(t0, shift_r); // t2 = [00000000|00dddddd|000000bb|bbbb0000] - __m128i t2 = __lsx_vand_v(in, v_3f03f0); + __m256i t2 = __lasx_xvand_v(in, v_3f03f0); // t3 = [00dddddd|00000000|00bbbbbb|00000000] // ((d << 8), (b << 4)) - __m128i t3 = __lsx_vsll_h(t2, shift_l); + __m256i t3 = __lasx_xvsll_h(t2, shift_l); // res = [00dddddd|00cccccc|00bbbbbb|00aaaaaa] = t1 | t3 - __m128i indices = __lsx_vor_v(t1, t3); - - __m128i indices_shuf0 = __lsx_vshuf_b(base64_tbl1, base64_tbl0, indices); - __m128i indices_shuf1 = __lsx_vshuf_b( - base64_tbl3, base64_tbl2, __lsx_vsub_b(indices, __lsx_vldi(32))); - __m128i indices_mask = __lsx_vslei_bu(indices, 31); - __m128i indices_result = - __lsx_vbitsel_v(indices_shuf1, indices_shuf0, indices_mask); + __m256i indices = __lasx_xvor_v(t1, t3); - __lsx_vst(indices_result, reinterpret_cast<__m128i *>(out), 0); - out += 16; + __m256i indices_shuf0 = __lasx_xvshuf_b(base64_tbl1, base64_tbl0, indices); + __m256i indices_shuf1 = __lasx_xvshuf_b( + base64_tbl3, base64_tbl2, __lasx_xvsub_b(indices, __lasx_xvldi(32))); + __m256i indices_mask = __lasx_xvslei_bu(indices, 31); + __m256i indices_result = + __lasx_xvbitsel_v(indices_shuf1, indices_shuf0, indices_mask); + __lasx_xvst(indices_result, reinterpret_cast<__m256i *>(out), 0); + out += 32; } return i / 3 * 4 + scalar::base64::tail_encode_base64((char *)out, src + i, @@ -19126,7 +19003,7 @@ static inline void compress(__m128i data, uint16_t mask, char *output) { tables::base64::thintable_epi8[mask2]}; // we increment by 0x08 the second half of the mask - v4u32 hi = {0, 0, 0x08080808, 0x08080808}; + const v4u32 hi = {0, 0, 0x08080808, 0x08080808}; __m128i shufmask1 = __lsx_vadd_b((__m128i)shufmask, (__m128i)hi); // this is the version "nearly pruned" @@ -19148,115 +19025,111 @@ static inline void compress(__m128i data, uint16_t mask, char *output) { } struct block64 { - __m128i chunks[4]; + __m256i chunks[2]; }; template -static inline uint16_t to_base64_mask(__m128i *src, bool *error) { - const v16u8 ascii_space_tbl = {0x20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x9, 0xa, 0x0, 0xc, 0xd, 0x0, 0x0}; +static inline uint32_t to_base64_mask(__m256i *src, bool *error) { + __m256i ascii_space_tbl = + ____m256i((__m128i)v16u8{0x20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x9, 0xa, 0x0, 0xc, 0xd, 0x0, 0x0}); // credit: aqrit - /* - '0'(0x30)-'9'(0x39) => delta_values_index = 4 - 'A'(0x41)-'Z'(0x5a) => delta_values_index = 4/5/12(4+8) - 'a'(0x61)-'z'(0x7a) => delta_values_index = 6/7/14(6+8) - '+'(0x2b) => delta_values_index = 3 - '/'(0x2f) => delta_values_index = 2+8 = 10 - '-'(0x2d) => delta_values_index = 2+8 = 10 - '_'(0x5f) => delta_values_index = 5+8 = 13 - */ - v16u8 delta_asso; + __m256i delta_asso; if (default_or_url) { - delta_asso = v16u8{0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x16}; + delta_asso = + ____m256i((__m128i)v16u8{0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x11, 0x0, 0x16}); } else { - delta_asso = v16u8{0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, - 0x0, 0x0, 0x0, 0x0, 0x0, 0xF, 0x0, 0xF}; + delta_asso = + ____m256i((__m128i)v16u8{0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x0, + 0x0, 0x0, 0x0, 0x0, 0xF, 0x0, 0xF}); } - v16i8 delta_values; + __m256i delta_values; if (default_or_url) { - delta_values = - v16i8{int8_t(0xBF), int8_t(0xE0), int8_t(0xB9), int8_t(0x13), - int8_t(0x04), int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), - int8_t(0xB9), int8_t(0x00), int8_t(0xFF), int8_t(0x11), - int8_t(0xFF), int8_t(0xBF), int8_t(0x10), int8_t(0xB9)}; + delta_values = ____m256i( + (__m128i)v16i8{int8_t(0xBF), int8_t(0xE0), int8_t(0xB9), int8_t(0x13), + int8_t(0x04), int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), + int8_t(0xB9), int8_t(0x00), int8_t(0xFF), int8_t(0x11), + int8_t(0xFF), int8_t(0xBF), int8_t(0x10), int8_t(0xB9)}); } else if (base64_url) { - delta_values = - v16i8{int8_t(0x00), int8_t(0x00), int8_t(0x00), int8_t(0x13), - int8_t(0x04), int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), - int8_t(0xB9), int8_t(0x00), int8_t(0x11), int8_t(0xC3), - int8_t(0xBF), int8_t(0xE0), int8_t(0xB9), int8_t(0xB9)}; + delta_values = ____m256i( + (__m128i)v16i8{int8_t(0x00), int8_t(0x00), int8_t(0x00), int8_t(0x13), + int8_t(0x04), int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), + int8_t(0xB9), int8_t(0x00), int8_t(0x11), int8_t(0xC3), + int8_t(0xBF), int8_t(0xE0), int8_t(0xB9), int8_t(0xB9)}); } else { - delta_values = - v16i8{int8_t(0x00), int8_t(0x00), int8_t(0x00), int8_t(0x13), - int8_t(0x04), int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), - int8_t(0xB9), int8_t(0x00), int8_t(0x10), int8_t(0xC3), - int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), int8_t(0xB9)}; + delta_values = ____m256i( + (__m128i)v16i8{int8_t(0x00), int8_t(0x00), int8_t(0x00), int8_t(0x13), + int8_t(0x04), int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), + int8_t(0xB9), int8_t(0x00), int8_t(0x10), int8_t(0xC3), + int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), int8_t(0xB9)}); } - v16u8 check_asso; + __m256i check_asso; if (default_or_url) { - check_asso = v16u8{0x0D, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x01, 0x03, 0x07, 0x0B, 0x0E, 0x0B, 0x06}; + check_asso = ____m256i((__m128i)v16u8{0x0D, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x03, 0x07, + 0x0B, 0x0E, 0x0B, 0x06}); + } else if (base64_url) { - check_asso = v16u8{0x0D, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x01, 0x03, 0x07, 0x0B, 0x06, 0x0B, 0x12}; + check_asso = ____m256i((__m128i)v16u8{0x0D, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x03, 0x07, + 0x0B, 0x06, 0x0B, 0x12}); } else { - check_asso = v16u8{0x0D, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x01, 0x03, 0x07, 0x0B, 0x0B, 0x0B, 0x0F}; + check_asso = ____m256i((__m128i)v16u8{0x0D, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x03, 0x07, + 0x0B, 0x0B, 0x0B, 0x0F}); } - v16i8 check_values; + __m256i check_values; if (default_or_url) { - check_values = - v16i8{int8_t(0x80), int8_t(0x80), int8_t(0x80), int8_t(0x80), - int8_t(0xCF), int8_t(0xBF), int8_t(0xD5), int8_t(0xA6), - int8_t(0xB5), int8_t(0xA1), int8_t(0x00), int8_t(0x80), - int8_t(0x00), int8_t(0x80), int8_t(0x00), int8_t(0x80)}; + + check_values = ____m256i( + (__m128i)v16i8{int8_t(0x80), int8_t(0x80), int8_t(0x80), int8_t(0x80), + int8_t(0xCF), int8_t(0xBF), int8_t(0xD5), int8_t(0xA6), + int8_t(0xB5), int8_t(0xA1), int8_t(0x00), int8_t(0x80), + int8_t(0x00), int8_t(0x80), int8_t(0x00), int8_t(0x80)}); } else if (base64_url) { - check_values = v16i8{int8_t(0x0), int8_t(0x80), int8_t(0x80), int8_t(0x80), - int8_t(0xCF), int8_t(0xBF), int8_t(0xD3), int8_t(0xA6), - int8_t(0xB5), int8_t(0x86), int8_t(0xD0), int8_t(0x80), - int8_t(0xB0), int8_t(0x80), int8_t(0x0), int8_t(0x0)}; + check_values = ____m256i( + (__m128i)v16i8{int8_t(0x0), int8_t(0x80), int8_t(0x80), int8_t(0x80), + int8_t(0xCF), int8_t(0xBF), int8_t(0xD3), int8_t(0xA6), + int8_t(0xB5), int8_t(0x86), int8_t(0xD0), int8_t(0x80), + int8_t(0xB0), int8_t(0x80), int8_t(0x0), int8_t(0x0)}); } else { - check_values = - v16i8{int8_t(0x80), int8_t(0x80), int8_t(0x80), int8_t(0x80), - int8_t(0xCF), int8_t(0xBF), int8_t(0xD5), int8_t(0xA6), - int8_t(0xB5), int8_t(0x86), int8_t(0xD1), int8_t(0x80), - int8_t(0xB1), int8_t(0x80), int8_t(0x91), int8_t(0x80)}; + check_values = ____m256i( + (__m128i)v16i8{int8_t(0x80), int8_t(0x80), int8_t(0x80), int8_t(0x80), + int8_t(0xCF), int8_t(0xBF), int8_t(0xD5), int8_t(0xA6), + int8_t(0xB5), int8_t(0x86), int8_t(0xD1), int8_t(0x80), + int8_t(0xB1), int8_t(0x80), int8_t(0x91), int8_t(0x80)}); } - const __m128i shifted = __lsx_vsrli_b(*src, 3); - __m128i asso_index = __lsx_vand_v(*src, __lsx_vldi(0xF)); - const __m128i delta_hash = - __lsx_vavgr_bu(__lsx_vshuf_b((__m128i)delta_asso, (__m128i)delta_asso, - (__m128i)asso_index), - shifted); - const __m128i check_hash = - __lsx_vavgr_bu(__lsx_vshuf_b((__m128i)check_asso, (__m128i)check_asso, - (__m128i)asso_index), - shifted); + __m256i shifted = __lasx_xvsrli_b(*src, 3); + __m256i asso_index = __lasx_xvand_v(*src, __lasx_xvldi(0xF)); + __m256i delta_hash = __lasx_xvavgr_bu( + __lasx_xvshuf_b(delta_asso, delta_asso, asso_index), shifted); + __m256i check_hash = __lasx_xvavgr_bu( + __lasx_xvshuf_b(check_asso, check_asso, asso_index), shifted); - const __m128i out = - __lsx_vsadd_b(__lsx_vshuf_b((__m128i)delta_values, (__m128i)delta_values, - (__m128i)delta_hash), - *src); - const __m128i chk = - __lsx_vsadd_b(__lsx_vshuf_b((__m128i)check_values, (__m128i)check_values, - (__m128i)check_hash), - *src); - unsigned int mask = __lsx_vpickve2gr_hu(__lsx_vmskltz_b(chk), 0); + __m256i out = __lasx_xvsadd_b( + __lasx_xvshuf_b(delta_values, delta_values, delta_hash), *src); + __m256i chk = __lasx_xvsadd_b( + __lasx_xvshuf_b(check_values, check_values, check_hash), *src); + __m256i chk_ltz = __lasx_xvmskltz_b(chk); + unsigned int mask = __lasx_xvpickve2gr_wu(chk_ltz, 0); + mask = mask | (__lsx_vpickve2gr_hu(lasx_extracti128_hi(chk_ltz), 0) << 16); if (mask) { - __m128i ascii_space = __lsx_vseq_b(__lsx_vshuf_b((__m128i)ascii_space_tbl, - (__m128i)ascii_space_tbl, - (__m128i)asso_index), - *src); - *error |= - (mask != __lsx_vpickve2gr_hu(__lsx_vmskltz_b((__m128i)ascii_space), 0)); + __m256i ascii_space = __lasx_xvseq_b( + __lasx_xvshuf_b(ascii_space_tbl, ascii_space_tbl, asso_index), *src); + __m256i ascii_space_ltz = __lasx_xvmskltz_b(ascii_space); + unsigned int ascii_space_mask = __lasx_xvpickve2gr_wu(ascii_space_ltz, 0); + ascii_space_mask = + ascii_space_mask | + (__lsx_vpickve2gr_hu(lasx_extracti128_hi(ascii_space_ltz), 0) << 16); + *error |= (mask != ascii_space_mask); } *src = out; - return (uint16_t)mask; + return (uint32_t)mask; } template @@ -19266,18 +19139,12 @@ static inline uint64_t to_base64_mask(block64 *b, bool *error) { to_base64_mask(&b->chunks[0], error); uint64_t m1 = to_base64_mask(&b->chunks[1], error); - uint64_t m2 = - to_base64_mask(&b->chunks[2], error); - uint64_t m3 = - to_base64_mask(&b->chunks[3], error); - return m0 | (m1 << 16) | (m2 << 32) | (m3 << 48); + return m0 | (m1 << 32); } static inline void copy_block(block64 *b, char *output) { - __lsx_vst(b->chunks[0], reinterpret_cast<__m128i *>(output), 0); - __lsx_vst(b->chunks[1], reinterpret_cast<__m128i *>(output), 16); - __lsx_vst(b->chunks[2], reinterpret_cast<__m128i *>(output), 32); - __lsx_vst(b->chunks[3], reinterpret_cast<__m128i *>(output), 48); + __lasx_xvst(b->chunks[0], reinterpret_cast<__m256i *>(output), 0); + __lasx_xvst(b->chunks[1], reinterpret_cast<__m256i *>(output), 32); } static inline uint64_t compress_block(block64 *b, uint64_t mask, char *output) { @@ -19285,84 +19152,150 @@ static inline uint64_t compress_block(block64 *b, uint64_t mask, char *output) { uint64_t count = __lsx_vpickve2gr_d(__lsx_vpcnt_h(__lsx_vreplgr2vr_d(nmask)), 0); uint16_t *count_ptr = (uint16_t *)&count; - compress(b->chunks[0], uint16_t(mask), output); - compress(b->chunks[1], uint16_t(mask >> 16), output + count_ptr[0]); - compress(b->chunks[2], uint16_t(mask >> 32), + compress(lasx_extracti128_lo(b->chunks[0]), uint16_t(mask), output); + compress(lasx_extracti128_hi(b->chunks[0]), uint16_t(mask >> 16), + output + count_ptr[0]); + compress(lasx_extracti128_lo(b->chunks[1]), uint16_t(mask >> 32), output + count_ptr[0] + count_ptr[1]); - compress(b->chunks[3], uint16_t(mask >> 48), + compress(lasx_extracti128_hi(b->chunks[1]), uint16_t(mask >> 48), output + count_ptr[0] + count_ptr[1] + count_ptr[2]); return count_ones(nmask); } +template bool is_power_of_two(T x) { return (x & (x - 1)) == 0; } + +inline size_t compress_block_single(block64 *b, uint64_t mask, char *output) { + const size_t pos64 = trailing_zeroes(mask); + const int8_t pos = pos64 & 0xf; + + // Predefine the index vector + const v16u8 v1 = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; + + switch (pos64 >> 4) { + case 0b00: { + const __m128i lane0 = lasx_extracti128_lo(b->chunks[0]); + const __m128i lane1 = lasx_extracti128_hi(b->chunks[0]); + + const __m128i v0 = __lsx_vreplgr2vr_b((uint8_t)(pos - 1)); + const __m128i v2 = __lsx_vslt_b(v0, (__m128i)v1); // v1 > v0 + const __m128i sh = __lsx_vsub_b((__m128i)v1, v2); + const __m128i compressed = __lsx_vshuf_b(lane0, lane0, sh); + + __lsx_vst(compressed, reinterpret_cast<__m128i *>(output + 0 * 16), 0); + __lsx_vst(lane1, reinterpret_cast<__m128i *>(output + 1 * 16 - 1), 0); + __lasx_xvst(b->chunks[1], reinterpret_cast<__m256i *>(output + 2 * 16 - 1), + 0); + } break; + case 0b01: { + const __m128i lane0 = lasx_extracti128_lo(b->chunks[0]); + const __m128i lane1 = lasx_extracti128_hi(b->chunks[0]); + __lsx_vst(lane0, reinterpret_cast<__m128i *>(output + 0 * 16), 0); + + const __m128i v0 = __lsx_vreplgr2vr_b((uint8_t)(pos - 1)); + const __m128i v2 = __lsx_vslt_b(v0, (__m128i)v1); + const __m128i sh = __lsx_vsub_b((__m128i)v1, v2); + const __m128i compressed = __lsx_vshuf_b(lane1, lane1, sh); + + __lsx_vst(compressed, reinterpret_cast<__m128i *>(output + 1 * 16), 0); + __lasx_xvst(b->chunks[1], reinterpret_cast<__m256i *>(output + 2 * 16 - 1), + 0); + } break; + case 0b10: { + __lasx_xvst(b->chunks[0], reinterpret_cast<__m256i *>(output + 0 * 16), 0); + + const __m128i lane2 = lasx_extracti128_lo(b->chunks[1]); + const __m128i lane3 = lasx_extracti128_hi(b->chunks[1]); + + const __m128i v0 = __lsx_vreplgr2vr_b((uint8_t)(pos - 1)); + const __m128i v2 = __lsx_vslt_b(v0, (__m128i)v1); + const __m128i sh = __lsx_vsub_b((__m128i)v1, v2); + const __m128i compressed = __lsx_vshuf_b(lane2, lane2, sh); + + __lsx_vst(compressed, reinterpret_cast<__m128i *>(output + 2 * 16), 0); + __lsx_vst(lane3, reinterpret_cast<__m128i *>(output + 3 * 16 - 1), 0); + } break; + case 0b11: { + __lasx_xvst(b->chunks[0], reinterpret_cast<__m256i *>(output + 0 * 16), 0); + __lsx_vst(lasx_extracti128_lo(b->chunks[1]), + reinterpret_cast<__m128i *>(output + 2 * 16), 0); + + const __m128i lane3 = lasx_extracti128_hi(b->chunks[1]); + + const __m128i v0 = __lsx_vreplgr2vr_b((uint8_t)(pos - 1)); + const __m128i v2 = __lsx_vslt_b(v0, (__m128i)v1); + const __m128i sh = __lsx_vsub_b((__m128i)v1, v2); + const __m128i compressed = __lsx_vshuf_b(lane3, lane3, sh); + + __lsx_vst(compressed, reinterpret_cast<__m128i *>(output + 3 * 16), 0); + } break; + } + return 63; +} + // The caller of this function is responsible to ensure that there are 64 bytes // available from reading at src. The data is read into a block64 structure. static inline void load_block(block64 *b, const char *src) { - b->chunks[0] = __lsx_vld(reinterpret_cast(src), 0); - b->chunks[1] = __lsx_vld(reinterpret_cast(src), 16); - b->chunks[2] = __lsx_vld(reinterpret_cast(src), 32); - b->chunks[3] = __lsx_vld(reinterpret_cast(src), 48); + b->chunks[0] = __lasx_xvld(reinterpret_cast(src), 0); + b->chunks[1] = __lasx_xvld(reinterpret_cast(src), 32); } // The caller of this function is responsible to ensure that there are 128 bytes // available from reading at src. The data is read into a block64 structure. static inline void load_block(block64 *b, const char16_t *src) { - __m128i m1 = __lsx_vld(reinterpret_cast(src), 0); - __m128i m2 = __lsx_vld(reinterpret_cast(src), 16); - __m128i m3 = __lsx_vld(reinterpret_cast(src), 32); - __m128i m4 = __lsx_vld(reinterpret_cast(src), 48); - __m128i m5 = __lsx_vld(reinterpret_cast(src), 64); - __m128i m6 = __lsx_vld(reinterpret_cast(src), 80); - __m128i m7 = __lsx_vld(reinterpret_cast(src), 96); - __m128i m8 = __lsx_vld(reinterpret_cast(src), 112); - b->chunks[0] = __lsx_vssrlni_bu_h(m2, m1, 0); - b->chunks[1] = __lsx_vssrlni_bu_h(m4, m3, 0); - b->chunks[2] = __lsx_vssrlni_bu_h(m6, m5, 0); - b->chunks[3] = __lsx_vssrlni_bu_h(m8, m7, 0); + __m256i m1 = __lasx_xvld(reinterpret_cast(src), 0); + __m256i m2 = __lasx_xvld(reinterpret_cast(src), 32); + __m256i m3 = __lasx_xvld(reinterpret_cast(src), 64); + __m256i m4 = __lasx_xvld(reinterpret_cast(src), 96); + b->chunks[0] = __lasx_xvpermi_d(__lasx_xvssrlni_bu_h(m2, m1, 0), 0b11011000); + b->chunks[1] = __lasx_xvpermi_d(__lasx_xvssrlni_bu_h(m4, m3, 0), 0b11011000); } -static inline void base64_decode(char *out, __m128i str) { - __m128i t0 = __lsx_vor_v( - __lsx_vslli_w(str, 26), - __lsx_vslli_w(__lsx_vand_v(str, lsx_splat_u32(0x0000FF00)), 12)); - __m128i t1 = __lsx_vsrli_w(__lsx_vand_v(str, lsx_splat_u32(0x003F0000)), 2); - __m128i t2 = __lsx_vor_v(t0, t1); - __m128i t3 = __lsx_vor_v(t2, __lsx_vsrli_w(str, 16)); - const v16u8 pack_shuffle = {3, 2, 1, 7, 6, 5, 11, 10, - 9, 15, 14, 13, 0, 0, 0, 0}; - t3 = __lsx_vshuf_b(t3, t3, (__m128i)pack_shuffle); +static inline void base64_decode(char *out, __m256i str) { + __m256i t0 = __lasx_xvor_v( + __lasx_xvslli_w(str, 26), + __lasx_xvslli_w(__lasx_xvand_v(str, lasx_splat_u32(0x0000ff00)), 12)); + __m256i t1 = + __lasx_xvsrli_w(__lasx_xvand_v(str, lasx_splat_u32(0x003f0000)), 2); + __m256i t2 = __lasx_xvor_v(t0, t1); + __m256i t3 = __lasx_xvor_v(t2, __lasx_xvsrli_w(str, 16)); + __m256i pack_shuffle = ____m256i( + (__m128i)v16u8{3, 2, 1, 7, 6, 5, 11, 10, 9, 15, 14, 13, 0, 0, 0, 0}); + t3 = __lasx_xvshuf_b(t3, t3, (__m256i)pack_shuffle); // Store the output: - // we only need 12. - __lsx_vstelm_d(t3, out, 0, 0); - __lsx_vstelm_w(t3, out + 8, 0, 2); + __lsx_vst(lasx_extracti128_lo(t3), out, 0); + __lsx_vst(lasx_extracti128_hi(t3), out, 12); } // decode 64 bytes and output 48 bytes static inline void base64_decode_block(char *out, const char *src) { - base64_decode(out, __lsx_vld(reinterpret_cast(src), 0)); - base64_decode(out + 12, - __lsx_vld(reinterpret_cast(src), 16)); + base64_decode(out, __lasx_xvld(reinterpret_cast(src), 0)); base64_decode(out + 24, - __lsx_vld(reinterpret_cast(src), 32)); - base64_decode(out + 36, - __lsx_vld(reinterpret_cast(src), 48)); + __lasx_xvld(reinterpret_cast(src), 32)); } + static inline void base64_decode_block_safe(char *out, const char *src) { - base64_decode_block(out, src); + base64_decode(out, __lasx_xvld(reinterpret_cast(src), 0)); + alignas(32) char buffer[32]; + base64_decode(buffer, + __lasx_xvld(reinterpret_cast(src), 32)); + std::memcpy(out + 24, buffer, 24); } + static inline void base64_decode_block(char *out, block64 *b) { base64_decode(out, b->chunks[0]); - base64_decode(out + 12, b->chunks[1]); - base64_decode(out + 24, b->chunks[2]); - base64_decode(out + 36, b->chunks[3]); + base64_decode(out + 24, b->chunks[1]); } static inline void base64_decode_block_safe(char *out, block64 *b) { - base64_decode_block(out, b); + base64_decode(out, b->chunks[0]); + alignas(32) char buffer[32]; + base64_decode(buffer, b->chunks[1]); + std::memcpy(out + 24, buffer, 24); } template + typename chartype> full_result -compress_decode_base64(char *dst, const char_type *src, size_t srclen, +compress_decode_base64(char *dst, const chartype *src, size_t srclen, base64_options options, last_chunk_handling_options last_chunk_options) { const uint8_t *to_base64 = @@ -19380,15 +19313,19 @@ compress_decode_base64(char *dst, const char_type *src, size_t srclen, } return {SUCCESS, full_input_length, 0}; } - const char_type *const srcinit = src; + char *end_of_safe_64byte_zone = + (srclen + 3) / 4 * 3 >= 63 ? dst + (srclen + 3) / 4 * 3 - 63 : dst; + + const chartype *const srcinit = src; const char *const dstinit = dst; - const char_type *const srcend = src + srclen; + const chartype *const srcend = src + srclen; - constexpr size_t block_size = 10; + constexpr size_t block_size = 6; + static_assert(block_size >= 2, "block_size must be at least two"); char buffer[block_size * 64]; char *bufferptr = buffer; if (srclen >= 64) { - const char_type *const srcend64 = src + srclen - 64; + const chartype *const srcend64 = src + srclen - 64; while (src <= srcend64) { block64 b; load_block(&b, src); @@ -19396,48 +19333,56 @@ compress_decode_base64(char *dst, const char_type *src, size_t srclen, bool error = false; uint64_t badcharmask = to_base64_mask(&b, &error); - if (badcharmask) { - if (error && !ignore_garbage) { - src -= 64; - while (src < srcend && scalar::base64::is_eight_byte(*src) && - to_base64[uint8_t(*src)] <= 64) { - src++; - } - if (src < srcend) { - // should never happen - } - return {error_code::INVALID_BASE64_CHARACTER, size_t(src - srcinit), - size_t(dst - dstinit)}; + if (error && !ignore_garbage) { + src -= 64; + while (src < srcend && scalar::base64::is_eight_byte(*src) && + to_base64[uint8_t(*src)] <= 64) { + src++; } + return {error_code::INVALID_BASE64_CHARACTER, size_t(src - srcinit), + size_t(dst - dstinit)}; } - if (badcharmask != 0) { - // optimization opportunity: check for simple masks like those made of - // continuous 1s followed by continuous 0s. And masks containing a - // single bad character. - bufferptr += compress_block(&b, badcharmask, bufferptr); - } else { - // optimization opportunity: if bufferptr == buffer and mask == 0, we - // can avoid the call to compress_block and decode directly. + if (is_power_of_two(badcharmask)) { + bufferptr += compress_block_single(&b, badcharmask, bufferptr); + } else { + bufferptr += compress_block(&b, badcharmask, bufferptr); + } + } else if (bufferptr != buffer) { copy_block(&b, bufferptr); bufferptr += 64; + } else { + if (dst >= end_of_safe_64byte_zone) { + base64_decode_block_safe(dst, &b); + } else { + base64_decode_block(dst, &b); + } + dst += 48; } if (bufferptr >= (block_size - 1) * 64 + buffer) { - for (size_t i = 0; i < (block_size - 1); i++) { + for (size_t i = 0; i < (block_size - 2); i++) { base64_decode_block(dst, buffer + i * 64); dst += 48; } + if (dst >= end_of_safe_64byte_zone) { + base64_decode_block_safe(dst, buffer + (block_size - 2) * 64); + } else { + base64_decode_block(dst, buffer + (block_size - 2) * 64); + } + dst += 48; std::memcpy(buffer, buffer + (block_size - 1) * 64, 64); // 64 might be too much bufferptr -= (block_size - 1) * 64; } } } + char *buffer_start = buffer; // Optimization note: if this is almost full, then it is worth our // time, otherwise, we should just decode directly. int last_block = (int)((bufferptr - buffer_start) % 64); if (last_block != 0 && srcend - src + last_block >= 64) { + while ((bufferptr - buffer_start) % 64 != 0 && src < srcend) { uint8_t val = to_base64[uint8_t(*src)]; *bufferptr = char(val); @@ -19452,7 +19397,11 @@ compress_decode_base64(char *dst, const char_type *src, size_t srclen, } for (; buffer_start + 64 <= bufferptr; buffer_start += 64) { - base64_decode_block(dst, buffer_start); + if (dst >= end_of_safe_64byte_zone) { + base64_decode_block_safe(dst, buffer_start); + } else { + base64_decode_block(dst, buffer_start); + } dst += 48; } if ((bufferptr - buffer_start) % 64 != 0) { @@ -19462,6 +19411,7 @@ compress_decode_base64(char *dst, const char_type *src, size_t srclen, (uint32_t(uint8_t(buffer_start[2])) << 1 * 6) + (uint32_t(uint8_t(buffer_start[3])) << 0 * 6)) << 8; + // lasx is little-endian triple = scalar::u32_swap_bytes(triple); std::memcpy(dst, &triple, 4); @@ -19474,6 +19424,7 @@ compress_decode_base64(char *dst, const char_type *src, size_t srclen, (uint32_t(uint8_t(buffer_start[2])) << 1 * 6) + (uint32_t(uint8_t(buffer_start[3])) << 0 * 6)) << 8; + // lasx is little-endian triple = scalar::u32_swap_bytes(triple); std::memcpy(dst, &triple, 3); @@ -19533,22 +19484,24 @@ compress_decode_base64(char *dst, const char_type *src, size_t srclen, } return {SUCCESS, srclen, size_t(dst - dstinit)}; } -/* end file src/lsx/lsx_base64.cpp */ -/* begin file src/lsx/lsx_find.cpp */ +/* end file src/lasx/lasx_base64.cpp */ +/* begin file src/lasx/lasx_find.cpp */ simdutf_really_inline const char *util_find(const char *start, const char *end, char character) noexcept { if (start >= end) return end; - const int step = 16; - __m128i char_vec = __lsx_vreplgr2vr_b(static_cast(character)); + const int step = 32; + __m256i char_vec = __lasx_xvreplgr2vr_b(static_cast(character)); while (end - start >= step) { - __m128i data = __lsx_vld(reinterpret_cast(start), 0); - __m128i cmp = __lsx_vseq_b(data, char_vec); - if (__lsx_bnz_v(cmp)) { - uint16_t mask = - static_cast(__lsx_vpickve2gr_hu(__lsx_vmsknz_b(cmp), 0)); + __m256i data = __lasx_xvld(reinterpret_cast(start), 0); + __m256i cmp = __lasx_xvseq_b(data, char_vec); + if (__lasx_xbnz_v(cmp)) { + __m256i res = __lasx_xvmsknz_b(cmp); + uint32_t mask0 = __lasx_xvpickve2gr_wu(res, 0); + uint32_t mask1 = __lasx_xvpickve2gr_wu(res, 4); + uint32_t mask = (mask0 | (mask1 << 16)); return start + trailing_zeroes(mask); } @@ -19571,15 +19524,17 @@ simdutf_really_inline const char16_t *util_find(const char16_t *start, if (start >= end) return end; - const int step = 8; - __m128i char_vec = __lsx_vreplgr2vr_h(static_cast(character)); + const int step = 16; + __m256i char_vec = __lasx_xvreplgr2vr_h(static_cast(character)); while (end - start >= step) { - __m128i data = __lsx_vld(reinterpret_cast(start), 0); - __m128i cmp = __lsx_vseq_h(data, char_vec); - if (__lsx_bnz_v(cmp)) { - uint16_t mask = - static_cast(__lsx_vpickve2gr_hu(__lsx_vmsknz_b(cmp), 0)); + __m256i data = __lasx_xvld(reinterpret_cast(start), 0); + __m256i cmp = __lasx_xvseq_h(data, char_vec); + if (__lasx_xbnz_v(cmp)) { + __m256i res = __lasx_xvmsknz_b(cmp); + uint32_t mask0 = __lasx_xvpickve2gr_wu(res, 0); + uint32_t mask1 = __lasx_xvpickve2gr_wu(res, 4); + uint32_t mask = (mask0 | (mask1 << 16)); return start + trailing_zeroes(mask) / 2; } @@ -19595,15 +19550,15 @@ simdutf_really_inline const char16_t *util_find(const char16_t *start, return end; } -/* end file src/lsx/lsx_find.cpp */ +/* end file src/lasx/lasx_find.cpp */ } // namespace -} // namespace lsx +} // namespace lasx } // namespace simdutf /* begin file src/generic/buf_block_reader.h */ namespace simdutf { -namespace lsx { +namespace lasx { namespace { // Walks through a buffer in block-sized increments, loading the last part with @@ -19634,40 +19589,6 @@ template struct buf_block_reader { size_t idx; }; -// Routines to print masks and text for debugging bitmask operations -simdutf_unused static char *format_input_text_64(const uint8_t *text) { - static char *buf = - reinterpret_cast(malloc(sizeof(simd8x64) + 1)); - for (size_t i = 0; i < sizeof(simd8x64); i++) { - buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); - } - buf[sizeof(simd8x64)] = '\0'; - return buf; -} - -// Routines to print masks and text for debugging bitmask operations -simdutf_unused static char *format_input_text(const simd8x64 &in) { - static char *buf = - reinterpret_cast(malloc(sizeof(simd8x64) + 1)); - in.store(reinterpret_cast(buf)); - for (size_t i = 0; i < sizeof(simd8x64); i++) { - if (buf[i] < ' ') { - buf[i] = '_'; - } - } - buf[sizeof(simd8x64)] = '\0'; - return buf; -} - -simdutf_unused static char *format_mask(uint64_t mask) { - static char *buf = reinterpret_cast(malloc(64 + 1)); - for (size_t i = 0; i < 64; i++) { - buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; - } - buf[64] = '\0'; - return buf; -} - template simdutf_really_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) @@ -19709,15 +19630,81 @@ simdutf_really_inline void buf_block_reader::advance() { } } // unnamed namespace -} // namespace lsx +} // namespace lasx } // namespace simdutf /* end file src/generic/buf_block_reader.h */ +/* begin file src/generic/base64lengths.h */ +namespace simdutf { +namespace lasx { +namespace { +namespace base64_lengths { + +simdutf_warn_unused size_t binary_length_from_base64(const char *input, + size_t length) { + size_t pos = 0; + size_t count = 0; + for (; pos + 64 <= length; pos += 64) { + simd8x64 block(reinterpret_cast(input + pos)); + uint64_t maybe_base64 = block.gteq(33); // >= 33 which is '!' in ASCII + count += count_ones(maybe_base64); + } + while (pos < length) { + count += (input[pos] > 0x20) ? 1 : 0; + pos++; + } + // Count padding at the end. + size_t padding = 0; + pos = length; + while (pos > 0 && padding < 2) { + char c = input[--pos]; + if (c == '=') { + padding++; + } else if (c > ' ') { + break; + } + } + return ((count - padding) * 3) / 4; +} + +simdutf_warn_unused size_t binary_length_from_base64(const char16_t *input, + size_t length) { + size_t pos = 0; + size_t count = 0; + for (; pos + 32 <= length; pos += 32) { + simd16x32 block(reinterpret_cast(input + pos)); + uint64_t maybe_base64 = block.gteq(33); // >= 33 which is '!' in ASCII + count += count_ones(maybe_base64); + } + while (pos < length) { + count += (input[pos] > 0x20) ? 1 : 0; + pos++; + } + // Count padding at the end. + size_t padding = 0; + pos = length; + while (pos > 0 && padding < 2) { + char16_t c = input[--pos]; + if (c == '=') { + padding++; + } else if (c > ' ') { + break; + } + } + return ((count - padding) * 3) / 4; +} + +} // namespace base64_lengths +} // unnamed namespace +} // namespace lasx +} // namespace simdutf +/* end file src/generic/base64lengths.h */ + // // Implementation-specific overrides // namespace simdutf { -namespace lsx { +namespace lasx { simdutf_warn_unused result implementation::base64_to_binary( const char *input, size_t length, char *output, base64_options options, @@ -19848,6 +19835,14 @@ size_t implementation::binary_to_base64(const char *input, size_t length, return encode_base64(output, input, length, options); } } + +size_t implementation::binary_to_base64_with_lines( + const char *input, size_t length, char *output, size_t line_length, + base64_options options) const noexcept { + return scalar::base64::tail_encode_base64_impl(output, input, length, + options, line_length); +} + const char *implementation::find(const char *start, const char *end, char character) const noexcept { return util_find(start, end, character); @@ -19858,30 +19853,46 @@ const char16_t *implementation::find(const char16_t *start, const char16_t *end, return util_find(start, end, character); } -} // namespace lsx +simdutf_warn_unused size_t implementation::binary_length_from_base64( + const char *input, size_t length) const noexcept { + return base64_lengths::binary_length_from_base64(input, length); +} + +simdutf_warn_unused size_t implementation::binary_length_from_base64( + const char16_t *input, size_t length) const noexcept { + return base64_lengths::binary_length_from_base64(input, length); +} + +} // namespace lasx } // namespace simdutf -/* begin file src/simdutf/lsx/end.h */ +/* begin file src/simdutf/lasx/end.h */ #undef SIMDUTF_SIMD_HAS_UNSIGNED_CMP -/* end file src/simdutf/lsx/end.h */ -/* end file src/lsx/implementation.cpp */ + +#if SIMDUTF_CAN_ALWAYS_RUN_LASX +// nothing needed. +#else +SIMDUTF_UNTARGET_REGION #endif -#if SIMDUTF_IMPLEMENTATION_LASX -/* begin file src/lasx/implementation.cpp */ -/* begin file src/simdutf/lasx/begin.h */ -// redefining SIMDUTF_IMPLEMENTATION to "lasx" -// #define SIMDUTF_IMPLEMENTATION lasx +/* end file src/simdutf/lasx/end.h */ +/* end file src/lasx/implementation.cpp */ +#endif +#if SIMDUTF_IMPLEMENTATION_LSX +/* begin file src/lsx/implementation.cpp */ +/* begin file src/simdutf/lsx/begin.h */ +// redefining SIMDUTF_IMPLEMENTATION to "lsx" +// #define SIMDUTF_IMPLEMENTATION lsx #define SIMDUTF_SIMD_HAS_UNSIGNED_CMP 1 -/* end file src/simdutf/lasx/begin.h */ +/* end file src/simdutf/lsx/begin.h */ namespace simdutf { -namespace lasx { +namespace lsx { namespace { -#ifndef SIMDUTF_LASX_H - #error "lasx.h must be included" +#ifndef SIMDUTF_LSX_H + #error "lsx.h must be included" #endif using namespace simd; -/* begin file src/lasx/lasx_base64.cpp */ +/* begin file src/lsx/lsx_base64.cpp */ /** * References and further reading: * @@ -19922,118 +19933,100 @@ size_t encode_base64(char *dst, const char *src, size_t srclen, : "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; uint8_t *out = (uint8_t *)dst; - v32u8 shuf; - __m256i v_fc0fc00, v_3f03f0, shift_r, shift_l, base64_tbl0, base64_tbl1, + v16u8 shuf; + __m128i v_fc0fc00, v_3f03f0, shift_r, shift_l, base64_tbl0, base64_tbl1, base64_tbl2, base64_tbl3; - if (srclen >= 28) { - shuf = v32u8{1, 0, 2, 1, 4, 3, 5, 4, 7, 6, 8, 7, 10, 9, 11, 10, - 1, 0, 2, 1, 4, 3, 5, 4, 7, 6, 8, 7, 10, 9, 11, 10}; - - v_fc0fc00 = __lasx_xvreplgr2vr_w(uint32_t(0x0fc0fc00)); - v_3f03f0 = __lasx_xvreplgr2vr_w(uint32_t(0x003f03f0)); - shift_r = __lasx_xvreplgr2vr_w(uint32_t(0x0006000a)); - shift_l = __lasx_xvreplgr2vr_w(uint32_t(0x00080004)); - base64_tbl0 = ____m256i(__lsx_vld(lookup_tbl, 0)); - base64_tbl1 = ____m256i(__lsx_vld(lookup_tbl, 16)); - base64_tbl2 = ____m256i(__lsx_vld(lookup_tbl, 32)); - base64_tbl3 = ____m256i(__lsx_vld(lookup_tbl, 48)); + if (srclen >= 16) { + shuf = v16u8{1, 0, 2, 1, 4, 3, 5, 4, 7, 6, 8, 7, 10, 9, 11, 10}; + v_fc0fc00 = __lsx_vreplgr2vr_w(uint32_t(0x0fc0fc00)); + v_3f03f0 = __lsx_vreplgr2vr_w(uint32_t(0x003f03f0)); + shift_r = __lsx_vreplgr2vr_w(uint32_t(0x0006000a)); + shift_l = __lsx_vreplgr2vr_w(uint32_t(0x00080004)); + base64_tbl0 = __lsx_vld(lookup_tbl, 0); + base64_tbl1 = __lsx_vld(lookup_tbl, 16); + base64_tbl2 = __lsx_vld(lookup_tbl, 32); + base64_tbl3 = __lsx_vld(lookup_tbl, 48); } + size_t i = 0; - for (; i + 100 <= srclen; i += 96) { - __m128i in0_lo = + for (; i + 52 <= srclen; i += 48) { + __m128i in0 = __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 0); - __m128i in0_hi = + __m128i in1 = __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 1); - __m128i in1_lo = + __m128i in2 = __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 2); - __m128i in1_hi = + __m128i in3 = __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 3); - __m128i in2_lo = - __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 4); - __m128i in2_hi = - __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 5); - __m128i in3_lo = - __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 6); - __m128i in3_hi = - __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 7); - - __m256i in0 = lasx_set_q(in0_hi, in0_lo); - __m256i in1 = lasx_set_q(in1_hi, in1_lo); - __m256i in2 = lasx_set_q(in2_hi, in2_lo); - __m256i in3 = lasx_set_q(in3_hi, in3_lo); - - in0 = __lasx_xvshuf_b(in0, in0, (__m256i)shuf); - in1 = __lasx_xvshuf_b(in1, in1, (__m256i)shuf); - in2 = __lasx_xvshuf_b(in2, in2, (__m256i)shuf); - in3 = __lasx_xvshuf_b(in3, in3, (__m256i)shuf); - __m256i t0_0 = __lasx_xvand_v(in0, v_fc0fc00); - __m256i t0_1 = __lasx_xvand_v(in1, v_fc0fc00); - __m256i t0_2 = __lasx_xvand_v(in2, v_fc0fc00); - __m256i t0_3 = __lasx_xvand_v(in3, v_fc0fc00); + in0 = __lsx_vshuf_b(in0, in0, (__m128i)shuf); + in1 = __lsx_vshuf_b(in1, in1, (__m128i)shuf); + in2 = __lsx_vshuf_b(in2, in2, (__m128i)shuf); + in3 = __lsx_vshuf_b(in3, in3, (__m128i)shuf); - __m256i t1_0 = __lasx_xvsrl_h(t0_0, shift_r); - __m256i t1_1 = __lasx_xvsrl_h(t0_1, shift_r); - __m256i t1_2 = __lasx_xvsrl_h(t0_2, shift_r); - __m256i t1_3 = __lasx_xvsrl_h(t0_3, shift_r); + __m128i t0_0 = __lsx_vand_v(in0, v_fc0fc00); + __m128i t0_1 = __lsx_vand_v(in1, v_fc0fc00); + __m128i t0_2 = __lsx_vand_v(in2, v_fc0fc00); + __m128i t0_3 = __lsx_vand_v(in3, v_fc0fc00); - __m256i t2_0 = __lasx_xvand_v(in0, v_3f03f0); - __m256i t2_1 = __lasx_xvand_v(in1, v_3f03f0); - __m256i t2_2 = __lasx_xvand_v(in2, v_3f03f0); - __m256i t2_3 = __lasx_xvand_v(in3, v_3f03f0); + __m128i t1_0 = __lsx_vsrl_h(t0_0, shift_r); + __m128i t1_1 = __lsx_vsrl_h(t0_1, shift_r); + __m128i t1_2 = __lsx_vsrl_h(t0_2, shift_r); + __m128i t1_3 = __lsx_vsrl_h(t0_3, shift_r); - __m256i t3_0 = __lasx_xvsll_h(t2_0, shift_l); - __m256i t3_1 = __lasx_xvsll_h(t2_1, shift_l); - __m256i t3_2 = __lasx_xvsll_h(t2_2, shift_l); - __m256i t3_3 = __lasx_xvsll_h(t2_3, shift_l); + __m128i t2_0 = __lsx_vand_v(in0, v_3f03f0); + __m128i t2_1 = __lsx_vand_v(in1, v_3f03f0); + __m128i t2_2 = __lsx_vand_v(in2, v_3f03f0); + __m128i t2_3 = __lsx_vand_v(in3, v_3f03f0); - __m256i input0 = __lasx_xvor_v(t1_0, t3_0); - __m256i input0_shuf0 = __lasx_xvshuf_b(base64_tbl1, base64_tbl0, input0); - __m256i input0_shuf1 = __lasx_xvshuf_b( - base64_tbl3, base64_tbl2, __lasx_xvsub_b(input0, __lasx_xvldi(32))); - __m256i input0_mask = __lasx_xvslei_bu(input0, 31); - __m256i input0_result = - __lasx_xvbitsel_v(input0_shuf1, input0_shuf0, input0_mask); - __lasx_xvst(input0_result, reinterpret_cast<__m256i *>(out), 0); - out += 32; + __m128i t3_0 = __lsx_vsll_h(t2_0, shift_l); + __m128i t3_1 = __lsx_vsll_h(t2_1, shift_l); + __m128i t3_2 = __lsx_vsll_h(t2_2, shift_l); + __m128i t3_3 = __lsx_vsll_h(t2_3, shift_l); - __m256i input1 = __lasx_xvor_v(t1_1, t3_1); - __m256i input1_shuf0 = __lasx_xvshuf_b(base64_tbl1, base64_tbl0, input1); - __m256i input1_shuf1 = __lasx_xvshuf_b( - base64_tbl3, base64_tbl2, __lasx_xvsub_b(input1, __lasx_xvldi(32))); - __m256i input1_mask = __lasx_xvslei_bu(input1, 31); - __m256i input1_result = - __lasx_xvbitsel_v(input1_shuf1, input1_shuf0, input1_mask); - __lasx_xvst(input1_result, reinterpret_cast<__m256i *>(out), 0); - out += 32; + __m128i input0 = __lsx_vor_v(t1_0, t3_0); + __m128i input0_shuf0 = __lsx_vshuf_b(base64_tbl1, base64_tbl0, input0); + __m128i input0_shuf1 = __lsx_vshuf_b(base64_tbl3, base64_tbl2, + __lsx_vsub_b(input0, __lsx_vldi(32))); + __m128i input0_mask = __lsx_vslei_bu(input0, 31); + __m128i input0_result = + __lsx_vbitsel_v(input0_shuf1, input0_shuf0, input0_mask); + __lsx_vst(input0_result, reinterpret_cast<__m128i *>(out), 0); + out += 16; - __m256i input2 = __lasx_xvor_v(t1_2, t3_2); - __m256i input2_shuf0 = __lasx_xvshuf_b(base64_tbl1, base64_tbl0, input2); - __m256i input2_shuf1 = __lasx_xvshuf_b( - base64_tbl3, base64_tbl2, __lasx_xvsub_b(input2, __lasx_xvldi(32))); - __m256i input2_mask = __lasx_xvslei_bu(input2, 31); - __m256i input2_result = - __lasx_xvbitsel_v(input2_shuf1, input2_shuf0, input2_mask); - __lasx_xvst(input2_result, reinterpret_cast<__m256i *>(out), 0); - out += 32; + __m128i input1 = __lsx_vor_v(t1_1, t3_1); + __m128i input1_shuf0 = __lsx_vshuf_b(base64_tbl1, base64_tbl0, input1); + __m128i input1_shuf1 = __lsx_vshuf_b(base64_tbl3, base64_tbl2, + __lsx_vsub_b(input1, __lsx_vldi(32))); + __m128i input1_mask = __lsx_vslei_bu(input1, 31); + __m128i input1_result = + __lsx_vbitsel_v(input1_shuf1, input1_shuf0, input1_mask); + __lsx_vst(input1_result, reinterpret_cast<__m128i *>(out), 0); + out += 16; - __m256i input3 = __lasx_xvor_v(t1_3, t3_3); - __m256i input3_shuf0 = __lasx_xvshuf_b(base64_tbl1, base64_tbl0, input3); - __m256i input3_shuf1 = __lasx_xvshuf_b( - base64_tbl3, base64_tbl2, __lasx_xvsub_b(input3, __lasx_xvldi(32))); - __m256i input3_mask = __lasx_xvslei_bu(input3, 31); - __m256i input3_result = - __lasx_xvbitsel_v(input3_shuf1, input3_shuf0, input3_mask); - __lasx_xvst(input3_result, reinterpret_cast<__m256i *>(out), 0); - out += 32; - } - for (; i + 28 <= srclen; i += 24) { + __m128i input2 = __lsx_vor_v(t1_2, t3_2); + __m128i input2_shuf0 = __lsx_vshuf_b(base64_tbl1, base64_tbl0, input2); + __m128i input2_shuf1 = __lsx_vshuf_b(base64_tbl3, base64_tbl2, + __lsx_vsub_b(input2, __lsx_vldi(32))); + __m128i input2_mask = __lsx_vslei_bu(input2, 31); + __m128i input2_result = + __lsx_vbitsel_v(input2_shuf1, input2_shuf0, input2_mask); + __lsx_vst(input2_result, reinterpret_cast<__m128i *>(out), 0); + out += 16; - __m128i in_lo = __lsx_vld(reinterpret_cast(input + i), 0); - __m128i in_hi = - __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 1); + __m128i input3 = __lsx_vor_v(t1_3, t3_3); + __m128i input3_shuf0 = __lsx_vshuf_b(base64_tbl1, base64_tbl0, input3); + __m128i input3_shuf1 = __lsx_vshuf_b(base64_tbl3, base64_tbl2, + __lsx_vsub_b(input3, __lsx_vldi(32))); + __m128i input3_mask = __lsx_vslei_bu(input3, 31); + __m128i input3_result = + __lsx_vbitsel_v(input3_shuf1, input3_shuf0, input3_mask); + __lsx_vst(input3_result, reinterpret_cast<__m128i *>(out), 0); + out += 16; + } + for (; i + 16 <= srclen; i += 12) { - __m256i in = lasx_set_q(in_hi, in_lo); + __m128i in = __lsx_vld(reinterpret_cast(input + i), 0); // bytes from groups A, B and C are needed in separate 32-bit lanes // in = [DDDD|CCCC|BBBB|AAAA] @@ -20047,32 +20040,33 @@ size_t encode_base64(char *dst, const char *src, size_t srclen, // [bbbbcccc|ccdddddd|aaaaaabb|bbbbcccc] // ^^^^ ^^^^^^^^ ^^^^^^^^ ^^^^ // processed bits - in = __lasx_xvshuf_b(in, in, (__m256i)shuf); + in = __lsx_vshuf_b(in, in, (__m128i)shuf); // unpacking // t0 = [0000cccc|cc000000|aaaaaa00|00000000] - __m256i t0 = __lasx_xvand_v(in, v_fc0fc00); + __m128i t0 = __lsx_vand_v(in, v_fc0fc00); // t1 = [00000000|00cccccc|00000000|00aaaaaa] // ((c >> 6), (a >> 10)) - __m256i t1 = __lasx_xvsrl_h(t0, shift_r); + __m128i t1 = __lsx_vsrl_h(t0, shift_r); // t2 = [00000000|00dddddd|000000bb|bbbb0000] - __m256i t2 = __lasx_xvand_v(in, v_3f03f0); + __m128i t2 = __lsx_vand_v(in, v_3f03f0); // t3 = [00dddddd|00000000|00bbbbbb|00000000] // ((d << 8), (b << 4)) - __m256i t3 = __lasx_xvsll_h(t2, shift_l); + __m128i t3 = __lsx_vsll_h(t2, shift_l); // res = [00dddddd|00cccccc|00bbbbbb|00aaaaaa] = t1 | t3 - __m256i indices = __lasx_xvor_v(t1, t3); + __m128i indices = __lsx_vor_v(t1, t3); - __m256i indices_shuf0 = __lasx_xvshuf_b(base64_tbl1, base64_tbl0, indices); - __m256i indices_shuf1 = __lasx_xvshuf_b( - base64_tbl3, base64_tbl2, __lasx_xvsub_b(indices, __lasx_xvldi(32))); - __m256i indices_mask = __lasx_xvslei_bu(indices, 31); - __m256i indices_result = - __lasx_xvbitsel_v(indices_shuf1, indices_shuf0, indices_mask); - __lasx_xvst(indices_result, reinterpret_cast<__m256i *>(out), 0); - out += 32; + __m128i indices_shuf0 = __lsx_vshuf_b(base64_tbl1, base64_tbl0, indices); + __m128i indices_shuf1 = __lsx_vshuf_b( + base64_tbl3, base64_tbl2, __lsx_vsub_b(indices, __lsx_vldi(32))); + __m128i indices_mask = __lsx_vslei_bu(indices, 31); + __m128i indices_result = + __lsx_vbitsel_v(indices_shuf1, indices_shuf0, indices_mask); + + __lsx_vst(indices_result, reinterpret_cast<__m128i *>(out), 0); + out += 16; } return i / 3 * 4 + scalar::base64::tail_encode_base64((char *)out, src + i, @@ -20096,7 +20090,7 @@ static inline void compress(__m128i data, uint16_t mask, char *output) { tables::base64::thintable_epi8[mask2]}; // we increment by 0x08 the second half of the mask - const v4u32 hi = {0, 0, 0x08080808, 0x08080808}; + v4u32 hi = {0, 0, 0x08080808, 0x08080808}; __m128i shufmask1 = __lsx_vadd_b((__m128i)shufmask, (__m128i)hi); // this is the version "nearly pruned" @@ -20118,111 +20112,115 @@ static inline void compress(__m128i data, uint16_t mask, char *output) { } struct block64 { - __m256i chunks[2]; + __m128i chunks[4]; }; template -static inline uint32_t to_base64_mask(__m256i *src, bool *error) { - __m256i ascii_space_tbl = - ____m256i((__m128i)v16u8{0x20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x9, 0xa, 0x0, 0xc, 0xd, 0x0, 0x0}); +static inline uint16_t to_base64_mask(__m128i *src, bool *error) { + const v16u8 ascii_space_tbl = {0x20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x9, 0xa, 0x0, 0xc, 0xd, 0x0, 0x0}; // credit: aqrit - __m256i delta_asso; + /* + '0'(0x30)-'9'(0x39) => delta_values_index = 4 + 'A'(0x41)-'Z'(0x5a) => delta_values_index = 4/5/12(4+8) + 'a'(0x61)-'z'(0x7a) => delta_values_index = 6/7/14(6+8) + '+'(0x2b) => delta_values_index = 3 + '/'(0x2f) => delta_values_index = 2+8 = 10 + '-'(0x2d) => delta_values_index = 2+8 = 10 + '_'(0x5f) => delta_values_index = 5+8 = 13 + */ + v16u8 delta_asso; if (default_or_url) { - delta_asso = - ____m256i((__m128i)v16u8{0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x11, 0x0, 0x16}); + delta_asso = v16u8{0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x16}; } else { - delta_asso = - ____m256i((__m128i)v16u8{0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x0, - 0x0, 0x0, 0x0, 0x0, 0xF, 0x0, 0xF}); + delta_asso = v16u8{0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, + 0x0, 0x0, 0x0, 0x0, 0x0, 0xF, 0x0, 0xF}; } - __m256i delta_values; + v16i8 delta_values; if (default_or_url) { - delta_values = ____m256i( - (__m128i)v16i8{int8_t(0xBF), int8_t(0xE0), int8_t(0xB9), int8_t(0x13), - int8_t(0x04), int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), - int8_t(0xB9), int8_t(0x00), int8_t(0xFF), int8_t(0x11), - int8_t(0xFF), int8_t(0xBF), int8_t(0x10), int8_t(0xB9)}); + delta_values = + v16i8{int8_t(0xBF), int8_t(0xE0), int8_t(0xB9), int8_t(0x13), + int8_t(0x04), int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), + int8_t(0xB9), int8_t(0x00), int8_t(0xFF), int8_t(0x11), + int8_t(0xFF), int8_t(0xBF), int8_t(0x10), int8_t(0xB9)}; } else if (base64_url) { - delta_values = ____m256i( - (__m128i)v16i8{int8_t(0x00), int8_t(0x00), int8_t(0x00), int8_t(0x13), - int8_t(0x04), int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), - int8_t(0xB9), int8_t(0x00), int8_t(0x11), int8_t(0xC3), - int8_t(0xBF), int8_t(0xE0), int8_t(0xB9), int8_t(0xB9)}); + delta_values = + v16i8{int8_t(0x00), int8_t(0x00), int8_t(0x00), int8_t(0x13), + int8_t(0x04), int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), + int8_t(0xB9), int8_t(0x00), int8_t(0x11), int8_t(0xC3), + int8_t(0xBF), int8_t(0xE0), int8_t(0xB9), int8_t(0xB9)}; } else { - delta_values = ____m256i( - (__m128i)v16i8{int8_t(0x00), int8_t(0x00), int8_t(0x00), int8_t(0x13), - int8_t(0x04), int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), - int8_t(0xB9), int8_t(0x00), int8_t(0x10), int8_t(0xC3), - int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), int8_t(0xB9)}); + delta_values = + v16i8{int8_t(0x00), int8_t(0x00), int8_t(0x00), int8_t(0x13), + int8_t(0x04), int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), + int8_t(0xB9), int8_t(0x00), int8_t(0x10), int8_t(0xC3), + int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), int8_t(0xB9)}; } - __m256i check_asso; + v16u8 check_asso; if (default_or_url) { - check_asso = ____m256i((__m128i)v16u8{0x0D, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x01, 0x01, 0x01, 0x03, 0x07, - 0x0B, 0x0E, 0x0B, 0x06}); - + check_asso = v16u8{0x0D, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x03, 0x07, 0x0B, 0x0E, 0x0B, 0x06}; } else if (base64_url) { - check_asso = ____m256i((__m128i)v16u8{0x0D, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x01, 0x01, 0x01, 0x03, 0x07, - 0x0B, 0x06, 0x0B, 0x12}); + check_asso = v16u8{0x0D, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x03, 0x07, 0x0B, 0x06, 0x0B, 0x12}; } else { - check_asso = ____m256i((__m128i)v16u8{0x0D, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x01, 0x01, 0x01, 0x03, 0x07, - 0x0B, 0x0B, 0x0B, 0x0F}); + check_asso = v16u8{0x0D, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x03, 0x07, 0x0B, 0x0B, 0x0B, 0x0F}; } - __m256i check_values; + v16i8 check_values; if (default_or_url) { - - check_values = ____m256i( - (__m128i)v16i8{int8_t(0x80), int8_t(0x80), int8_t(0x80), int8_t(0x80), - int8_t(0xCF), int8_t(0xBF), int8_t(0xD5), int8_t(0xA6), - int8_t(0xB5), int8_t(0xA1), int8_t(0x00), int8_t(0x80), - int8_t(0x00), int8_t(0x80), int8_t(0x00), int8_t(0x80)}); + check_values = + v16i8{int8_t(0x80), int8_t(0x80), int8_t(0x80), int8_t(0x80), + int8_t(0xCF), int8_t(0xBF), int8_t(0xD5), int8_t(0xA6), + int8_t(0xB5), int8_t(0xA1), int8_t(0x00), int8_t(0x80), + int8_t(0x00), int8_t(0x80), int8_t(0x00), int8_t(0x80)}; } else if (base64_url) { - check_values = ____m256i( - (__m128i)v16i8{int8_t(0x0), int8_t(0x80), int8_t(0x80), int8_t(0x80), - int8_t(0xCF), int8_t(0xBF), int8_t(0xD3), int8_t(0xA6), - int8_t(0xB5), int8_t(0x86), int8_t(0xD0), int8_t(0x80), - int8_t(0xB0), int8_t(0x80), int8_t(0x0), int8_t(0x0)}); + check_values = v16i8{int8_t(0x0), int8_t(0x80), int8_t(0x80), int8_t(0x80), + int8_t(0xCF), int8_t(0xBF), int8_t(0xD3), int8_t(0xA6), + int8_t(0xB5), int8_t(0x86), int8_t(0xD0), int8_t(0x80), + int8_t(0xB0), int8_t(0x80), int8_t(0x0), int8_t(0x0)}; } else { - check_values = ____m256i( - (__m128i)v16i8{int8_t(0x80), int8_t(0x80), int8_t(0x80), int8_t(0x80), - int8_t(0xCF), int8_t(0xBF), int8_t(0xD5), int8_t(0xA6), - int8_t(0xB5), int8_t(0x86), int8_t(0xD1), int8_t(0x80), - int8_t(0xB1), int8_t(0x80), int8_t(0x91), int8_t(0x80)}); + check_values = + v16i8{int8_t(0x80), int8_t(0x80), int8_t(0x80), int8_t(0x80), + int8_t(0xCF), int8_t(0xBF), int8_t(0xD5), int8_t(0xA6), + int8_t(0xB5), int8_t(0x86), int8_t(0xD1), int8_t(0x80), + int8_t(0xB1), int8_t(0x80), int8_t(0x91), int8_t(0x80)}; } - __m256i shifted = __lasx_xvsrli_b(*src, 3); - __m256i asso_index = __lasx_xvand_v(*src, __lasx_xvldi(0xF)); - __m256i delta_hash = __lasx_xvavgr_bu( - __lasx_xvshuf_b(delta_asso, delta_asso, asso_index), shifted); - __m256i check_hash = __lasx_xvavgr_bu( - __lasx_xvshuf_b(check_asso, check_asso, asso_index), shifted); + const __m128i shifted = __lsx_vsrli_b(*src, 3); + __m128i asso_index = __lsx_vand_v(*src, __lsx_vldi(0xF)); + const __m128i delta_hash = + __lsx_vavgr_bu(__lsx_vshuf_b((__m128i)delta_asso, (__m128i)delta_asso, + (__m128i)asso_index), + shifted); + const __m128i check_hash = + __lsx_vavgr_bu(__lsx_vshuf_b((__m128i)check_asso, (__m128i)check_asso, + (__m128i)asso_index), + shifted); - __m256i out = __lasx_xvsadd_b( - __lasx_xvshuf_b(delta_values, delta_values, delta_hash), *src); - __m256i chk = __lasx_xvsadd_b( - __lasx_xvshuf_b(check_values, check_values, check_hash), *src); - __m256i chk_ltz = __lasx_xvmskltz_b(chk); - unsigned int mask = __lasx_xvpickve2gr_wu(chk_ltz, 0); - mask = mask | (__lsx_vpickve2gr_hu(lasx_extracti128_hi(chk_ltz), 0) << 16); + const __m128i out = + __lsx_vsadd_b(__lsx_vshuf_b((__m128i)delta_values, (__m128i)delta_values, + (__m128i)delta_hash), + *src); + const __m128i chk = + __lsx_vsadd_b(__lsx_vshuf_b((__m128i)check_values, (__m128i)check_values, + (__m128i)check_hash), + *src); + unsigned int mask = __lsx_vpickve2gr_hu(__lsx_vmskltz_b(chk), 0); if (mask) { - __m256i ascii_space = __lasx_xvseq_b( - __lasx_xvshuf_b(ascii_space_tbl, ascii_space_tbl, asso_index), *src); - __m256i ascii_space_ltz = __lasx_xvmskltz_b(ascii_space); - unsigned int ascii_space_mask = __lasx_xvpickve2gr_wu(ascii_space_ltz, 0); - ascii_space_mask = - ascii_space_mask | - (__lsx_vpickve2gr_hu(lasx_extracti128_hi(ascii_space_ltz), 0) << 16); - *error |= (mask != ascii_space_mask); + __m128i ascii_space = __lsx_vseq_b(__lsx_vshuf_b((__m128i)ascii_space_tbl, + (__m128i)ascii_space_tbl, + (__m128i)asso_index), + *src); + *error |= + (mask != __lsx_vpickve2gr_hu(__lsx_vmskltz_b((__m128i)ascii_space), 0)); } *src = out; - return (uint32_t)mask; + return (uint16_t)mask; } template @@ -20232,12 +20230,18 @@ static inline uint64_t to_base64_mask(block64 *b, bool *error) { to_base64_mask(&b->chunks[0], error); uint64_t m1 = to_base64_mask(&b->chunks[1], error); - return m0 | (m1 << 32); + uint64_t m2 = + to_base64_mask(&b->chunks[2], error); + uint64_t m3 = + to_base64_mask(&b->chunks[3], error); + return m0 | (m1 << 16) | (m2 << 32) | (m3 << 48); } static inline void copy_block(block64 *b, char *output) { - __lasx_xvst(b->chunks[0], reinterpret_cast<__m256i *>(output), 0); - __lasx_xvst(b->chunks[1], reinterpret_cast<__m256i *>(output), 32); + __lsx_vst(b->chunks[0], reinterpret_cast<__m128i *>(output), 0); + __lsx_vst(b->chunks[1], reinterpret_cast<__m128i *>(output), 16); + __lsx_vst(b->chunks[2], reinterpret_cast<__m128i *>(output), 32); + __lsx_vst(b->chunks[3], reinterpret_cast<__m128i *>(output), 48); } static inline uint64_t compress_block(block64 *b, uint64_t mask, char *output) { @@ -20245,80 +20249,152 @@ static inline uint64_t compress_block(block64 *b, uint64_t mask, char *output) { uint64_t count = __lsx_vpickve2gr_d(__lsx_vpcnt_h(__lsx_vreplgr2vr_d(nmask)), 0); uint16_t *count_ptr = (uint16_t *)&count; - compress(lasx_extracti128_lo(b->chunks[0]), uint16_t(mask), output); - compress(lasx_extracti128_hi(b->chunks[0]), uint16_t(mask >> 16), - output + count_ptr[0]); - compress(lasx_extracti128_lo(b->chunks[1]), uint16_t(mask >> 32), + compress(b->chunks[0], uint16_t(mask), output); + compress(b->chunks[1], uint16_t(mask >> 16), output + count_ptr[0]); + compress(b->chunks[2], uint16_t(mask >> 32), output + count_ptr[0] + count_ptr[1]); - compress(lasx_extracti128_hi(b->chunks[1]), uint16_t(mask >> 48), + compress(b->chunks[3], uint16_t(mask >> 48), output + count_ptr[0] + count_ptr[1] + count_ptr[2]); return count_ones(nmask); } +template bool is_power_of_two(T x) { return (x & (x - 1)) == 0; } + +inline size_t compress_block_single(block64 *b, uint64_t mask, char *output) { + const size_t pos64 = trailing_zeroes(mask); + const int8_t pos = pos64 & 0xf; + // Predefine the index vector + const v16u8 v1 = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; + + switch (pos64 >> 4) { + case 0b00: { + const __m128i v0 = __lsx_vreplgr2vr_b((uint8_t)(pos - 1)); + const __m128i v2 = __lsx_vslt_b(v0, (__m128i)v1); // v1 > v0 + const __m128i sh = __lsx_vsub_b((__m128i)v1, v2); + const __m128i compressed = __lsx_vshuf_b(b->chunks[0], b->chunks[0], sh); + __lsx_vst(compressed, reinterpret_cast<__m128i *>(output + 0 * 16), 0); + __lsx_vst(b->chunks[1], reinterpret_cast<__m128i *>(output + 1 * 16 - 1), + 0); + __lsx_vst(b->chunks[2], reinterpret_cast<__m128i *>(output + 2 * 16 - 1), + 0); + __lsx_vst(b->chunks[3], reinterpret_cast<__m128i *>(output + 3 * 16 - 1), + 0); + } break; + + case 0b01: { + __lsx_vst(b->chunks[0], reinterpret_cast<__m128i *>(output + 0 * 16), 0); + + const __m128i v0 = __lsx_vreplgr2vr_b((uint8_t)(pos - 1)); + const __m128i v2 = __lsx_vslt_b(v0, (__m128i)v1); + const __m128i sh = __lsx_vsub_b((__m128i)v1, v2); + const __m128i compressed = __lsx_vshuf_b(b->chunks[1], b->chunks[1], sh); + + __lsx_vst(compressed, reinterpret_cast<__m128i *>(output + 1 * 16), 0); + __lsx_vst(b->chunks[2], reinterpret_cast<__m128i *>(output + 2 * 16 - 1), + 0); + __lsx_vst(b->chunks[3], reinterpret_cast<__m128i *>(output + 3 * 16 - 1), + 0); + } break; + + case 0b10: { + __lsx_vst(b->chunks[0], reinterpret_cast<__m128i *>(output + 0 * 16), 0); + __lsx_vst(b->chunks[1], reinterpret_cast<__m128i *>(output + 1 * 16), 0); + + const __m128i v0 = __lsx_vreplgr2vr_b((uint8_t)(pos - 1)); + const __m128i v2 = __lsx_vslt_b(v0, (__m128i)v1); + const __m128i sh = __lsx_vsub_b((__m128i)v1, v2); + const __m128i compressed = __lsx_vshuf_b(b->chunks[2], b->chunks[2], sh); + + __lsx_vst(compressed, reinterpret_cast<__m128i *>(output + 2 * 16), 0); + __lsx_vst(b->chunks[3], reinterpret_cast<__m128i *>(output + 3 * 16 - 1), + 0); + } break; + + case 0b11: { + __lsx_vst(b->chunks[0], reinterpret_cast<__m128i *>(output + 0 * 16), 0); + __lsx_vst(b->chunks[1], reinterpret_cast<__m128i *>(output + 1 * 16), 0); + __lsx_vst(b->chunks[2], reinterpret_cast<__m128i *>(output + 2 * 16), 0); + + const __m128i v0 = __lsx_vreplgr2vr_b((uint8_t)(pos - 1)); + const __m128i v2 = __lsx_vslt_b(v0, (__m128i)v1); + const __m128i sh = __lsx_vsub_b((__m128i)v1, v2); + const __m128i compressed = __lsx_vshuf_b(b->chunks[3], b->chunks[3], sh); + + __lsx_vst(compressed, reinterpret_cast<__m128i *>(output + 3 * 16), 0); + } break; + } + return 63; +} + // The caller of this function is responsible to ensure that there are 64 bytes // available from reading at src. The data is read into a block64 structure. static inline void load_block(block64 *b, const char *src) { - b->chunks[0] = __lasx_xvld(reinterpret_cast(src), 0); - b->chunks[1] = __lasx_xvld(reinterpret_cast(src), 32); + b->chunks[0] = __lsx_vld(reinterpret_cast(src), 0); + b->chunks[1] = __lsx_vld(reinterpret_cast(src), 16); + b->chunks[2] = __lsx_vld(reinterpret_cast(src), 32); + b->chunks[3] = __lsx_vld(reinterpret_cast(src), 48); } // The caller of this function is responsible to ensure that there are 128 bytes // available from reading at src. The data is read into a block64 structure. static inline void load_block(block64 *b, const char16_t *src) { - __m256i m1 = __lasx_xvld(reinterpret_cast(src), 0); - __m256i m2 = __lasx_xvld(reinterpret_cast(src), 32); - __m256i m3 = __lasx_xvld(reinterpret_cast(src), 64); - __m256i m4 = __lasx_xvld(reinterpret_cast(src), 96); - b->chunks[0] = __lasx_xvpermi_d(__lasx_xvssrlni_bu_h(m2, m1, 0), 0b11011000); - b->chunks[1] = __lasx_xvpermi_d(__lasx_xvssrlni_bu_h(m4, m3, 0), 0b11011000); + __m128i m1 = __lsx_vld(reinterpret_cast(src), 0); + __m128i m2 = __lsx_vld(reinterpret_cast(src), 16); + __m128i m3 = __lsx_vld(reinterpret_cast(src), 32); + __m128i m4 = __lsx_vld(reinterpret_cast(src), 48); + __m128i m5 = __lsx_vld(reinterpret_cast(src), 64); + __m128i m6 = __lsx_vld(reinterpret_cast(src), 80); + __m128i m7 = __lsx_vld(reinterpret_cast(src), 96); + __m128i m8 = __lsx_vld(reinterpret_cast(src), 112); + b->chunks[0] = __lsx_vssrlni_bu_h(m2, m1, 0); + b->chunks[1] = __lsx_vssrlni_bu_h(m4, m3, 0); + b->chunks[2] = __lsx_vssrlni_bu_h(m6, m5, 0); + b->chunks[3] = __lsx_vssrlni_bu_h(m8, m7, 0); } -static inline void base64_decode(char *out, __m256i str) { - __m256i t0 = __lasx_xvor_v( - __lasx_xvslli_w(str, 26), - __lasx_xvslli_w(__lasx_xvand_v(str, lasx_splat_u32(0x0000ff00)), 12)); - __m256i t1 = - __lasx_xvsrli_w(__lasx_xvand_v(str, lasx_splat_u32(0x003f0000)), 2); - __m256i t2 = __lasx_xvor_v(t0, t1); - __m256i t3 = __lasx_xvor_v(t2, __lasx_xvsrli_w(str, 16)); - __m256i pack_shuffle = ____m256i( - (__m128i)v16u8{3, 2, 1, 7, 6, 5, 11, 10, 9, 15, 14, 13, 0, 0, 0, 0}); - t3 = __lasx_xvshuf_b(t3, t3, (__m256i)pack_shuffle); +static inline void base64_decode(char *out, __m128i str) { + __m128i t0 = __lsx_vor_v( + __lsx_vslli_w(str, 26), + __lsx_vslli_w(__lsx_vand_v(str, lsx_splat_u32(0x0000FF00)), 12)); + __m128i t1 = __lsx_vsrli_w(__lsx_vand_v(str, lsx_splat_u32(0x003F0000)), 2); + __m128i t2 = __lsx_vor_v(t0, t1); + __m128i t3 = __lsx_vor_v(t2, __lsx_vsrli_w(str, 16)); + const v16u8 pack_shuffle = {3, 2, 1, 7, 6, 5, 11, 10, + 9, 15, 14, 13, 0, 0, 0, 0}; + t3 = __lsx_vshuf_b(t3, t3, (__m128i)pack_shuffle); // Store the output: - __lsx_vst(lasx_extracti128_lo(t3), out, 0); - __lsx_vst(lasx_extracti128_hi(t3), out, 12); + // we only need 12. + __lsx_vstelm_d(t3, out, 0, 0); + __lsx_vstelm_w(t3, out + 8, 0, 2); } // decode 64 bytes and output 48 bytes static inline void base64_decode_block(char *out, const char *src) { - base64_decode(out, __lasx_xvld(reinterpret_cast(src), 0)); + base64_decode(out, __lsx_vld(reinterpret_cast(src), 0)); + base64_decode(out + 12, + __lsx_vld(reinterpret_cast(src), 16)); base64_decode(out + 24, - __lasx_xvld(reinterpret_cast(src), 32)); + __lsx_vld(reinterpret_cast(src), 32)); + base64_decode(out + 36, + __lsx_vld(reinterpret_cast(src), 48)); } - static inline void base64_decode_block_safe(char *out, const char *src) { - base64_decode(out, __lasx_xvld(reinterpret_cast(src), 0)); - char buffer[32]; - base64_decode(buffer, - __lasx_xvld(reinterpret_cast(src), 32)); - std::memcpy(out + 24, buffer, 24); + base64_decode_block(out, src); } - static inline void base64_decode_block(char *out, block64 *b) { base64_decode(out, b->chunks[0]); - base64_decode(out + 24, b->chunks[1]); + base64_decode(out + 12, b->chunks[1]); + base64_decode(out + 24, b->chunks[2]); + base64_decode(out + 36, b->chunks[3]); } static inline void base64_decode_block_safe(char *out, block64 *b) { - base64_decode(out, b->chunks[0]); - char buffer[32]; - base64_decode(buffer, b->chunks[1]); - std::memcpy(out + 24, buffer, 24); + base64_decode_block(out, b); } template + typename char_type> full_result -compress_decode_base64(char *dst, const chartype *src, size_t srclen, +compress_decode_base64(char *dst, const char_type *src, size_t srclen, base64_options options, last_chunk_handling_options last_chunk_options) { const uint8_t *to_base64 = @@ -20336,19 +20412,15 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, } return {SUCCESS, full_input_length, 0}; } - char *end_of_safe_64byte_zone = - (srclen + 3) / 4 * 3 >= 63 ? dst + (srclen + 3) / 4 * 3 - 63 : dst; - - const chartype *const srcinit = src; + const char_type *const srcinit = src; const char *const dstinit = dst; - const chartype *const srcend = src + srclen; + const char_type *const srcend = src + srclen; - constexpr size_t block_size = 6; - static_assert(block_size >= 2, "block_size must be at least two"); + constexpr size_t block_size = 10; char buffer[block_size * 64]; char *bufferptr = buffer; if (srclen >= 64) { - const chartype *const srcend64 = src + srclen - 64; + const char_type *const srcend64 = src + srclen - 64; while (src <= srcend64) { block64 b; load_block(&b, src); @@ -20356,55 +20428,49 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, bool error = false; uint64_t badcharmask = to_base64_mask(&b, &error); - if (error && !ignore_garbage) { - src -= 64; - while (src < srcend && scalar::base64::is_eight_byte(*src) && - to_base64[uint8_t(*src)] <= 64) { - src++; + if (badcharmask) { + if (error && !ignore_garbage) { + src -= 64; + while (src < srcend && scalar::base64::is_eight_byte(*src) && + to_base64[uint8_t(*src)] <= 64) { + src++; + } + if (src < srcend) { + // should never happen + } + return {error_code::INVALID_BASE64_CHARACTER, size_t(src - srcinit), + size_t(dst - dstinit)}; } - return {error_code::INVALID_BASE64_CHARACTER, size_t(src - srcinit), - size_t(dst - dstinit)}; } + if (badcharmask != 0) { - // optimization opportunity: check for simple masks like those made of - // continuous 1s followed by continuous 0s. And masks containing a - // single bad character. - bufferptr += compress_block(&b, badcharmask, bufferptr); - } else if (bufferptr != buffer) { - copy_block(&b, bufferptr); - bufferptr += 64; - } else { - if (dst >= end_of_safe_64byte_zone) { - base64_decode_block_safe(dst, &b); + if (is_power_of_two(badcharmask)) { + bufferptr += compress_block_single(&b, badcharmask, bufferptr); } else { - base64_decode_block(dst, &b); + bufferptr += compress_block(&b, badcharmask, bufferptr); } - dst += 48; + } else { + // optimization opportunity: if bufferptr == buffer and mask == 0, we + // can avoid the call to compress_block and decode directly. + copy_block(&b, bufferptr); + bufferptr += 64; } if (bufferptr >= (block_size - 1) * 64 + buffer) { - for (size_t i = 0; i < (block_size - 2); i++) { + for (size_t i = 0; i < (block_size - 1); i++) { base64_decode_block(dst, buffer + i * 64); dst += 48; } - if (dst >= end_of_safe_64byte_zone) { - base64_decode_block_safe(dst, buffer + (block_size - 2) * 64); - } else { - base64_decode_block(dst, buffer + (block_size - 2) * 64); - } - dst += 48; std::memcpy(buffer, buffer + (block_size - 1) * 64, 64); // 64 might be too much bufferptr -= (block_size - 1) * 64; } } } - char *buffer_start = buffer; // Optimization note: if this is almost full, then it is worth our // time, otherwise, we should just decode directly. int last_block = (int)((bufferptr - buffer_start) % 64); if (last_block != 0 && srcend - src + last_block >= 64) { - while ((bufferptr - buffer_start) % 64 != 0 && src < srcend) { uint8_t val = to_base64[uint8_t(*src)]; *bufferptr = char(val); @@ -20419,11 +20485,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, } for (; buffer_start + 64 <= bufferptr; buffer_start += 64) { - if (dst >= end_of_safe_64byte_zone) { - base64_decode_block_safe(dst, buffer_start); - } else { - base64_decode_block(dst, buffer_start); - } + base64_decode_block(dst, buffer_start); dst += 48; } if ((bufferptr - buffer_start) % 64 != 0) { @@ -20433,6 +20495,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, (uint32_t(uint8_t(buffer_start[2])) << 1 * 6) + (uint32_t(uint8_t(buffer_start[3])) << 0 * 6)) << 8; + // lsx is little-endian triple = scalar::u32_swap_bytes(triple); std::memcpy(dst, &triple, 4); @@ -20445,6 +20508,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, (uint32_t(uint8_t(buffer_start[2])) << 1 * 6) + (uint32_t(uint8_t(buffer_start[3])) << 0 * 6)) << 8; + // lsx is little-endian triple = scalar::u32_swap_bytes(triple); std::memcpy(dst, &triple, 3); @@ -20504,24 +20568,22 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, } return {SUCCESS, srclen, size_t(dst - dstinit)}; } -/* end file src/lasx/lasx_base64.cpp */ -/* begin file src/lasx/lasx_find.cpp */ +/* end file src/lsx/lsx_base64.cpp */ +/* begin file src/lsx/lsx_find.cpp */ simdutf_really_inline const char *util_find(const char *start, const char *end, char character) noexcept { if (start >= end) return end; - const int step = 32; - __m256i char_vec = __lasx_xvreplgr2vr_b(static_cast(character)); + const int step = 16; + __m128i char_vec = __lsx_vreplgr2vr_b(static_cast(character)); while (end - start >= step) { - __m256i data = __lasx_xvld(reinterpret_cast(start), 0); - __m256i cmp = __lasx_xvseq_b(data, char_vec); - if (__lasx_xbnz_v(cmp)) { - __m256i res = __lasx_xvmsknz_b(cmp); - uint32_t mask0 = __lasx_xvpickve2gr_wu(res, 0); - uint32_t mask1 = __lasx_xvpickve2gr_wu(res, 4); - uint32_t mask = (mask0 | (mask1 << 16)); + __m128i data = __lsx_vld(reinterpret_cast(start), 0); + __m128i cmp = __lsx_vseq_b(data, char_vec); + if (__lsx_bnz_v(cmp)) { + uint16_t mask = + static_cast(__lsx_vpickve2gr_hu(__lsx_vmsknz_b(cmp), 0)); return start + trailing_zeroes(mask); } @@ -20544,17 +20606,15 @@ simdutf_really_inline const char16_t *util_find(const char16_t *start, if (start >= end) return end; - const int step = 16; - __m256i char_vec = __lasx_xvreplgr2vr_h(static_cast(character)); + const int step = 8; + __m128i char_vec = __lsx_vreplgr2vr_h(static_cast(character)); while (end - start >= step) { - __m256i data = __lasx_xvld(reinterpret_cast(start), 0); - __m256i cmp = __lasx_xvseq_h(data, char_vec); - if (__lasx_xbnz_v(cmp)) { - __m256i res = __lasx_xvmsknz_b(cmp); - uint32_t mask0 = __lasx_xvpickve2gr_wu(res, 0); - uint32_t mask1 = __lasx_xvpickve2gr_wu(res, 4); - uint32_t mask = (mask0 | (mask1 << 16)); + __m128i data = __lsx_vld(reinterpret_cast(start), 0); + __m128i cmp = __lsx_vseq_h(data, char_vec); + if (__lsx_bnz_v(cmp)) { + uint16_t mask = + static_cast(__lsx_vpickve2gr_hu(__lsx_vmsknz_b(cmp), 0)); return start + trailing_zeroes(mask) / 2; } @@ -20570,15 +20630,15 @@ simdutf_really_inline const char16_t *util_find(const char16_t *start, return end; } -/* end file src/lasx/lasx_find.cpp */ +/* end file src/lsx/lsx_find.cpp */ } // namespace -} // namespace lasx +} // namespace lsx } // namespace simdutf /* begin file src/generic/buf_block_reader.h */ namespace simdutf { -namespace lasx { +namespace lsx { namespace { // Walks through a buffer in block-sized increments, loading the last part with @@ -20609,40 +20669,6 @@ template struct buf_block_reader { size_t idx; }; -// Routines to print masks and text for debugging bitmask operations -simdutf_unused static char *format_input_text_64(const uint8_t *text) { - static char *buf = - reinterpret_cast(malloc(sizeof(simd8x64) + 1)); - for (size_t i = 0; i < sizeof(simd8x64); i++) { - buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); - } - buf[sizeof(simd8x64)] = '\0'; - return buf; -} - -// Routines to print masks and text for debugging bitmask operations -simdutf_unused static char *format_input_text(const simd8x64 &in) { - static char *buf = - reinterpret_cast(malloc(sizeof(simd8x64) + 1)); - in.store(reinterpret_cast(buf)); - for (size_t i = 0; i < sizeof(simd8x64); i++) { - if (buf[i] < ' ') { - buf[i] = '_'; - } - } - buf[sizeof(simd8x64)] = '\0'; - return buf; -} - -simdutf_unused static char *format_mask(uint64_t mask) { - static char *buf = reinterpret_cast(malloc(64 + 1)); - for (size_t i = 0; i < 64; i++) { - buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; - } - buf[64] = '\0'; - return buf; -} - template simdutf_really_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) @@ -20684,15 +20710,81 @@ simdutf_really_inline void buf_block_reader::advance() { } } // unnamed namespace -} // namespace lasx +} // namespace lsx } // namespace simdutf /* end file src/generic/buf_block_reader.h */ +/* begin file src/generic/base64lengths.h */ +namespace simdutf { +namespace lsx { +namespace { +namespace base64_lengths { + +simdutf_warn_unused size_t binary_length_from_base64(const char *input, + size_t length) { + size_t pos = 0; + size_t count = 0; + for (; pos + 64 <= length; pos += 64) { + simd8x64 block(reinterpret_cast(input + pos)); + uint64_t maybe_base64 = block.gteq(33); // >= 33 which is '!' in ASCII + count += count_ones(maybe_base64); + } + while (pos < length) { + count += (input[pos] > 0x20) ? 1 : 0; + pos++; + } + // Count padding at the end. + size_t padding = 0; + pos = length; + while (pos > 0 && padding < 2) { + char c = input[--pos]; + if (c == '=') { + padding++; + } else if (c > ' ') { + break; + } + } + return ((count - padding) * 3) / 4; +} + +simdutf_warn_unused size_t binary_length_from_base64(const char16_t *input, + size_t length) { + size_t pos = 0; + size_t count = 0; + for (; pos + 32 <= length; pos += 32) { + simd16x32 block(reinterpret_cast(input + pos)); + uint64_t maybe_base64 = block.gteq(33); // >= 33 which is '!' in ASCII + count += count_ones(maybe_base64); + } + while (pos < length) { + count += (input[pos] > 0x20) ? 1 : 0; + pos++; + } + // Count padding at the end. + size_t padding = 0; + pos = length; + while (pos > 0 && padding < 2) { + char16_t c = input[--pos]; + if (c == '=') { + padding++; + } else if (c > ' ') { + break; + } + } + return ((count - padding) * 3) / 4; +} + +} // namespace base64_lengths +} // unnamed namespace +} // namespace lsx +} // namespace simdutf +/* end file src/generic/base64lengths.h */ + // // Implementation-specific overrides // namespace simdutf { -namespace lasx { +namespace lsx { simdutf_warn_unused result implementation::base64_to_binary( const char *input, size_t length, char *output, base64_options options, @@ -20824,6 +20916,13 @@ size_t implementation::binary_to_base64(const char *input, size_t length, } } +size_t implementation::binary_to_base64_with_lines( + const char *input, size_t length, char *output, size_t line_length, + base64_options options) const noexcept { + return scalar::base64::tail_encode_base64_impl(output, input, length, + options, line_length); +} + const char *implementation::find(const char *start, const char *end, char character) const noexcept { return util_find(start, end, character); @@ -20834,14 +20933,404 @@ const char16_t *implementation::find(const char16_t *start, const char16_t *end, return util_find(start, end, character); } -} // namespace lasx +simdutf_warn_unused size_t implementation::binary_length_from_base64( + const char *input, size_t length) const noexcept { + return base64_lengths::binary_length_from_base64(input, length); +} + +simdutf_warn_unused size_t implementation::binary_length_from_base64( + const char16_t *input, size_t length) const noexcept { + return base64_lengths::binary_length_from_base64(input, length); +} + +} // namespace lsx } // namespace simdutf -/* begin file src/simdutf/lasx/end.h */ +/* begin file src/simdutf/lsx/end.h */ #undef SIMDUTF_SIMD_HAS_UNSIGNED_CMP -/* end file src/simdutf/lasx/end.h */ -/* end file src/lasx/implementation.cpp */ +/* end file src/simdutf/lsx/end.h */ +/* end file src/lsx/implementation.cpp */ +#endif + +/* begin file src/simdutf_c.cpp */ +/* begin file include/simdutf_c.h */ +/*** + * simdutf_c.h.h - C API for simdutf + * This is currently experimental. + * We are committed to keeping the C API, but there might be mistakes in our + * implementation. Please report any issues you find. + */ + +#ifndef SIMDUTF_C_H +#define SIMDUTF_C_H + +#include +#include +#include + +#ifdef __has_include + #if __has_include() + #include + #else // __has_include() + #define char16_t uint16_t + #define char32_t uint32_t + #endif // __has_include() +#else // __has_include() + #define char16_t uint16_t + #define char32_t uint32_t +#endif // __has_include + +#ifdef __cplusplus +extern "C" { #endif +/* C-friendly subset of simdutf errors */ +typedef enum simdutf_error_code { + SIMDUTF_ERROR_SUCCESS = 0, + SIMDUTF_ERROR_HEADER_BITS, + SIMDUTF_ERROR_TOO_SHORT, + SIMDUTF_ERROR_TOO_LONG, + SIMDUTF_ERROR_OVERLONG, + SIMDUTF_ERROR_TOO_LARGE, + SIMDUTF_ERROR_SURROGATE, + SIMDUTF_ERROR_INVALID_BASE64_CHARACTER, + SIMDUTF_ERROR_BASE64_INPUT_REMAINDER, + SIMDUTF_ERROR_BASE64_EXTRA_BITS, + SIMDUTF_ERROR_OUTPUT_BUFFER_TOO_SMALL, + SIMDUTF_ERROR_OTHER +} simdutf_error_code; + +typedef struct simdutf_result { + simdutf_error_code error; + size_t count; /* position of error or number of code units validated */ +} simdutf_result; + +typedef struct simdutf_full_result { + simdutf_error_code error; + size_t input_count; /* number of input units consumed */ + size_t output_count; /* number of output bytes written */ +} simdutf_full_result; + +typedef enum simdutf_encoding_type { + SIMDUTF_ENCODING_UNSPECIFIED = 0, + SIMDUTF_ENCODING_UTF8 = 1, + SIMDUTF_ENCODING_UTF16_LE = 2, + SIMDUTF_ENCODING_UTF16_BE = 4, + SIMDUTF_ENCODING_UTF32_LE = 8, + SIMDUTF_ENCODING_UTF32_BE = 16 +} simdutf_encoding_type; + +/* Validate UTF-8: returns true iff input is valid UTF-8 */ +bool simdutf_validate_utf8(const char *buf, size_t len); + +/* Validate UTF-8 with detailed result */ +simdutf_result simdutf_validate_utf8_with_errors(const char *buf, size_t len); + +/* Encoding detection */ +simdutf_encoding_type simdutf_autodetect_encoding(const char *input, + size_t length); +int simdutf_detect_encodings(const char *input, size_t length); + +/* ASCII validation */ +bool simdutf_validate_ascii(const char *buf, size_t len); +simdutf_result simdutf_validate_ascii_with_errors(const char *buf, size_t len); + +/* UTF-16 ASCII checks */ +bool simdutf_validate_utf16_as_ascii(const char16_t *buf, size_t len); +bool simdutf_validate_utf16be_as_ascii(const char16_t *buf, size_t len); +bool simdutf_validate_utf16le_as_ascii(const char16_t *buf, size_t len); + +/* UTF-16/UTF-8/UTF-32 validation (native/endian-specific) */ +bool simdutf_validate_utf16(const char16_t *buf, size_t len); +bool simdutf_validate_utf16le(const char16_t *buf, size_t len); +bool simdutf_validate_utf16be(const char16_t *buf, size_t len); +simdutf_result simdutf_validate_utf16_with_errors(const char16_t *buf, + size_t len); +simdutf_result simdutf_validate_utf16le_with_errors(const char16_t *buf, + size_t len); +simdutf_result simdutf_validate_utf16be_with_errors(const char16_t *buf, + size_t len); + +bool simdutf_validate_utf32(const char32_t *buf, size_t len); +simdutf_result simdutf_validate_utf32_with_errors(const char32_t *buf, + size_t len); + +/* to_well_formed UTF-16 helpers */ +void simdutf_to_well_formed_utf16le(const char16_t *input, size_t len, + char16_t *output); +void simdutf_to_well_formed_utf16be(const char16_t *input, size_t len, + char16_t *output); +void simdutf_to_well_formed_utf16(const char16_t *input, size_t len, + char16_t *output); + +/* Counting */ +size_t simdutf_count_utf16(const char16_t *input, size_t length); +size_t simdutf_count_utf16le(const char16_t *input, size_t length); +size_t simdutf_count_utf16be(const char16_t *input, size_t length); +size_t simdutf_count_utf8(const char *input, size_t length); + +/* Length estimators */ +size_t simdutf_utf8_length_from_latin1(const char *input, size_t length); +size_t simdutf_latin1_length_from_utf8(const char *input, size_t length); +size_t simdutf_latin1_length_from_utf16(size_t length); +size_t simdutf_latin1_length_from_utf32(size_t length); +size_t simdutf_utf16_length_from_utf8(const char *input, size_t length); +size_t simdutf_utf32_length_from_utf8(const char *input, size_t length); +size_t simdutf_utf8_length_from_utf16(const char16_t *input, size_t length); +size_t simdutf_utf8_length_from_utf32(const char32_t *input, size_t length); +simdutf_result +simdutf_utf8_length_from_utf16_with_replacement(const char16_t *input, + size_t length); +size_t simdutf_utf8_length_from_utf16le(const char16_t *input, size_t length); +size_t simdutf_utf8_length_from_utf16be(const char16_t *input, size_t length); +simdutf_result +simdutf_utf8_length_from_utf16le_with_replacement(const char16_t *input, + size_t length); +simdutf_result +simdutf_utf8_length_from_utf16be_with_replacement(const char16_t *input, + size_t length); + +/* Conversions: latin1 <-> utf8, utf8 <-> utf16/utf32, utf16 <-> utf8, etc. */ +size_t simdutf_convert_latin1_to_utf8(const char *input, size_t length, + char *output); +size_t simdutf_convert_latin1_to_utf8_safe(const char *input, size_t length, + char *output, size_t utf8_len); +size_t simdutf_convert_latin1_to_utf16le(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_latin1_to_utf16be(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_latin1_to_utf16(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_latin1_to_utf32(const char *input, size_t length, + char32_t *output); + +size_t simdutf_convert_utf8_to_latin1(const char *input, size_t length, + char *output); +size_t simdutf_convert_utf8_to_utf16le(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_utf8_to_utf16be(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_utf8_to_utf16(const char *input, size_t length, + char16_t *output); + +size_t simdutf_convert_utf8_to_utf32(const char *input, size_t length, + char32_t *output); +simdutf_result simdutf_convert_utf8_to_latin1_with_errors(const char *input, + size_t length, + char *output); +simdutf_result simdutf_convert_utf8_to_utf16_with_errors(const char *input, + size_t length, + char16_t *output); +simdutf_result simdutf_convert_utf8_to_utf16le_with_errors(const char *input, + size_t length, + char16_t *output); +simdutf_result simdutf_convert_utf8_to_utf16be_with_errors(const char *input, + size_t length, + char16_t *output); +simdutf_result simdutf_convert_utf8_to_utf32_with_errors(const char *input, + size_t length, + char32_t *output); + +/* Conversions assuming valid input */ +size_t simdutf_convert_valid_utf8_to_latin1(const char *input, size_t length, + char *output); +size_t simdutf_convert_valid_utf8_to_utf16le(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_valid_utf8_to_utf16be(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_valid_utf8_to_utf32(const char *input, size_t length, + char32_t *output); + +/* UTF-16 -> UTF-8 and related conversions */ +size_t simdutf_convert_utf16_to_utf8(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_utf16le_to_utf8(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_utf16be_to_utf8(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_utf16_to_utf8_safe(const char16_t *input, size_t length, + char *output, size_t utf8_len); +size_t simdutf_convert_utf16_to_latin1(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_utf16le_to_latin1(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_utf16be_to_latin1(const char16_t *input, size_t length, + char *output); +simdutf_result +simdutf_convert_utf16_to_latin1_with_errors(const char16_t *input, + size_t length, char *output); +simdutf_result +simdutf_convert_utf16le_to_latin1_with_errors(const char16_t *input, + size_t length, char *output); +simdutf_result +simdutf_convert_utf16be_to_latin1_with_errors(const char16_t *input, + size_t length, char *output); + +simdutf_result simdutf_convert_utf16_to_utf8_with_errors(const char16_t *input, + size_t length, + char *output); +simdutf_result +simdutf_convert_utf16le_to_utf8_with_errors(const char16_t *input, + size_t length, char *output); +simdutf_result +simdutf_convert_utf16be_to_utf8_with_errors(const char16_t *input, + size_t length, char *output); + +size_t simdutf_convert_valid_utf16_to_utf8(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_valid_utf16_to_latin1(const char16_t *input, + size_t length, char *output); +size_t simdutf_convert_valid_utf16le_to_latin1(const char16_t *input, + size_t length, char *output); +size_t simdutf_convert_valid_utf16be_to_latin1(const char16_t *input, + size_t length, char *output); + +size_t simdutf_convert_valid_utf16le_to_utf8(const char16_t *input, + size_t length, char *output); +size_t simdutf_convert_valid_utf16be_to_utf8(const char16_t *input, + size_t length, char *output); + +/* UTF-16 <-> UTF-32 conversions */ +size_t simdutf_convert_utf16_to_utf32(const char16_t *input, size_t length, + char32_t *output); +size_t simdutf_convert_utf16le_to_utf32(const char16_t *input, size_t length, + char32_t *output); +size_t simdutf_convert_utf16be_to_utf32(const char16_t *input, size_t length, + char32_t *output); +simdutf_result simdutf_convert_utf16_to_utf32_with_errors(const char16_t *input, + size_t length, + char32_t *output); +simdutf_result +simdutf_convert_utf16le_to_utf32_with_errors(const char16_t *input, + size_t length, char32_t *output); +simdutf_result +simdutf_convert_utf16be_to_utf32_with_errors(const char16_t *input, + size_t length, char32_t *output); + +/* Valid UTF-16 conversions */ +size_t simdutf_convert_valid_utf16_to_utf32(const char16_t *input, + size_t length, char32_t *output); +size_t simdutf_convert_valid_utf16le_to_utf32(const char16_t *input, + size_t length, char32_t *output); +size_t simdutf_convert_valid_utf16be_to_utf32(const char16_t *input, + size_t length, char32_t *output); + +/* UTF-32 -> ... conversions */ +size_t simdutf_convert_utf32_to_utf8(const char32_t *input, size_t length, + char *output); +simdutf_result simdutf_convert_utf32_to_utf8_with_errors(const char32_t *input, + size_t length, + char *output); +size_t simdutf_convert_valid_utf32_to_utf8(const char32_t *input, size_t length, + char *output); + +size_t simdutf_convert_utf32_to_utf16(const char32_t *input, size_t length, + char16_t *output); +size_t simdutf_convert_utf32_to_utf16le(const char32_t *input, size_t length, + char16_t *output); +size_t simdutf_convert_utf32_to_utf16be(const char32_t *input, size_t length, + char16_t *output); +simdutf_result +simdutf_convert_utf32_to_latin1_with_errors(const char32_t *input, + size_t length, char *output); + +/* --- Find helpers --- */ +const char *simdutf_find(const char *start, const char *end, char character); +const char16_t *simdutf_find_utf16(const char16_t *start, const char16_t *end, + char16_t character); + +/* --- Base64 enums and helpers --- */ +typedef enum simdutf_base64_options { + SIMDUTF_BASE64_DEFAULT = 0, + SIMDUTF_BASE64_URL = 1, + SIMDUTF_BASE64_DEFAULT_NO_PADDING = 2, + SIMDUTF_BASE64_URL_WITH_PADDING = 3, + SIMDUTF_BASE64_DEFAULT_ACCEPT_GARBAGE = 4, + SIMDUTF_BASE64_URL_ACCEPT_GARBAGE = 5, + SIMDUTF_BASE64_DEFAULT_OR_URL = 8, + SIMDUTF_BASE64_DEFAULT_OR_URL_ACCEPT_GARBAGE = 12 +} simdutf_base64_options; + +typedef enum simdutf_last_chunk_handling_options { + SIMDUTF_LAST_CHUNK_LOOSE = 0, + SIMDUTF_LAST_CHUNK_STRICT = 1, + SIMDUTF_LAST_CHUNK_STOP_BEFORE_PARTIAL = 2, + SIMDUTF_LAST_CHUNK_ONLY_FULL_CHUNKS = 3 +} simdutf_last_chunk_handling_options; + +/* maximal binary length estimators */ +size_t simdutf_maximal_binary_length_from_base64(const char *input, + size_t length); +size_t simdutf_maximal_binary_length_from_base64_utf16(const char16_t *input, + size_t length); + +/* base64 decoding/encoding */ +simdutf_result simdutf_base64_to_binary( + const char *input, size_t length, char *output, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options); +simdutf_result simdutf_base64_to_binary_utf16( + const char16_t *input, size_t length, char *output, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options); + +size_t simdutf_base64_length_from_binary(size_t length, + simdutf_base64_options options); +size_t simdutf_base64_length_from_binary_with_lines( + size_t length, simdutf_base64_options options, size_t line_length); + +size_t simdutf_binary_to_base64(const char *input, size_t length, char *output, + simdutf_base64_options options); +size_t simdutf_binary_to_base64_with_lines(const char *input, size_t length, + char *output, size_t line_length, + simdutf_base64_options options); + +/* safe decoding that provides an in/out outlen parameter */ +simdutf_result simdutf_base64_to_binary_safe( + const char *input, size_t length, char *output, size_t *outlen, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options, + bool decode_up_to_bad_char); +simdutf_result simdutf_base64_to_binary_safe_utf16( + const char16_t *input, size_t length, char *output, size_t *outlen, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options, + bool decode_up_to_bad_char); + +/* detailed decoding returning input_count and output_count */ +simdutf_full_result simdutf_base64_to_binary_details( + const char *input, size_t length, char *output, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options); +simdutf_full_result simdutf_base64_to_binary_details_utf16( + const char16_t *input, size_t length, char *output, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options); + +/* single-character base64 validation */ +bool simdutf_base64_valid(char input, simdutf_base64_options options); +bool simdutf_base64_valid_utf16(char16_t input, simdutf_base64_options options); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* SIMDUTF_C_H */ +/* end file include/simdutf_c.h */ + +static simdutf_result to_c_result(const simdutf::result &r) { + simdutf_result out; + out.error = static_cast(r.error); + out.count = r.count; + return out; +} + +/* The C wrapper depends on the library features. Only expose the C API + when all relevant feature is enabled. This helps the + single-header generator to omit the C wrapper when features are + disabled. */ +// clang-format off +// clang-format on +/* end file src/simdutf_c.cpp */ SIMDUTF_POP_DISABLE_WARNINGS /* end file src/simdutf.cpp */ diff --git a/cpp/simdutf.h b/cpp/simdutf.h index 3f8d4f3..3900b0c 100644 --- a/cpp/simdutf.h +++ b/cpp/simdutf.h @@ -1,4 +1,4 @@ -/* auto-generated on 2025-09-10 16:01:38 -0400. Do not edit! */ +/* auto-generated on 2026-05-07 06:54:06 +0200. Do not edit! */ /* begin file include/simdutf.h */ #ifndef SIMDUTF_H #define SIMDUTF_H @@ -20,6 +20,11 @@ #endif #endif +// C++ 26 +#if !defined(SIMDUTF_CPLUSPLUS26) && (SIMDUTF_CPLUSPLUS >= 202602L) + #define SIMDUTF_CPLUSPLUS26 1 +#endif + // C++ 23 #if !defined(SIMDUTF_CPLUSPLUS23) && (SIMDUTF_CPLUSPLUS >= 202302L) #define SIMDUTF_CPLUSPLUS23 1 @@ -45,8 +50,8 @@ #define SIMDUTF_CPLUSPLUS11 1 #endif -#ifndef SIMDUTF_CPLUSPLUS11 - #error simdutf requires a compiler compliant with the C++11 standard +#ifndef SIMDUTF_CPLUSPLUS17 + #error simdutf requires a compiler compliant with the C++17 standard #endif #endif // SIMDUTF_COMPILER_CHECK_H @@ -63,6 +68,7 @@ #include #include #include +#include #include #ifndef _WIN32 // strcasecmp, strncasecmp @@ -200,9 +206,25 @@ #elif defined(__loongarch_lp64) #if defined(__loongarch_sx) && defined(__loongarch_asx) #define SIMDUTF_IS_LSX 1 - #define SIMDUTF_IS_LASX 1 + #define SIMDUTF_IS_LASX 1 // We can always run both #elif defined(__loongarch_sx) #define SIMDUTF_IS_LSX 1 + // Adjust for runtime dispatching support. + #if defined(__GNUC__) && !defined(__clang__) && \ + !defined(__INTEL_COMPILER) && !defined(__NVCOMPILER) + #if __GNUC__ > 15 || (__GNUC__ == 15 && __GNUC_MINOR__ >= 0) + // We are ok, we will support runtime dispatch for LASX. + #else + // We disable runtime dispatch for LASX, which means that we will not be + // able to use LASX even if it is supported by the hardware. Loongson + // users should update to GCC 15 or better. + #define SIMDUTF_IMPLEMENTATION_LASX 0 + #endif + #else + // We are not using GCC, so we assume that we can support runtime dispatch + // for LASX. https://godbolt.org/z/jcMnrjYhs + #define SIMDUTF_IMPLEMENTATION_LASX 0 + #endif #endif #else // The simdutf library is designed @@ -250,7 +272,7 @@ // // We are going to use runtime dispatch. -#ifdef SIMDUTF_IS_X86_64 +#if defined(SIMDUTF_IS_X86_64) || defined(SIMDUTF_IS_LSX) #ifdef __clang__ // clang does not have GCC push pop // warning: clang attribute push can't be used within a namespace in clang @@ -267,7 +289,7 @@ #define SIMDUTF_UNTARGET_REGION _Pragma("GCC pop_options") #endif // clang then gcc -#endif // x86 +#endif // defined(SIMDUTF_IS_X86_64) || defined(SIMDUTF_IS_LSX) // Default target region macros don't do anything. #ifndef SIMDUTF_TARGET_REGION @@ -526,6 +548,14 @@ #endif // MSC_VER +// Will evaluate to constexpr in C++23 or later. This makes it possible to mark +// functions constexpr if the "if consteval" feature is available to use. +#if SIMDUTF_CPLUSPLUS23 + #define simdutf_constexpr23 constexpr +#else + #define simdutf_constexpr23 +#endif + #ifndef SIMDUTF_DLLIMPORTEXPORT #if defined(SIMDUTF_VISUAL_STUDIO) // Visual Studio /** @@ -576,7 +606,13 @@ /* begin file include/simdutf/encoding_types.h */ #ifndef SIMDUTF_ENCODING_TYPES_H #define SIMDUTF_ENCODING_TYPES_H -#include +#include + +#if !defined(SIMDUTF_NO_STD_TEXT_ENCODING) && \ + defined(__cpp_lib_text_encoding) && __cpp_lib_text_encoding >= 202306L + #define SIMDUTF_HAS_STD_TEXT_ENCODING 1 + #include +#endif namespace simdutf { @@ -591,11 +627,27 @@ enum encoding_type { unspecified = 0 }; -enum endianness { LITTLE = 0, BIG = 1 }; +#ifndef SIMDUTF_IS_BIG_ENDIAN + #error "SIMDUTF_IS_BIG_ENDIAN needs to be defined." +#endif + +enum endianness { + LITTLE = 0, + BIG = 1, + NATIVE = +#if SIMDUTF_IS_BIG_ENDIAN + BIG +#else + LITTLE +#endif +}; -bool match_system(endianness e); +simdutf_warn_unused simdutf_really_inline constexpr bool +match_system(endianness e) { + return e == endianness::NATIVE; +} -std::string to_string(encoding_type bom); +simdutf_warn_unused std::string_view to_string(encoding_type bom); // Note that BOM for UTF8 is discouraged. namespace BOM { @@ -607,23 +659,144 @@ namespace BOM { * @return the corresponding encoding */ -encoding_type check_bom(const uint8_t *byte, size_t length); -encoding_type check_bom(const char *byte, size_t length); +simdutf_warn_unused encoding_type check_bom(const uint8_t *byte, size_t length); +simdutf_warn_unused encoding_type check_bom(const char *byte, size_t length); /** * Returns the size, in bytes, of the BOM for a given encoding type. * Note that UTF8 BOM are discouraged. * @param bom the encoding type * @return the size in bytes of the corresponding BOM */ -size_t bom_byte_size(encoding_type bom); +simdutf_warn_unused size_t bom_byte_size(encoding_type bom); } // namespace BOM + +#ifdef SIMDUTF_HAS_STD_TEXT_ENCODING +/** + * Convert a simdutf encoding type to a std::text_encoding. + * + * @param enc the simdutf encoding type + * @return the corresponding std::text_encoding, or + * std::text_encoding::id::unknown for unspecified/unsupported + */ +simdutf_warn_unused constexpr std::text_encoding +to_std_encoding(encoding_type enc) noexcept { + switch (enc) { + case UTF8: + return std::text_encoding(std::text_encoding::id::UTF8); + case UTF16_LE: + return std::text_encoding(std::text_encoding::id::UTF16LE); + case UTF16_BE: + return std::text_encoding(std::text_encoding::id::UTF16BE); + case UTF32_LE: + return std::text_encoding(std::text_encoding::id::UTF32LE); + case UTF32_BE: + return std::text_encoding(std::text_encoding::id::UTF32BE); + case Latin1: + return std::text_encoding(std::text_encoding::id::ISOLatin1); + case unspecified: + default: + return std::text_encoding(std::text_encoding::id::unknown); + } +} + +/** + * Convert a std::text_encoding to a simdutf encoding type. + * + * @param enc the std::text_encoding + * @return the corresponding simdutf encoding type, or + * encoding_type::unspecified if the encoding is not supported + */ +simdutf_warn_unused constexpr encoding_type +from_std_encoding(const std::text_encoding &enc) noexcept { + switch (enc.mib()) { + case std::text_encoding::id::UTF8: + return UTF8; + case std::text_encoding::id::UTF16LE: + return UTF16_LE; + case std::text_encoding::id::UTF16BE: + return UTF16_BE; + case std::text_encoding::id::UTF32LE: + return UTF32_LE; + case std::text_encoding::id::UTF32BE: + return UTF32_BE; + case std::text_encoding::id::ISOLatin1: + return Latin1; + default: + return unspecified; + } +} + +/** + * Get the native-endian UTF-16 encoding type for this system. + * + * @return UTF16_LE on little-endian systems, UTF16_BE on big-endian systems + */ +simdutf_warn_unused constexpr encoding_type native_utf16_encoding() noexcept { + #if SIMDUTF_IS_BIG_ENDIAN + return UTF16_BE; + #else + return UTF16_LE; + #endif +} + +/** + * Get the native-endian UTF-32 encoding type for this system. + * + * @return UTF32_LE on little-endian systems, UTF32_BE on big-endian systems + */ +simdutf_warn_unused constexpr encoding_type native_utf32_encoding() noexcept { + #if SIMDUTF_IS_BIG_ENDIAN + return UTF32_BE; + #else + return UTF32_LE; + #endif +} + +/** + * Convert a std::text_encoding to a simdutf encoding type, + * using native endianness for UTF-16/UTF-32 without explicit endianness. + * + * When the input is std::text_encoding::id::UTF16 or UTF32 (without LE/BE + * suffix), this returns the native-endian simdutf variant. + * + * @param enc the std::text_encoding + * @return the corresponding simdutf encoding type, or + * encoding_type::unspecified if the encoding is not supported + */ +simdutf_warn_unused constexpr encoding_type +from_std_encoding_native(const std::text_encoding &enc) noexcept { + switch (enc.mib()) { + case std::text_encoding::id::UTF8: + return UTF8; + case std::text_encoding::id::UTF16: + return native_utf16_encoding(); + case std::text_encoding::id::UTF16LE: + return UTF16_LE; + case std::text_encoding::id::UTF16BE: + return UTF16_BE; + case std::text_encoding::id::UTF32: + return native_utf32_encoding(); + case std::text_encoding::id::UTF32LE: + return UTF32_LE; + case std::text_encoding::id::UTF32BE: + return UTF32_BE; + case std::text_encoding::id::ISOLatin1: + return Latin1; + default: + return unspecified; + } +} +#endif // SIMDUTF_HAS_STD_TEXT_ENCODING + } // namespace simdutf #endif /* end file include/simdutf/encoding_types.h */ /* begin file include/simdutf/error.h */ #ifndef SIMDUTF_ERROR_H #define SIMDUTF_ERROR_H +#include + namespace simdutf { enum error_code { @@ -641,9 +814,19 @@ enum error_code { // U+10FFFF,less than or equal than U+7F for ASCII OR less than // equal than U+FF for Latin1 SURROGATE, // The decoded character must be not be in U+D800...DFFF (UTF-8 or - // UTF-32) OR a high surrogate must be followed by a low surrogate + // UTF-32) + // OR + // a high surrogate must be followed by a low surrogate // and a low surrogate must be preceded by a high surrogate - // (UTF-16) OR there must be no surrogate at all (Latin1) + // (UTF-16) + // OR + // there must be no surrogate at all and one is + // found (Latin1 functions) + // OR + // *specifically* for the function + // utf8_length_from_utf16_with_replacement, a surrogate (whether + // in error or not) has been found (I.e., whether we are in the + // Basic Multilingual Plane or not). INVALID_BASE64_CHARACTER, // Found a character that cannot be part of a valid // base64 string. This may include a misplaced // padding character ('='). @@ -655,7 +838,7 @@ enum error_code { OUTPUT_BUFFER_TOO_SMALL, // The provided buffer is too small. OTHER // Not related to validation/transcoding. }; -#if SIMDUTF_CPLUSPLUS17 + inline std::string_view error_to_string(error_code code) noexcept { switch (code) { case SUCCESS: @@ -684,7 +867,6 @@ inline std::string_view error_to_string(error_code code) noexcept { return "OTHER"; } } -#endif struct result { error_code error; @@ -692,17 +874,18 @@ struct result { // case of success, indicates the number of code units // validated/written. - simdutf_really_inline result() noexcept + simdutf_really_inline simdutf_constexpr23 result() noexcept : error{error_code::SUCCESS}, count{0} {} - simdutf_really_inline result(error_code err, size_t pos) noexcept + simdutf_really_inline simdutf_constexpr23 result(error_code err, + size_t pos) noexcept : error{err}, count{pos} {} - simdutf_really_inline bool is_ok() const noexcept { + simdutf_really_inline simdutf_constexpr23 bool is_ok() const noexcept { return error == error_code::SUCCESS; } - simdutf_really_inline bool is_err() const noexcept { + simdutf_really_inline simdutf_constexpr23 bool is_err() const noexcept { return error != error_code::SUCCESS; } }; @@ -714,18 +897,19 @@ struct full_result { bool padding_error = false; // true if the error is due to padding, only // meaningful when error is not SUCCESS - simdutf_really_inline full_result() noexcept + simdutf_really_inline simdutf_constexpr23 full_result() noexcept : error{error_code::SUCCESS}, input_count{0}, output_count{0} {} - simdutf_really_inline full_result(error_code err, size_t pos_in, - size_t pos_out) noexcept + simdutf_really_inline simdutf_constexpr23 full_result(error_code err, + size_t pos_in, + size_t pos_out) noexcept : error{err}, input_count{pos_in}, output_count{pos_out} {} - simdutf_really_inline full_result(error_code err, size_t pos_in, - size_t pos_out, bool padding_err) noexcept + simdutf_really_inline simdutf_constexpr23 full_result( + error_code err, size_t pos_in, size_t pos_out, bool padding_err) noexcept : error{err}, input_count{pos_in}, output_count{pos_out}, padding_error{padding_err} {} - simdutf_really_inline operator result() const noexcept { + simdutf_really_inline simdutf_constexpr23 operator result() const noexcept { if (error == error_code::SUCCESS) { return result{error, output_count}; } else { @@ -749,18 +933,18 @@ SIMDUTF_DISABLE_UNDESIRED_WARNINGS #define SIMDUTF_SIMDUTF_VERSION_H /** The version of simdutf being used (major.minor.revision) */ -#define SIMDUTF_VERSION "7.4.0" +#define SIMDUTF_VERSION "9.0.0" namespace simdutf { enum { /** * The major version (MAJOR.minor.revision) of simdutf being used. */ - SIMDUTF_VERSION_MAJOR = 7, + SIMDUTF_VERSION_MAJOR = 9, /** * The minor version (major.MINOR.revision) of simdutf being used. */ - SIMDUTF_VERSION_MINOR = 4, + SIMDUTF_VERSION_MINOR = 0, /** * The revision (major.minor.REVISION) of simdutf being used. */ @@ -776,7 +960,6 @@ enum { #if !defined(SIMDUTF_NO_THREADS) #include #endif -#include #ifdef SIMDUTF_INTERNAL_TESTS #include #endif @@ -1107,14 +1290,13 @@ static inline uint32_t detect_supported_architectures() { #endif // SIMDutf_INTERNAL_ISADETECTION_H /* end file include/simdutf/internal/isadetection.h */ +#include #if SIMDUTF_SPAN #include #include #include #include -#endif -#if SIMDUTF_CPLUSPLUS17 - #include + #include // for std::unreachable #endif // The following defines are conditionally enabled/disabled during amalgamation. // By default all features are enabled, regular code shouldn't check them. Only @@ -1133,20 +1315,163 @@ static inline uint32_t detect_supported_architectures() { #define SIMDUTF_FEATURE_UTF32 0 #define SIMDUTF_FEATURE_BASE64 1 +#if SIMDUTF_CPLUSPLUS23 +/* begin file include/simdutf/constexpr_ptr.h */ +#ifndef SIMDUTF_CONSTEXPR_PTR_H +#define SIMDUTF_CONSTEXPR_PTR_H + +#include + namespace simdutf { +namespace detail { +/** + * The constexpr_ptr class is a workaround for reinterpret_cast not being + * allowed during constant evaluation. + */ +template + requires(sizeof(to) == sizeof(from)) +struct constexpr_ptr { + const from *p; + + constexpr explicit constexpr_ptr(const from *ptr) noexcept : p(ptr) {} + + constexpr to operator*() const noexcept { return static_cast(*p); } + + constexpr constexpr_ptr &operator++() noexcept { + ++p; + return *this; + } + + constexpr constexpr_ptr operator++(int) noexcept { + auto old = *this; + ++p; + return old; + } + + constexpr constexpr_ptr &operator--() noexcept { + --p; + return *this; + } + + constexpr constexpr_ptr operator--(int) noexcept { + auto old = *this; + --p; + return old; + } + + constexpr constexpr_ptr &operator+=(std::ptrdiff_t n) noexcept { + p += n; + return *this; + } + + constexpr constexpr_ptr &operator-=(std::ptrdiff_t n) noexcept { + p -= n; + return *this; + } + + constexpr constexpr_ptr operator+(std::ptrdiff_t n) const noexcept { + return constexpr_ptr{p + n}; + } + + constexpr constexpr_ptr operator-(std::ptrdiff_t n) const noexcept { + return constexpr_ptr{p - n}; + } + + constexpr std::ptrdiff_t operator-(const constexpr_ptr &o) const noexcept { + return p - o.p; + } + + constexpr to operator[](std::ptrdiff_t n) const noexcept { + return static_cast(*(p + n)); + } + + // to prevent compilation errors for memcpy, even if it is never + // called during constant evaluation + constexpr operator const void *() const noexcept { return p; } +}; + +template +constexpr constexpr_ptr constexpr_cast_ptr(from *p) noexcept { + return constexpr_ptr{p}; +} + +/** + * helper type for constexpr_writeptr, so it is possible to + * do "*ptr = val;" + */ +template +struct constexpr_write_ptr_proxy { + + constexpr explicit constexpr_write_ptr_proxy(TargetType *raw) : p(raw) {} + + constexpr constexpr_write_ptr_proxy &operator=(SrcType v) { + *p = static_cast(v); + return *this; + } + + TargetType *p; +}; + +/** + * helper for working around reinterpret_cast not being allowed during constexpr + * evaluation. will try to act as a SrcType* but actually write to the pointer + * given in the constructor, which is of another type TargetType + */ +template struct constexpr_write_ptr { + constexpr explicit constexpr_write_ptr(TargetType *raw) : p(raw) {} + + constexpr constexpr_write_ptr_proxy operator*() const { + return constexpr_write_ptr_proxy{p}; + } + + constexpr constexpr_write_ptr_proxy + operator[](std::ptrdiff_t n) const { + return constexpr_write_ptr_proxy{p + n}; + } + + constexpr constexpr_write_ptr &operator++() { + ++p; + return *this; + } + + constexpr constexpr_write_ptr operator++(int) { + constexpr_write_ptr old = *this; + ++p; + return old; + } + + constexpr std::ptrdiff_t operator-(const constexpr_write_ptr &other) const { + return p - other.p; + } + + TargetType *p; +}; + +template +constexpr auto constexpr_cast_writeptr(TargetType *raw) { + return constexpr_write_ptr{raw}; +} + +} // namespace detail +} // namespace simdutf +#endif +/* end file include/simdutf/constexpr_ptr.h */ +#endif #if SIMDUTF_SPAN /// helpers placed in namespace detail are not a part of the public API +namespace simdutf { namespace detail { /** * matches a byte, in the many ways C++ allows. note that these * are all distinct types. */ template -concept byte_like = std::is_same_v || // - std::is_same_v || // - std::is_same_v || // - std::is_same_v; +concept byte_like = std::is_same_v || // + std::is_same_v || // + std::is_same_v || // + std::is_same_v || // + std::is_same_v; template concept is_byte_like = byte_like>; @@ -1179,165 +1504,5619 @@ concept output_span_of_byte_like = requires(T &t) { { *t.data() } noexcept -> is_byte_like; { *t.data() } noexcept -> is_mutable; }; -} // namespace detail -#endif /** - * Validate the ASCII string as a UTF-16LE sequence. - * An UTF-16 sequence is considered an ASCII sequence - * if it could be converted to an ASCII string losslessly. - * - * Overridden by each implementation. - * - * @param buf the UTF-16LE string to validate. - * @param len the length of the string in bytes. - * @return true if and only if the string is valid ASCII. + * a pointer like object, when indexed, results in a byte like result. + * valid examples: char*, const char*, std::array + * invalid examples: int*, std::array */ -simdutf_warn_unused bool validate_utf16le_as_ascii(const char16_t *buf, - size_t len) noexcept; -#if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused bool -validate_utf16le_as_ascii(std::span input) noexcept { - return validate_utf16le_as_ascii(input.data(), input.size()); -} -#endif // SIMDUTF_SPAN - - #ifndef SIMDUTF_NEED_TRAILING_ZEROES - #define SIMDUTF_NEED_TRAILING_ZEROES 1 - #endif -// base64_options are used to specify the base64 encoding options. -// ASCII spaces are ' ', '\t', '\n', '\r', '\f' -// garbage characters are characters that are not part of the base64 alphabet -// nor ASCII spaces. -constexpr uint64_t base64_reverse_padding = - 2; /* modifier for base64_default and base64_url */ -enum base64_options : uint64_t { - base64_default = 0, /* standard base64 format (with padding) */ - base64_url = 1, /* base64url format (no padding) */ - base64_default_no_padding = - base64_default | - base64_reverse_padding, /* standard base64 format without padding */ - base64_url_with_padding = - base64_url | base64_reverse_padding, /* base64url with padding */ - base64_default_accept_garbage = - 4, /* standard base64 format accepting garbage characters, the input stops - with the first '=' if any */ - base64_url_accept_garbage = - 5, /* base64url format accepting garbage characters, the input stops with - the first '=' if any */ - base64_default_or_url = - 8, /* standard/base64url hybrid format (only meaningful for decoding!) */ - base64_default_or_url_accept_garbage = - 12, /* standard/base64url hybrid format accepting garbage characters - (only meaningful for decoding!), the input stops with the first '=' - if any */ +template +concept indexes_into_byte_like = requires(InputPtr p) { + { std::decay_t{} } -> simdutf::detail::byte_like; +}; +template +concept indexes_into_utf16 = requires(InputPtr p) { + { std::decay_t{} } -> std::same_as; +}; +template +concept indexes_into_utf32 = requires(InputPtr p) { + { std::decay_t{} } -> std::same_as; }; - #if SIMDUTF_CPLUSPLUS17 -inline std::string_view to_string(base64_options options) { - switch (options) { - case base64_default: - return "base64_default"; - case base64_url: - return "base64_url"; - case base64_reverse_padding: - return "base64_reverse_padding"; - case base64_url_with_padding: - return "base64_url_with_padding"; - case base64_default_accept_garbage: - return "base64_default_accept_garbage"; - case base64_url_accept_garbage: - return "base64_url_accept_garbage"; - case base64_default_or_url: - return "base64_default_or_url"; - case base64_default_or_url_accept_garbage: - return "base64_default_or_url_accept_garbage"; - } - return ""; -} - #endif // SIMDUTF_CPLUSPLUS17 +template +concept index_assignable_from_char = requires(InputPtr p, char s) { + { p[0] = s }; +}; -// last_chunk_handling_options are used to specify the handling of the last -// chunk in base64 decoding. -// https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 -enum last_chunk_handling_options : uint64_t { - loose = 0, /* standard base64 format, decode partial final chunk */ - strict = 1, /* error when the last chunk is partial, 2 or 3 chars, and - unpadded, or non-zero bit padding */ - stop_before_partial = - 2, /* if the last chunk is partial, ignore it (no error) */ - only_full_chunks = - 3 /* only decode full blocks (4 base64 characters, no padding) */ +/** + * a pointer like object that results in a uint32_t when indexed. + * valid examples: uint32_t* + */ +template +concept indexes_into_uint32 = requires(InputPtr p) { + { std::decay_t{} } -> std::same_as; }; +} // namespace detail +} // namespace simdutf +#endif // SIMDUTF_SPAN -inline bool is_partial(last_chunk_handling_options options) { - return (options == stop_before_partial) || (options == only_full_chunks); +// these includes are needed for constexpr support. they are +// not part of the public api. +/* begin file include/simdutf/scalar/swap_bytes.h */ +#ifndef SIMDUTF_SWAP_BYTES_H +#define SIMDUTF_SWAP_BYTES_H + +namespace simdutf { +namespace scalar { + +constexpr inline simdutf_warn_unused uint16_t +u16_swap_bytes(const uint16_t word) { + return uint16_t((word >> 8) | (word << 8)); } - #if SIMDUTF_CPLUSPLUS17 -inline std::string_view to_string(last_chunk_handling_options options) { - switch (options) { - case loose: - return "loose"; - case strict: - return "strict"; - case stop_before_partial: - return "stop_before_partial"; - case only_full_chunks: - return "only_full_chunks"; - } - return ""; +constexpr inline simdutf_warn_unused uint32_t +u32_swap_bytes(const uint32_t word) { + return ((word >> 24) & 0xff) | // move byte 3 to byte 0 + ((word << 8) & 0xff0000) | // move byte 1 to byte 2 + ((word >> 8) & 0xff00) | // move byte 2 to byte 1 + ((word << 24) & 0xff000000); // byte 0 to byte 3 } - #endif -/** - * Provide the maximal binary length in bytes given the base64 input. - * In general, if the input contains ASCII spaces, the result will be less than - * the maximum length. - * - * @param input the base64 input to process - * @param length the length of the base64 input in bytes - * @return maximum number of binary bytes - */ -simdutf_warn_unused size_t -maximal_binary_length_from_base64(const char *input, size_t length) noexcept; - #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t -maximal_binary_length_from_base64( - const detail::input_span_of_byte_like auto &input) noexcept { - return maximal_binary_length_from_base64( - reinterpret_cast(input.data()), input.size()); +namespace utf32 { +template constexpr uint32_t swap_if_needed(uint32_t c) { + return !match_system(big_endian) ? scalar::u32_swap_bytes(c) : c; } - #endif // SIMDUTF_SPAN +} // namespace utf32 -/** - * Provide the maximal binary length in bytes given the base64 input. - * In general, if the input contains ASCII spaces, the result will be less than - * the maximum length. - * - * @param input the base64 input to process, in ASCII stored as 16-bit - * units - * @param length the length of the base64 input in 16-bit units - * @return maximal number of binary bytes - */ -simdutf_warn_unused size_t maximal_binary_length_from_base64( - const char16_t *input, size_t length) noexcept; - #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t -maximal_binary_length_from_base64(std::span input) noexcept { - return maximal_binary_length_from_base64(input.data(), input.size()); +namespace utf16 { +template constexpr uint16_t swap_if_needed(uint16_t c) { + return !match_system(big_endian) ? scalar::u16_swap_bytes(c) : c; } - #endif // SIMDUTF_SPAN +} // namespace utf16 -/** - * Convert a base64 input to a binary output. - * - * This function follows the WHATWG forgiving-base64 format, which means that it - * will ignore any ASCII spaces in the input. You may provide a padded input - * (with one or two equal signs at the end) or an unpadded input (without any - * equal signs at the end). - * - * See https://infra.spec.whatwg.org/#forgiving-base64-decode +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/swap_bytes.h */ +/* begin file include/simdutf/scalar/ascii.h */ +#ifndef SIMDUTF_ASCII_H +#define SIMDUTF_ASCII_H + +namespace simdutf { +namespace scalar { +namespace { +namespace ascii { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_warn_unused simdutf_constexpr23 bool validate(InputPtr data, + size_t len) noexcept { + uint64_t pos = 0; + +#if SIMDUTF_CPLUSPLUS23 + // avoid memcpy during constant evaluation + if !consteval +#endif + // process in blocks of 16 bytes when possible + { + for (; pos + 16 <= len; pos += 16) { + uint64_t v1; + std::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) != 0) { + return false; + } + } + } + + // process the tail byte-by-byte + for (; pos < len; pos++) { + if (static_cast(data[pos]) >= 0b10000000) { + return false; + } + } + return true; +} +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_warn_unused simdutf_constexpr23 result +validate_with_errors(InputPtr data, size_t len) noexcept { + size_t pos = 0; +#if SIMDUTF_CPLUSPLUS23 + // avoid memcpy during constant evaluation + if !consteval +#endif + { + // process in blocks of 16 bytes when possible + for (; pos + 16 <= len; pos += 16) { + uint64_t v1; + std::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) != 0) { + for (; pos < len; pos++) { + if (static_cast(data[pos]) >= 0b10000000) { + return result(error_code::TOO_LARGE, pos); + } + } + } + } + } + + // process the tail byte-by-byte + for (; pos < len; pos++) { + if (static_cast(data[pos]) >= 0b10000000) { + return result(error_code::TOO_LARGE, pos); + } + } + return result(error_code::SUCCESS, pos); +} + +} // namespace ascii +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/ascii.h */ +/* begin file include/simdutf/scalar/atomic_util.h */ +#ifndef SIMDUTF_ATOMIC_UTIL_H +#define SIMDUTF_ATOMIC_UTIL_H +#if SIMDUTF_ATOMIC_REF + #include +namespace simdutf { +namespace scalar { + +// This function is a memcpy that uses atomic operations to read from the +// source. +inline void memcpy_atomic_read(char *dst, const char *src, size_t len) { + static_assert(std::atomic_ref::required_alignment == sizeof(char), + "std::atomic_ref requires the same alignment as char_type"); + // We expect all 64-bit systems to be able to read 64-bit words from an + // aligned memory region atomically. You might be able to do better on + // specific systems, e.g., x64 systems can read 128-bit words atomically. + constexpr size_t alignment = sizeof(uint64_t); + + // Lambda for atomic byte-by-byte copy + auto bbb_memcpy_atomic_read = [](char *bytedst, const char *bytesrc, + size_t bytelen) noexcept { + char *mutable_src = const_cast(bytesrc); + for (size_t j = 0; j < bytelen; ++j) { + bytedst[j] = + std::atomic_ref(mutable_src[j]).load(std::memory_order_relaxed); + } + }; + + // Handle unaligned start + size_t offset = reinterpret_cast(src) % alignment; + if (offset) { + size_t to_align = std::min(len, alignment - offset); + bbb_memcpy_atomic_read(dst, src, to_align); + src += to_align; + dst += to_align; + len -= to_align; + } + + // Process aligned 64-bit chunks + while (len >= alignment) { + auto *src_aligned = reinterpret_cast(const_cast(src)); + const auto dst_value = + std::atomic_ref(*src_aligned).load(std::memory_order_relaxed); + std::memcpy(dst, &dst_value, sizeof(uint64_t)); + src += alignment; + dst += alignment; + len -= alignment; + } + + // Handle remaining bytes + if (len) { + bbb_memcpy_atomic_read(dst, src, len); + } +} + +// This function is a memcpy that uses atomic operations to write to the +// destination. +inline void memcpy_atomic_write(char *dst, const char *src, size_t len) { + static_assert(std::atomic_ref::required_alignment == sizeof(char), + "std::atomic_ref requires the same alignment as char"); + // We expect all 64-bit systems to be able to write 64-bit words to an aligned + // memory region atomically. + // You might be able to do better on specific systems, e.g., x64 systems can + // write 128-bit words atomically. + constexpr size_t alignment = sizeof(uint64_t); + + // Lambda for atomic byte-by-byte write + auto bbb_memcpy_atomic_write = [](char *bytedst, const char *bytesrc, + size_t bytelen) noexcept { + for (size_t j = 0; j < bytelen; ++j) { + std::atomic_ref(bytedst[j]) + .store(bytesrc[j], std::memory_order_relaxed); + } + }; + + // Handle unaligned start + size_t offset = reinterpret_cast(dst) % alignment; + if (offset) { + size_t to_align = std::min(len, alignment - offset); + bbb_memcpy_atomic_write(dst, src, to_align); + dst += to_align; + src += to_align; + len -= to_align; + } + + // Process aligned 64-bit chunks + while (len >= alignment) { + auto *dst_aligned = reinterpret_cast(dst); + uint64_t src_val; + std::memcpy(&src_val, src, sizeof(uint64_t)); // Non-atomic read from src + std::atomic_ref(*dst_aligned) + .store(src_val, std::memory_order_relaxed); + dst += alignment; + src += alignment; + len -= alignment; + } + + // Handle remaining bytes + if (len) { + bbb_memcpy_atomic_write(dst, src, len); + } +} +} // namespace scalar +} // namespace simdutf +#endif // SIMDUTF_ATOMIC_REF +#endif // SIMDUTF_ATOMIC_UTIL_H +/* end file include/simdutf/scalar/atomic_util.h */ +/* begin file include/simdutf/scalar/latin1.h */ +#ifndef SIMDUTF_LATIN1_H +#define SIMDUTF_LATIN1_H + +namespace simdutf { +namespace scalar { +namespace { +namespace latin1 { + +simdutf_really_inline size_t utf8_length_from_latin1(const char *buf, + size_t len) { + const uint8_t *c = reinterpret_cast(buf); + size_t answer = 0; + for (size_t i = 0; i < len; i++) { + if ((c[i] >> 7)) { + answer++; + } + } + return answer + len; +} + +} // namespace latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/latin1.h */ +/* begin file include/simdutf/scalar/latin1_to_utf16/latin1_to_utf16.h */ +#ifndef SIMDUTF_LATIN1_TO_UTF16_H +#define SIMDUTF_LATIN1_TO_UTF16_H + +namespace simdutf { +namespace scalar { +namespace { +namespace latin1_to_utf16 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + + while (pos < len) { + uint16_t word = + uint8_t(data[pos]); // extend Latin-1 char to 16-bit Unicode code point + *utf16_output++ = + char16_t(match_system(big_endian) ? word : u16_swap_bytes(word)); + pos++; + } + + return utf16_output - start; +} + +template +inline result convert_with_errors(const char *buf, size_t len, + char16_t *utf16_output) { + const uint8_t *data = reinterpret_cast(buf); + size_t pos = 0; + char16_t *start{utf16_output}; + + while (pos < len) { + uint16_t word = + uint16_t(data[pos]); // extend Latin-1 char to 16-bit Unicode code point + *utf16_output++ = + char16_t(match_system(big_endian) ? word : u16_swap_bytes(word)); + pos++; + } + + return result(error_code::SUCCESS, utf16_output - start); +} + +} // namespace latin1_to_utf16 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/latin1_to_utf16/latin1_to_utf16.h */ +/* begin file include/simdutf/scalar/latin1_to_utf32/latin1_to_utf32.h */ +#ifndef SIMDUTF_LATIN1_TO_UTF32_H +#define SIMDUTF_LATIN1_TO_UTF32_H + +namespace simdutf { +namespace scalar { +namespace { +namespace latin1_to_utf32 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + char32_t *utf32_output) { + char32_t *start{utf32_output}; + for (size_t i = 0; i < len; i++) { + *utf32_output++ = uint8_t(data[i]); + } + return utf32_output - start; +} + +} // namespace latin1_to_utf32 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/latin1_to_utf32/latin1_to_utf32.h */ +/* begin file include/simdutf/scalar/latin1_to_utf8/latin1_to_utf8.h */ +#ifndef SIMDUTF_LATIN1_TO_UTF8_H +#define SIMDUTF_LATIN1_TO_UTF8_H + +namespace simdutf { +namespace scalar { +namespace { +namespace latin1_to_utf8 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_byte_like && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + OutputPtr utf8_output) { + // const unsigned char *data = reinterpret_cast(buf); + size_t pos = 0; + size_t utf8_pos = 0; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | + v2}; // We are only interested in these bits: 1000 1000 1000 + // 1000, so it makes sense to concatenate everything + if ((v & 0x8080808080808080) == + 0) { // if NONE of these are set, e.g. all of them are zero, then + // everything is ASCII + size_t final_pos = pos + 16; + while (pos < final_pos) { + utf8_output[utf8_pos++] = char(data[pos]); + pos++; + } + continue; + } + } // if (pos + 16 <= len) + } // !consteval scope + + unsigned char byte = data[pos]; + if ((byte & 0x80) == 0) { // if ASCII + // will generate one UTF-8 bytes + utf8_output[utf8_pos++] = char(byte); + pos++; + } else { + // will generate two UTF-8 bytes + utf8_output[utf8_pos++] = char((byte >> 6) | 0b11000000); + utf8_output[utf8_pos++] = char((byte & 0b111111) | 0b10000000); + pos++; + } + } // while + return utf8_pos; +} + +simdutf_really_inline size_t convert(const char *buf, size_t len, + char *utf8_output) { + return convert(reinterpret_cast(buf), len, + utf8_output); +} + +inline size_t convert_safe(const char *buf, size_t len, char *utf8_output, + size_t utf8_len) { + const unsigned char *data = reinterpret_cast(buf); + size_t pos = 0; + size_t skip_pos = 0; + size_t utf8_pos = 0; + while (pos < len && utf8_pos < utf8_len) { + // try to convert the next block of 16 ASCII bytes + if (pos >= skip_pos && pos + 16 <= len && + utf8_pos + 16 <= utf8_len) { // if it is safe to read 16 more bytes, + // check that they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | + v2}; // We are only interested in these bits: 1000 1000 1000 + // 1000, so it makes sense to concatenate everything + if ((v & 0x8080808080808080) == + 0) { // if NONE of these are set, e.g. all of them are zero, then + // everything is ASCII + ::memcpy(utf8_output + utf8_pos, buf + pos, 16); + utf8_pos += 16; + pos += 16; + } else { + // At least one of the next 16 bytes are not ASCII, we will process them + // one by one + skip_pos = pos + 16; + } + } else { + const auto byte = data[pos]; + if ((byte & 0x80) == 0) { // if ASCII + // will generate one UTF-8 bytes + utf8_output[utf8_pos++] = char(byte); + pos++; + } else if (utf8_pos + 2 <= utf8_len) { + // will generate two UTF-8 bytes + utf8_output[utf8_pos++] = char((byte >> 6) | 0b11000000); + utf8_output[utf8_pos++] = char((byte & 0b111111) | 0b10000000); + pos++; + } else { + break; + } + } + } + return utf8_pos; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_byte_like && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 size_t convert_safe_constexpr(InputPtr data, size_t len, + OutputPtr utf8_output, + size_t utf8_len) { + size_t pos = 0; + size_t utf8_pos = 0; + while (pos < len && utf8_pos < utf8_len) { + const unsigned char byte = data[pos]; + if ((byte & 0x80) == 0) { // if ASCII + // will generate one UTF-8 bytes + utf8_output[utf8_pos++] = char(byte); + pos++; + } else if (utf8_pos + 2 <= utf8_len) { + // will generate two UTF-8 bytes + utf8_output[utf8_pos++] = char((byte >> 6) | 0b11000000); + utf8_output[utf8_pos++] = char((byte & 0b111111) | 0b10000000); + pos++; + } else { + break; + } + } + return utf8_pos; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 simdutf_warn_unused size_t +utf8_length_from_latin1(InputPtr input, size_t length) noexcept { + size_t answer = length; + size_t i = 0; + +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + auto pop = [](uint64_t v) { + return (size_t)(((v >> 7) & UINT64_C(0x0101010101010101)) * + UINT64_C(0x0101010101010101) >> + 56); + }; + for (; i + 32 <= length; i += 32) { + uint64_t v; + memcpy(&v, input + i, 8); + answer += pop(v); + memcpy(&v, input + i + 8, sizeof(v)); + answer += pop(v); + memcpy(&v, input + i + 16, sizeof(v)); + answer += pop(v); + memcpy(&v, input + i + 24, sizeof(v)); + answer += pop(v); + } + for (; i + 8 <= length; i += 8) { + uint64_t v; + memcpy(&v, input + i, sizeof(v)); + answer += pop(v); + } + } // !consteval scope + for (; i + 1 <= length; i += 1) { + answer += static_cast(input[i]) >> 7; + } + return answer; +} + +} // namespace latin1_to_utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/latin1_to_utf8/latin1_to_utf8.h */ +/* begin file include/simdutf/scalar/utf16.h */ +#ifndef SIMDUTF_UTF16_H +#define SIMDUTF_UTF16_H + +namespace simdutf { +namespace scalar { +namespace utf16 { + +template +simdutf_warn_unused simdutf_constexpr23 bool +validate_as_ascii(const char16_t *data, size_t len) noexcept { + for (size_t pos = 0; pos < len; pos++) { + char16_t word = scalar::utf16::swap_if_needed(data[pos]); + if (word >= 0x80) { + return false; + } + } + return true; +} + +template +inline simdutf_warn_unused simdutf_constexpr23 bool +validate(const char16_t *data, size_t len) noexcept { + uint64_t pos = 0; + while (pos < len) { + char16_t word = scalar::utf16::swap_if_needed(data[pos]); + if ((word & 0xF800) == 0xD800) { + if (pos + 1 >= len) { + return false; + } + char16_t diff = char16_t(word - 0xD800); + if (diff > 0x3FF) { + return false; + } + char16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + char16_t diff2 = char16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return false; + } + pos += 2; + } else { + pos++; + } + } + return true; +} + +template +inline simdutf_warn_unused simdutf_constexpr23 result +validate_with_errors(const char16_t *data, size_t len) noexcept { + size_t pos = 0; + while (pos < len) { + char16_t word = scalar::utf16::swap_if_needed(data[pos]); + if ((word & 0xF800) == 0xD800) { + if (pos + 1 >= len) { + return result(error_code::SURROGATE, pos); + } + char16_t diff = char16_t(word - 0xD800); + if (diff > 0x3FF) { + return result(error_code::SURROGATE, pos); + } + char16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + char16_t diff2 = uint16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return result(error_code::SURROGATE, pos); + } + pos += 2; + } else { + pos++; + } + } + return result(error_code::SUCCESS, pos); +} + +template +simdutf_constexpr23 size_t count_code_points(const char16_t *p, size_t len) { + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + char16_t word = scalar::utf16::swap_if_needed(p[i]); + counter += ((word & 0xFC00) != 0xDC00); + } + return counter; +} + +template +simdutf_constexpr23 size_t utf8_length_from_utf16(const char16_t *p, + size_t len) { + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + char16_t word = scalar::utf16::swap_if_needed(p[i]); + counter++; // ASCII + counter += static_cast( + word > + 0x7F); // non-ASCII is at least 2 bytes, surrogates are 2*2 == 4 bytes + counter += static_cast((word > 0x7FF && word <= 0xD7FF) || + (word >= 0xE000)); // three-byte + } + return counter; +} + +template +simdutf_constexpr23 size_t utf32_length_from_utf16(const char16_t *p, + size_t len) { + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + char16_t word = scalar::utf16::swap_if_needed(p[i]); + counter += ((word & 0xFC00) != 0xDC00); + } + return counter; +} + +simdutf_really_inline simdutf_constexpr23 void +change_endianness_utf16(const char16_t *input, size_t size, char16_t *output) { + for (size_t i = 0; i < size; i++) { + *output++ = char16_t(input[i] >> 8 | input[i] << 8); + } +} + +template +simdutf_warn_unused simdutf_constexpr23 size_t +trim_partial_utf16(const char16_t *input, size_t length) { + if (length == 0) { + return 0; + } + uint16_t last_word = uint16_t(input[length - 1]); + last_word = scalar::utf16::swap_if_needed(last_word); + length -= ((last_word & 0xFC00) == 0xD800); + return length; +} + +template constexpr bool is_high_surrogate(char16_t c) { + c = scalar::utf16::swap_if_needed(c); + return (0xd800 <= c && c <= 0xdbff); +} + +template constexpr bool is_low_surrogate(char16_t c) { + c = scalar::utf16::swap_if_needed(c); + return (0xdc00 <= c && c <= 0xdfff); +} + +simdutf_really_inline constexpr bool high_surrogate(char16_t c) { + return (0xd800 <= c && c <= 0xdbff); +} + +simdutf_really_inline constexpr bool low_surrogate(char16_t c) { + return (0xdc00 <= c && c <= 0xdfff); +} + +template +simdutf_constexpr23 result +utf8_length_from_utf16_with_replacement(const char16_t *p, size_t len) { + bool any_surrogates = false; + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + if (is_high_surrogate(p[i])) { + any_surrogates = true; + // surrogate pair + if (i + 1 < len && is_low_surrogate(p[i + 1])) { + counter += 4; + i++; // skip low surrogate + } else { + counter += 3; // unpaired high surrogate replaced by U+FFFD + } + continue; + } else if (is_low_surrogate(p[i])) { + any_surrogates = true; + counter += 3; // unpaired low surrogate replaced by U+FFFD + continue; + } + char16_t word = !match_system(big_endian) ? u16_swap_bytes(p[i]) : p[i]; + counter++; // at least 1 byte + counter += + static_cast(word > 0x7F); // non-ASCII is at least 2 bytes + counter += static_cast(word > 0x7FF); // three-byte + } + return {any_surrogates ? error_code::SURROGATE : error_code::SUCCESS, + counter}; +} + +// variable templates are a C++14 extension +template constexpr char16_t replacement() { + return !match_system(big_endian) ? scalar::u16_swap_bytes(0xfffd) : 0xfffd; +} + +template +simdutf_constexpr23 void to_well_formed_utf16(const char16_t *input, size_t len, + char16_t *output) { + const char16_t replacement = utf16::replacement(); + bool high_surrogate_prev = false, high_surrogate, low_surrogate; + size_t i = 0; + for (; i < len; i++) { + char16_t c = input[i]; + high_surrogate = is_high_surrogate(c); + low_surrogate = is_low_surrogate(c); + if (high_surrogate_prev && !low_surrogate) { + output[i - 1] = replacement; + } + + if (!high_surrogate_prev && low_surrogate) { + output[i] = replacement; + } else { + output[i] = input[i]; + } + high_surrogate_prev = high_surrogate; + } + + /* string may not end with high surrogate */ + if (high_surrogate_prev) { + output[i - 1] = replacement; + } +} + +} // namespace utf16 +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16.h */ +/* begin file include/simdutf/scalar/utf16_to_latin1/utf16_to_latin1.h */ +#ifndef SIMDUTF_UTF16_TO_LATIN1_H +#define SIMDUTF_UTF16_TO_LATIN1_H + +#include // for std::memcpy + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_latin1 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf16 && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + OutputPtr latin_output) { + if (len == 0) { + return 0; + } + size_t pos = 0; + const auto latin_output_start = latin_output; + uint16_t word = 0; + uint16_t too_large = 0; + + while (pos < len) { + word = !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + too_large |= word; + *latin_output++ = char(word & 0xFF); + pos++; + } + if ((too_large & 0xFF00) != 0) { + return 0; + } + + return latin_output - latin_output_start; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf16 && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, + OutputPtr latin_output) { + if (len == 0) { + return result(error_code::SUCCESS, 0); + } + size_t pos = 0; + auto start = latin_output; + uint16_t word; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + if (pos + 16 <= len) { // if it is safe to read 32 more bytes, check that + // they are Latin1 + uint64_t v1, v2, v3, v4; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + ::memcpy(&v2, data + pos + 4, sizeof(uint64_t)); + ::memcpy(&v3, data + pos + 8, sizeof(uint64_t)); + ::memcpy(&v4, data + pos + 12, sizeof(uint64_t)); + + if constexpr (!match_system(big_endian)) { + v1 = (v1 >> 8) | (v1 << (64 - 8)); + } + if constexpr (!match_system(big_endian)) { + v2 = (v2 >> 8) | (v2 << (64 - 8)); + } + if constexpr (!match_system(big_endian)) { + v3 = (v3 >> 8) | (v3 << (64 - 8)); + } + if constexpr (!match_system(big_endian)) { + v4 = (v4 >> 8) | (v4 << (64 - 8)); + } + + if (((v1 | v2 | v3 | v4) & 0xFF00FF00FF00FF00) == 0) { + size_t final_pos = pos + 16; + while (pos < final_pos) { + *latin_output++ = !match_system(big_endian) + ? char(u16_swap_bytes(data[pos])) + : char(data[pos]); + pos++; + } + continue; + } + } + } + + word = !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xFF00) == 0) { + *latin_output++ = char(word & 0xFF); + pos++; + } else { + return result(error_code::TOO_LARGE, pos); + } + } + return result(error_code::SUCCESS, latin_output - start); +} + +} // namespace utf16_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_latin1/utf16_to_latin1.h */ +/* begin file include/simdutf/scalar/utf16_to_latin1/valid_utf16_to_latin1.h */ +#ifndef SIMDUTF_VALID_UTF16_TO_LATIN1_H +#define SIMDUTF_VALID_UTF16_TO_LATIN1_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_latin1 { + +template +simdutf_constexpr23 inline size_t +convert_valid_impl(InputIterator data, size_t len, + OutputIterator latin_output) { + static_assert( + std::is_same::type, uint16_t>::value, + "must decay to uint16_t"); + size_t pos = 0; + const auto start = latin_output; + uint16_t word = 0; + + while (pos < len) { + word = !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + *latin_output++ = char(word); + pos++; + } + + return latin_output - start; +} + +template +simdutf_really_inline size_t convert_valid(const char16_t *buf, size_t len, + char *latin_output) { + return convert_valid_impl(reinterpret_cast(buf), + len, latin_output); +} +} // namespace utf16_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_latin1/valid_utf16_to_latin1.h */ +/* begin file include/simdutf/scalar/utf16_to_utf32/utf16_to_utf32.h */ +#ifndef SIMDUTF_UTF16_TO_UTF32_H +#define SIMDUTF_UTF16_TO_UTF32_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_utf32 { + +template +simdutf_constexpr23 size_t convert(const char16_t *data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xF800) != 0xD800) { + // No surrogate pair, extend 16-bit word to 32-bit word + *utf32_output++ = char32_t(word); + pos++; + } else { + // must be a surrogate pair + uint16_t diff = uint16_t(word - 0xD800); + if (diff > 0x3FF) { + return 0; + } + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return 0; + } + uint32_t value = (diff << 10) + diff2 + 0x10000; + *utf32_output++ = char32_t(value); + pos += 2; + } + } + return utf32_output - start; +} + +template +simdutf_constexpr23 result convert_with_errors(const char16_t *data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xF800) != 0xD800) { + // No surrogate pair, extend 16-bit word to 32-bit word + *utf32_output++ = char32_t(word); + pos++; + } else { + // must be a surrogate pair + uint16_t diff = uint16_t(word - 0xD800); + if (diff > 0x3FF) { + return result(error_code::SURROGATE, pos); + } + if (pos + 1 >= len) { + return result(error_code::SURROGATE, pos); + } // minimal bound checking + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return result(error_code::SURROGATE, pos); + } + uint32_t value = (diff << 10) + diff2 + 0x10000; + *utf32_output++ = char32_t(value); + pos += 2; + } + } + return result(error_code::SUCCESS, utf32_output - start); +} + +} // namespace utf16_to_utf32 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_utf32/utf16_to_utf32.h */ +/* begin file include/simdutf/scalar/utf16_to_utf32/valid_utf16_to_utf32.h */ +#ifndef SIMDUTF_VALID_UTF16_TO_UTF32_H +#define SIMDUTF_VALID_UTF16_TO_UTF32_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_utf32 { + +template +simdutf_constexpr23 size_t convert_valid(const char16_t *data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xF800) != 0xD800) { + // No surrogate pair, extend 16-bit word to 32-bit word + *utf32_output++ = char32_t(word); + pos++; + } else { + // must be a surrogate pair + uint16_t diff = uint16_t(word - 0xD800); + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + uint32_t value = (diff << 10) + diff2 + 0x10000; + *utf32_output++ = char32_t(value); + pos += 2; + } + } + return utf32_output - start; +} + +} // namespace utf16_to_utf32 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_utf32/valid_utf16_to_utf32.h */ +/* begin file include/simdutf/scalar/utf16_to_utf8/utf16_to_utf8.h */ +#ifndef SIMDUTF_UTF16_TO_UTF8_H +#define SIMDUTF_UTF16_TO_UTF8_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_utf8 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_utf16 +// FIXME constrain output as well +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + OutputPtr utf8_output) { + size_t pos = 0; + const auto start = utf8_output; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 8 bytes + if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if constexpr (!match_system(big_endian)) { + v = (v >> 8) | (v << (64 - 8)); + } + if ((v & 0xFF80FF80FF80FF80) == 0) { + size_t final_pos = pos + 4; + while (pos < final_pos) { + *utf8_output++ = !match_system(big_endian) + ? char(u16_swap_bytes(data[pos])) + : char(data[pos]); + pos++; + } + continue; + } + } + } + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xFF80) == 0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if ((word & 0xF800) != 0xD800) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // must be a surrogate pair + if (pos + 1 >= len) { + return 0; + } + uint16_t diff = uint16_t(word - 0xD800); + if (diff > 0x3FF) { + return 0; + } + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return 0; + } + uint32_t value = (diff << 10) + diff2 + 0x10000; + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((value >> 18) | 0b11110000); + *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((value & 0b111111) | 0b10000000); + pos += 2; + } + } + return utf8_output - start; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf16 && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 full_result convert_with_errors(InputPtr data, size_t len, + OutputPtr utf8_output, + size_t utf8_len = 0) { + if (check_output && utf8_len == 0) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, 0, 0); + } + + size_t pos = 0; + auto start = utf8_output; + auto end = utf8_output + utf8_len; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 8 bytes + if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if constexpr (!match_system(big_endian)) + v = (v >> 8) | (v << (64 - 8)); + if ((v & 0xFF80FF80FF80FF80) == 0) { + size_t final_pos = pos + 4; + while (pos < final_pos) { + if (check_output && size_t(end - utf8_output) < 1) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, + utf8_output - start); + } + *utf8_output++ = !match_system(big_endian) + ? char(u16_swap_bytes(data[pos])) + : char(data[pos]); + pos++; + } + continue; + } + } + } + + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xFF80) == 0) { + // will generate one UTF-8 bytes + if (check_output && size_t(end - utf8_output) < 1) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, + utf8_output - start); + } + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + if (check_output && size_t(end - utf8_output) < 2) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, + utf8_output - start); + } + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + + } else if ((word & 0xF800) != 0xD800) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + if (check_output && size_t(end - utf8_output) < 3) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, + utf8_output - start); + } + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + + if (check_output && size_t(end - utf8_output) < 4) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, + utf8_output - start); + } + // must be a surrogate pair + if (pos + 1 >= len) { + return full_result(error_code::SURROGATE, pos, utf8_output - start); + } + uint16_t diff = uint16_t(word - 0xD800); + if (diff > 0x3FF) { + return full_result(error_code::SURROGATE, pos, utf8_output - start); + } + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return full_result(error_code::SURROGATE, pos, utf8_output - start); + } + uint32_t value = (diff << 10) + diff2 + 0x10000; + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((value >> 18) | 0b11110000); + *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((value & 0b111111) | 0b10000000); + pos += 2; + } + } + return full_result(error_code::SUCCESS, pos, utf8_output - start); +} + +template +inline result simple_convert_with_errors(const char16_t *buf, size_t len, + char *utf8_output) { + return convert_with_errors(buf, len, utf8_output, 0); +} + +template +simdutf_constexpr23 size_t convert_with_replacement(const char16_t *data, + size_t len, + char *utf8_output) { + size_t pos = 0; + char *start = utf8_output; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 8 bytes + if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if constexpr (!match_system(big_endian)) { + v = (v >> 8) | (v << (64 - 8)); + } + if ((v & 0xFF80FF80FF80FF80) == 0) { + size_t final_pos = pos + 4; + while (pos < final_pos) { + *utf8_output++ = !match_system(big_endian) + ? char(u16_swap_bytes(data[pos])) + : char(data[pos]); + pos++; + } + continue; + } + } + } + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xFF80) == 0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if ((word & 0xF800) != 0xD800) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // surrogate range + uint16_t diff = uint16_t(word - 0xD800); + if (diff <= 0x3FF && pos + 1 < len) { + // high surrogate, check for valid pair + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if (diff2 <= 0x3FF) { + // valid surrogate pair + uint32_t value = (diff << 10) + diff2 + 0x10000; + // will generate four UTF-8 bytes + *utf8_output++ = char((value >> 18) | 0b11110000); + *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((value & 0b111111) | 0b10000000); + pos += 2; + continue; + } + } + // unpaired surrogate: replace with U+FFFD (0xEF 0xBF 0xBD) + *utf8_output++ = char(0xef); + *utf8_output++ = char(0xbf); + *utf8_output++ = char(0xbd); + pos++; + } + } + return utf8_output - start; +} + +} // namespace utf16_to_utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_utf8/utf16_to_utf8.h */ +/* begin file include/simdutf/scalar/utf16_to_utf8/valid_utf16_to_utf8.h */ +#ifndef SIMDUTF_VALID_UTF16_TO_UTF8_H +#define SIMDUTF_VALID_UTF16_TO_UTF8_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_utf8 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf16 && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, + OutputPtr utf8_output) { + size_t pos = 0; + auto start = utf8_output; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 4 ASCII characters + if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if constexpr (!match_system(big_endian)) { + v = (v >> 8) | (v << (64 - 8)); + } + if ((v & 0xFF80FF80FF80FF80) == 0) { + size_t final_pos = pos + 4; + while (pos < final_pos) { + *utf8_output++ = !match_system(big_endian) + ? char(u16_swap_bytes(data[pos])) + : char(data[pos]); + pos++; + } + continue; + } + } + } + + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xFF80) == 0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if ((word & 0xF800) != 0xD800) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // must be a surrogate pair + uint16_t diff = uint16_t(word - 0xD800); + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + uint32_t value = (diff << 10) + diff2 + 0x10000; + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((value >> 18) | 0b11110000); + *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((value & 0b111111) | 0b10000000); + pos += 2; + } + } + return utf8_output - start; +} + +} // namespace utf16_to_utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_utf8/valid_utf16_to_utf8.h */ +/* begin file include/simdutf/scalar/utf32.h */ +#ifndef SIMDUTF_UTF32_H +#define SIMDUTF_UTF32_H + +namespace simdutf { +namespace scalar { +namespace utf32 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_uint32 +#endif +simdutf_warn_unused simdutf_constexpr23 bool validate(InputPtr data, + size_t len) noexcept { + uint64_t pos = 0; + for (; pos < len; pos++) { + uint32_t word = data[pos]; + if (word > 0x10FFFF || (word >= 0xD800 && word <= 0xDFFF)) { + return false; + } + } + return true; +} + +simdutf_warn_unused simdutf_really_inline bool validate(const char32_t *buf, + size_t len) noexcept { + return validate(reinterpret_cast(buf), len); +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_uint32 +#endif +simdutf_warn_unused simdutf_constexpr23 result +validate_with_errors(InputPtr data, size_t len) noexcept { + size_t pos = 0; + for (; pos < len; pos++) { + uint32_t word = data[pos]; + if (word > 0x10FFFF) { + return result(error_code::TOO_LARGE, pos); + } + if (word >= 0xD800 && word <= 0xDFFF) { + return result(error_code::SURROGATE, pos); + } + } + return result(error_code::SUCCESS, pos); +} + +simdutf_warn_unused simdutf_really_inline result +validate_with_errors(const char32_t *buf, size_t len) noexcept { + return validate_with_errors(reinterpret_cast(buf), len); +} + +inline simdutf_constexpr23 size_t utf8_length_from_utf32(const char32_t *p, + size_t len) { + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + // credit: @ttsugriy for the vectorizable approach + counter++; // ASCII + counter += static_cast(p[i] > 0x7F); // two-byte + counter += static_cast(p[i] > 0x7FF); // three-byte + counter += static_cast(p[i] > 0xFFFF); // four-bytes + } + return counter; +} + +inline simdutf_warn_unused simdutf_constexpr23 size_t +utf16_length_from_utf32(const char32_t *p, size_t len) { + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + counter++; // non-surrogate word + counter += static_cast(p[i] > 0xFFFF); // surrogate pair + } + return counter; +} + +} // namespace utf32 +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32.h */ +/* begin file include/simdutf/scalar/utf32_to_latin1/utf32_to_latin1.h */ +#ifndef SIMDUTF_UTF32_TO_LATIN1_H +#define SIMDUTF_UTF32_TO_LATIN1_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_latin1 { + +inline simdutf_constexpr23 size_t convert(const char32_t *data, size_t len, + char *latin1_output) { + char *start = latin1_output; + uint32_t utf32_char; + size_t pos = 0; + uint32_t too_large = 0; + + while (pos < len) { + utf32_char = (uint32_t)data[pos]; + too_large |= utf32_char; + *latin1_output++ = (char)(utf32_char & 0xFF); + pos++; + } + if ((too_large & 0xFFFFFF00) != 0) { + return 0; + } + return latin1_output - start; +} + +inline simdutf_constexpr23 result convert_with_errors(const char32_t *data, + size_t len, + char *latin1_output) { + char *start{latin1_output}; + size_t pos = 0; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that + // they are Latin1 + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0xFFFFFF00FFFFFF00) == 0) { + *latin1_output++ = char(data[pos]); + *latin1_output++ = char(data[pos + 1]); + pos += 2; + continue; + } + } + } + + uint32_t utf32_char = data[pos]; + if ((utf32_char & 0xFFFFFF00) == + 0) { // Check if the character can be represented in Latin-1 + *latin1_output++ = (char)(utf32_char & 0xFF); + pos++; + } else { + return result(error_code::TOO_LARGE, pos); + }; + } + return result(error_code::SUCCESS, latin1_output - start); +} + +} // namespace utf32_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_latin1/utf32_to_latin1.h */ +/* begin file include/simdutf/scalar/utf32_to_latin1/valid_utf32_to_latin1.h */ +#ifndef SIMDUTF_VALID_UTF32_TO_LATIN1_H +#define SIMDUTF_VALID_UTF32_TO_LATIN1_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_latin1 { + +template +simdutf_constexpr23 size_t convert_valid(ReadPtr data, size_t len, + WritePtr latin1_output) { + static_assert( + std::is_same::type, uint32_t>::value, + "dereferencing the data pointer must result in a uint32_t"); + auto start = latin1_output; + uint32_t utf32_char; + size_t pos = 0; + + while (pos < len) { + utf32_char = data[pos]; + +#if SIMDUTF_CPLUSPLUS23 + // avoid using the 8 byte at a time optimization in constant evaluation + // mode. memcpy can't be used and replacing it with bitwise or gave worse + // codegen (when not during constant evaluation). + if !consteval { +#endif + if (pos + 2 <= len) { + // if it is safe to read 8 more bytes, check that they are Latin1 + uint64_t v; + std::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0xFFFFFF00FFFFFF00) == 0) { + *latin1_output++ = char(data[pos]); + *latin1_output++ = char(data[pos + 1]); + pos += 2; + continue; + } else { + // output can not be represented in latin1 + return 0; + } + } +#if SIMDUTF_CPLUSPLUS23 + } // if ! consteval +#endif + if ((utf32_char & 0xFFFFFF00) == 0) { + *latin1_output++ = char(utf32_char); + } else { + // output can not be represented in latin1 + return 0; + } + pos++; + } + return latin1_output - start; +} + +simdutf_really_inline size_t convert_valid(const char32_t *buf, size_t len, + char *latin1_output) { + return convert_valid(reinterpret_cast(buf), len, + latin1_output); +} + +} // namespace utf32_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_latin1/valid_utf32_to_latin1.h */ +/* begin file include/simdutf/scalar/utf32_to_utf16/utf32_to_utf16.h */ +#ifndef SIMDUTF_UTF32_TO_UTF16_H +#define SIMDUTF_UTF32_TO_UTF16_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_utf16 { + +template +simdutf_constexpr23 size_t convert(const char32_t *data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { + uint32_t word = data[pos]; + if ((word & 0xFFFF0000) == 0) { + if (word >= 0xD800 && word <= 0xDFFF) { + return 0; + } + // will not generate a surrogate pair + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(uint16_t(word))) + : char16_t(word); + } else { + // will generate a surrogate pair + if (word > 0x10FFFF) { + return 0; + } + word -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); + if constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + } + pos++; + } + return utf16_output - start; +} + +template +simdutf_constexpr23 result convert_with_errors(const char32_t *data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { + uint32_t word = data[pos]; + if ((word & 0xFFFF0000) == 0) { + if (word >= 0xD800 && word <= 0xDFFF) { + return result(error_code::SURROGATE, pos); + } + // will not generate a surrogate pair + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(uint16_t(word))) + : char16_t(word); + } else { + // will generate a surrogate pair + if (word > 0x10FFFF) { + return result(error_code::TOO_LARGE, pos); + } + word -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); + if constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + } + pos++; + } + return result(error_code::SUCCESS, utf16_output - start); +} + +} // namespace utf32_to_utf16 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_utf16/utf32_to_utf16.h */ +/* begin file include/simdutf/scalar/utf32_to_utf16/valid_utf32_to_utf16.h */ +#ifndef SIMDUTF_VALID_UTF32_TO_UTF16_H +#define SIMDUTF_VALID_UTF32_TO_UTF16_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_utf16 { + +template +simdutf_constexpr23 size_t convert_valid(const char32_t *data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { + uint32_t word = data[pos]; + if ((word & 0xFFFF0000) == 0) { + // will not generate a surrogate pair + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(uint16_t(word))) + : char16_t(word); + pos++; + } else { + // will generate a surrogate pair + word -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); + if constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + pos++; + } + } + return utf16_output - start; +} + +} // namespace utf32_to_utf16 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_utf16/valid_utf32_to_utf16.h */ +/* begin file include/simdutf/scalar/utf32_to_utf8/utf32_to_utf8.h */ +#ifndef SIMDUTF_UTF32_TO_UTF8_H +#define SIMDUTF_UTF32_TO_UTF8_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_utf8 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf32 && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + OutputPtr utf8_output) { + size_t pos = 0; + auto start = utf8_output; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { // try to convert the next block of 2 ASCII characters + if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0xFFFFFF80FFFFFF80) == 0) { + *utf8_output++ = char(data[pos]); + *utf8_output++ = char(data[pos + 1]); + pos += 2; + continue; + } + } + } + + uint32_t word = data[pos]; + if ((word & 0xFFFFFF80) == 0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xFFFFF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if ((word & 0xFFFF0000) == 0) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + if (word >= 0xD800 && word <= 0xDFFF) { + return 0; + } + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + if (word > 0x10FFFF) { + return 0; + } + *utf8_output++ = char((word >> 18) | 0b11110000); + *utf8_output++ = char(((word >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } + } + return utf8_output - start; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf32 && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, + OutputPtr utf8_output) { + size_t pos = 0; + auto start = utf8_output; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { // try to convert the next block of 2 ASCII characters + if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0xFFFFFF80FFFFFF80) == 0) { + *utf8_output++ = char(data[pos]); + *utf8_output++ = char(data[pos + 1]); + pos += 2; + continue; + } + } + } + + uint32_t word = data[pos]; + if ((word & 0xFFFFFF80) == 0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xFFFFF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if ((word & 0xFFFF0000) == 0) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + if (word >= 0xD800 && word <= 0xDFFF) { + return result(error_code::SURROGATE, pos); + } + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + if (word > 0x10FFFF) { + return result(error_code::TOO_LARGE, pos); + } + *utf8_output++ = char((word >> 18) | 0b11110000); + *utf8_output++ = char(((word >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } + } + return result(error_code::SUCCESS, utf8_output - start); +} + +} // namespace utf32_to_utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_utf8/utf32_to_utf8.h */ +/* begin file include/simdutf/scalar/utf32_to_utf8/valid_utf32_to_utf8.h */ +#ifndef SIMDUTF_VALID_UTF32_TO_UTF8_H +#define SIMDUTF_VALID_UTF32_TO_UTF8_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_utf8 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf32 && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, + OutputPtr utf8_output) { + size_t pos = 0; + auto start = utf8_output; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { // try to convert the next block of 2 ASCII characters + if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0xFFFFFF80FFFFFF80) == 0) { + *utf8_output++ = char(data[pos]); + *utf8_output++ = char(data[pos + 1]); + pos += 2; + continue; + } + } + } + + uint32_t word = data[pos]; + if ((word & 0xFFFFFF80) == 0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xFFFFF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if ((word & 0xFFFF0000) == 0) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 18) | 0b11110000); + *utf8_output++ = char(((word >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } + } + return utf8_output - start; +} + +} // namespace utf32_to_utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_utf8/valid_utf32_to_utf8.h */ +/* begin file include/simdutf/scalar/utf8.h */ +#ifndef SIMDUTF_UTF8_H +#define SIMDUTF_UTF8_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8 { + +// credit: based on code from Google Fuchsia (Apache Licensed) +template +simdutf_constexpr23 simdutf_warn_unused bool validate(BytePtr data, + size_t len) noexcept { + static_assert( + std::is_same::type, uint8_t>::value, + "dereferencing the data pointer must result in a uint8_t"); + uint64_t pos = 0; + uint32_t code_point = 0; + while (pos < len) { + uint64_t next_pos; +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { // check if the next 16 bytes are ascii. + next_pos = pos + 16; + if (next_pos <= len) { // if it is safe to read 16 more bytes, check + // that they are ascii + uint64_t v1{}; + std::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2{}; + std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + pos = next_pos; + continue; + } + } + } + + unsigned char byte = data[pos]; + + while (byte < 0b10000000) { + if (++pos == len) { + return true; + } + byte = data[pos]; + } + + if ((byte & 0b11100000) == 0b11000000) { + next_pos = pos + 2; + if (next_pos > len) { + return false; + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return false; + } + // range check + code_point = (byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); + if ((code_point < 0x80) || (0x7ff < code_point)) { + return false; + } + } else if ((byte & 0b11110000) == 0b11100000) { + next_pos = pos + 3; + if (next_pos > len) { + return false; + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return false; + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return false; + } + // range check + code_point = (byte & 0b00001111) << 12 | + (data[pos + 1] & 0b00111111) << 6 | + (data[pos + 2] & 0b00111111); + if ((code_point < 0x800) || (0xffff < code_point) || + (0xd7ff < code_point && code_point < 0xe000)) { + return false; + } + } else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000 + next_pos = pos + 4; + if (next_pos > len) { + return false; + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return false; + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return false; + } + if ((data[pos + 3] & 0b11000000) != 0b10000000) { + return false; + } + // range check + code_point = + (byte & 0b00000111) << 18 | (data[pos + 1] & 0b00111111) << 12 | + (data[pos + 2] & 0b00111111) << 6 | (data[pos + 3] & 0b00111111); + if (code_point <= 0xffff || 0x10ffff < code_point) { + return false; + } + } else { + // we may have a continuation + return false; + } + pos = next_pos; + } + return true; +} + +simdutf_really_inline simdutf_warn_unused bool validate(const char *buf, + size_t len) noexcept { + return validate(reinterpret_cast(buf), len); +} + +template +simdutf_constexpr23 simdutf_warn_unused result +validate_with_errors(BytePtr data, size_t len) noexcept { + static_assert( + std::is_same::type, uint8_t>::value, + "dereferencing the data pointer must result in a uint8_t"); + size_t pos = 0; + uint32_t code_point = 0; + while (pos < len) { + // check of the next 16 bytes are ascii. + size_t next_pos = pos + 16; + if (next_pos <= + len) { // if it is safe to read 16 more bytes, check that they are ascii + uint64_t v1; + std::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + pos = next_pos; + continue; + } + } + unsigned char byte = data[pos]; + + while (byte < 0b10000000) { + if (++pos == len) { + return result(error_code::SUCCESS, len); + } + byte = data[pos]; + } + + if ((byte & 0b11100000) == 0b11000000) { + next_pos = pos + 2; + if (next_pos > len) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + code_point = (byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); + if ((code_point < 0x80) || (0x7ff < code_point)) { + return result(error_code::OVERLONG, pos); + } + } else if ((byte & 0b11110000) == 0b11100000) { + next_pos = pos + 3; + if (next_pos > len) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + code_point = (byte & 0b00001111) << 12 | + (data[pos + 1] & 0b00111111) << 6 | + (data[pos + 2] & 0b00111111); + if ((code_point < 0x800) || (0xffff < code_point)) { + return result(error_code::OVERLONG, pos); + } + if (0xd7ff < code_point && code_point < 0xe000) { + return result(error_code::SURROGATE, pos); + } + } else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000 + next_pos = pos + 4; + if (next_pos > len) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 3] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + code_point = + (byte & 0b00000111) << 18 | (data[pos + 1] & 0b00111111) << 12 | + (data[pos + 2] & 0b00111111) << 6 | (data[pos + 3] & 0b00111111); + if (code_point <= 0xffff) { + return result(error_code::OVERLONG, pos); + } + if (0x10ffff < code_point) { + return result(error_code::TOO_LARGE, pos); + } + } else { + // we either have too many continuation bytes or an invalid leading byte + if ((byte & 0b11000000) == 0b10000000) { + return result(error_code::TOO_LONG, pos); + } else { + return result(error_code::HEADER_BITS, pos); + } + } + pos = next_pos; + } + return result(error_code::SUCCESS, len); +} + +simdutf_really_inline simdutf_warn_unused result +validate_with_errors(const char *buf, size_t len) noexcept { + return validate_with_errors(reinterpret_cast(buf), len); +} + +// Finds the previous leading byte starting backward from buf and validates with +// errors from there Used to pinpoint the location of an error when an invalid +// chunk is detected We assume that the stream starts with a leading byte, and +// to check that it is the case, we ask that you pass a pointer to the start of +// the stream (start). +inline simdutf_warn_unused result rewind_and_validate_with_errors( + const char *start, const char *buf, size_t len) noexcept { + // First check that we start with a leading byte + if ((*start & 0b11000000) == 0b10000000) { + return result(error_code::TOO_LONG, 0); + } + size_t extra_len{0}; + // A leading byte cannot be further than 4 bytes away + for (int i = 0; i < 5; i++) { + unsigned char byte = *buf; + if ((byte & 0b11000000) != 0b10000000) { + break; + } else { + buf--; + extra_len++; + } + } + + result res = validate_with_errors(buf, len + extra_len); + res.count -= extra_len; + return res; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t count_code_points(InputPtr data, size_t len) { + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + // -65 is 0b10111111, anything larger in two-complement's should start a new + // code point. + if (int8_t(data[i]) > -65) { + counter++; + } + } + return counter; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t utf16_length_from_utf8(InputPtr data, size_t len) { + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + if (int8_t(data[i]) > -65) { + counter++; + } + if (uint8_t(data[i]) >= 240) { + counter++; + } + } + return counter; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_warn_unused simdutf_constexpr23 size_t +trim_partial_utf8(InputPtr input, size_t length) { + if (length < 3) { + switch (length) { + case 2: + if (uint8_t(input[length - 1]) >= 0xc0) { + return length - 1; + } // 2-, 3- and 4-byte characters with only 1 byte left + if (uint8_t(input[length - 2]) >= 0xe0) { + return length - 2; + } // 3- and 4-byte characters with only 2 bytes left + return length; + case 1: + if (uint8_t(input[length - 1]) >= 0xc0) { + return length - 1; + } // 2-, 3- and 4-byte characters with only 1 byte left + return length; + case 0: + return length; + } + } + if (uint8_t(input[length - 1]) >= 0xc0) { + return length - 1; + } // 2-, 3- and 4-byte characters with only 1 byte left + if (uint8_t(input[length - 2]) >= 0xe0) { + return length - 2; + } // 3- and 4-byte characters with only 1 byte left + if (uint8_t(input[length - 3]) >= 0xf0) { + return length - 3; + } // 4-byte characters with only 3 bytes left + return length; +} + +} // namespace utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8.h */ +/* begin file include/simdutf/scalar/utf8_to_latin1/utf8_to_latin1.h */ +#ifndef SIMDUTF_UTF8_TO_LATIN1_H +#define SIMDUTF_UTF8_TO_LATIN1_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_latin1 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_byte_like && + simdutf::detail::indexes_into_byte_like) +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + OutputPtr latin_output) { + size_t pos = 0; + auto start = latin_output; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; // We are only interested in these bits: 1000 1000 + // 1000 1000 .... etc + if ((v & 0x8080808080808080) == + 0) { // if NONE of these are set, e.g. all of them are zero, then + // everything is ASCII + size_t final_pos = pos + 16; + while (pos < final_pos) { + *latin_output++ = char(data[pos]); + pos++; + } + continue; + } + } + } + + // suppose it is not an all ASCII byte sequence + uint8_t leading_byte = data[pos]; // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *latin_output++ = char(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == + 0b11000000) { // the first three bits indicate: + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return 0; + } // checks if the next byte is a valid continuation byte in UTF-8. A + // valid continuation byte starts with 10. + // range check - + uint32_t code_point = + (leading_byte & 0b00011111) << 6 | + (data[pos + 1] & + 0b00111111); // assembles the Unicode code point from the two bytes. + // It does this by discarding the leading 110 and 10 + // bits from the two bytes, shifting the remaining bits + // of the first byte, and then combining the results + // with a bitwise OR operation. + if (code_point < 0x80 || 0xFF < code_point) { + return 0; // We only care about the range 129-255 which is Non-ASCII + // latin1 characters. A code_point beneath 0x80 is invalid as + // it is already covered by bytes whose leading bit is zero. + } + *latin_output++ = char(code_point); + pos += 2; + } else { + return 0; + } + } + return latin_output - start; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, + char *latin_output) { + size_t pos = 0; + char *start{latin_output}; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; // We are only interested in these bits: 1000 1000 + // 1000 1000...etc + if ((v & 0x8080808080808080) == + 0) { // if NONE of these are set, e.g. all of them are zero, then + // everything is ASCII + size_t final_pos = pos + 16; + while (pos < final_pos) { + *latin_output++ = char(data[pos]); + pos++; + } + continue; + } + } + } + // suppose it is not an all ASCII byte sequence + uint8_t leading_byte = data[pos]; // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *latin_output++ = char(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == + 0b11000000) { // the first three bits indicate: + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } // checks if the next byte is a valid continuation byte in UTF-8. A + // valid continuation byte starts with 10. + // range check - + uint32_t code_point = + (leading_byte & 0b00011111) << 6 | + (data[pos + 1] & + 0b00111111); // assembles the Unicode code point from the two bytes. + // It does this by discarding the leading 110 and 10 + // bits from the two bytes, shifting the remaining bits + // of the first byte, and then combining the results + // with a bitwise OR operation. + if (code_point < 0x80) { + return result(error_code::OVERLONG, pos); + } + if (0xFF < code_point) { + return result(error_code::TOO_LARGE, pos); + } // We only care about the range 129-255 which is Non-ASCII latin1 + // characters + *latin_output++ = char(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8 + return result(error_code::TOO_LARGE, pos); + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + return result(error_code::TOO_LARGE, pos); + } else { + // we either have too many continuation bytes or an invalid leading byte + if ((leading_byte & 0b11000000) == 0b10000000) { + return result(error_code::TOO_LONG, pos); + } + + return result(error_code::HEADER_BITS, pos); + } + } + return result(error_code::SUCCESS, latin_output - start); +} + +inline result rewind_and_convert_with_errors(size_t prior_bytes, + const char *buf, size_t len, + char *latin1_output) { + size_t extra_len{0}; + // We potentially need to go back in time and find a leading byte. + // In theory '3' would be sufficient, but sometimes the error can go back + // quite far. + size_t how_far_back = prior_bytes; + // size_t how_far_back = 3; // 3 bytes in the past + current position + // if(how_far_back >= prior_bytes) { how_far_back = prior_bytes; } + bool found_leading_bytes{false}; + // important: it is i <= how_far_back and not 'i < how_far_back'. + for (size_t i = 0; i <= how_far_back; i++) { + unsigned char byte = buf[-static_cast(i)]; + found_leading_bytes = ((byte & 0b11000000) != 0b10000000); + if (found_leading_bytes) { + if (i > 0 && byte < 128) { + // If we had to go back and the leading byte is ascii + // then we can stop right away. + return result(error_code::TOO_LONG, 0 - i + 1); + } + buf -= i; + extra_len = i; + break; + } + } + // + // It is possible for this function to return a negative count in its result. + // C++ Standard Section 18.1 defines size_t is in which is described + // in C Standard as . C Standard Section 4.1.5 defines size_t as an + // unsigned integral type of the result of the sizeof operator + // + // An unsigned type will simply wrap round arithmetically (well defined). + // + if (!found_leading_bytes) { + // If how_far_back == 3, we may have four consecutive continuation bytes!!! + // [....] [continuation] [continuation] [continuation] | [buf is + // continuation] Or we possibly have a stream that does not start with a + // leading byte. + return result(error_code::TOO_LONG, 0 - how_far_back); + } + result res = convert_with_errors(buf, len + extra_len, latin1_output); + if (res.error) { + res.count -= extra_len; + } + return res; +} + +} // namespace utf8_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_latin1/utf8_to_latin1.h */ +/* begin file include/simdutf/scalar/utf8_to_latin1/valid_utf8_to_latin1.h */ +#ifndef SIMDUTF_VALID_UTF8_TO_LATIN1_H +#define SIMDUTF_VALID_UTF8_TO_LATIN1_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_latin1 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, + char *latin_output) { + + size_t pos = 0; + char *start{latin_output}; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | + v2}; // We are only interested in these bits: 1000 1000 1000 + // 1000, so it makes sense to concatenate everything + if ((v & 0x8080808080808080) == + 0) { // if NONE of these are set, e.g. all of them are zero, then + // everything is ASCII + size_t final_pos = pos + 16; + while (pos < final_pos) { + *latin_output++ = uint8_t(data[pos]); + pos++; + } + continue; + } + } + } + + // suppose it is not an all ASCII byte sequence + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *latin_output++ = char(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == + 0b11000000) { // the first three bits indicate: + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + break; + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return 0; + } // checks if the next byte is a valid continuation byte in UTF-8. A + // valid continuation byte starts with 10. + // range check - + uint32_t code_point = + (leading_byte & 0b00011111) << 6 | + (uint8_t(data[pos + 1]) & + 0b00111111); // assembles the Unicode code point from the two bytes. + // It does this by discarding the leading 110 and 10 + // bits from the two bytes, shifting the remaining bits + // of the first byte, and then combining the results + // with a bitwise OR operation. + *latin_output++ = char(code_point); + pos += 2; + } else { + // we may have a continuation but we do not do error checking + return 0; + } + } + return latin_output - start; +} + +} // namespace utf8_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_latin1/valid_utf8_to_latin1.h */ +/* begin file include/simdutf/scalar/utf8_to_utf16/utf8_to_utf16.h */ +#ifndef SIMDUTF_UTF8_TO_UTF16_H +#define SIMDUTF_UTF8_TO_UTF16_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_utf16 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + // try to convert the next block of 16 ASCII bytes + { + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 16; + while (pos < final_pos) { + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(data[pos])) + : char16_t(data[pos]); + pos++; + } + continue; + } + } + } + + uint8_t leading_byte = data[pos]; // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(leading_byte)) + : char16_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return 0; + } + // range check + uint32_t code_point = + (leading_byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); + if (code_point < 0x80 || 0x7ff < code_point) { + return 0; + } + if constexpr (!match_system(big_endian)) { + code_point = uint32_t(u16_swap_bytes(uint16_t(code_point))); + } + *utf16_output++ = char16_t(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 2 >= len) { + return 0; + } // minimal bound checking + + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return 0; + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return 0; + } + // range check + uint32_t code_point = (leading_byte & 0b00001111) << 12 | + (data[pos + 1] & 0b00111111) << 6 | + (data[pos + 2] & 0b00111111); + if (code_point < 0x800 || 0xffff < code_point || + (0xd7ff < code_point && code_point < 0xe000)) { + return 0; + } + if constexpr (!match_system(big_endian)) { + code_point = uint32_t(u16_swap_bytes(uint16_t(code_point))); + } + *utf16_output++ = char16_t(code_point); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + return 0; + } // minimal bound checking + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return 0; + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return 0; + } + if ((data[pos + 3] & 0b11000000) != 0b10000000) { + return 0; + } + + // range check + uint32_t code_point = (leading_byte & 0b00000111) << 18 | + (data[pos + 1] & 0b00111111) << 12 | + (data[pos + 2] & 0b00111111) << 6 | + (data[pos + 3] & 0b00111111); + if (code_point <= 0xffff || 0x10ffff < code_point) { + return 0; + } + code_point -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (code_point >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (code_point & 0x3FF)); + if constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + pos += 4; + } else { + return 0; + } + } + return utf16_output - start; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 16; + while (pos < final_pos) { + const char16_t byte = uint8_t(data[pos]); + *utf16_output++ = + !match_system(big_endian) ? u16_swap_bytes(byte) : byte; + pos++; + } + continue; + } + } + } + + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(leading_byte)) + : char16_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 1 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + uint32_t code_point = (leading_byte & 0b00011111) << 6 | + (uint8_t(data[pos + 1]) & 0b00111111); + if (code_point < 0x80 || 0x7ff < code_point) { + return result(error_code::OVERLONG, pos); + } + if constexpr (!match_system(big_endian)) { + code_point = uint32_t(u16_swap_bytes(uint16_t(code_point))); + } + *utf16_output++ = char16_t(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 2 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + uint32_t code_point = (leading_byte & 0b00001111) << 12 | + (uint8_t(data[pos + 1]) & 0b00111111) << 6 | + (uint8_t(data[pos + 2]) & 0b00111111); + if ((code_point < 0x800) || (0xffff < code_point)) { + return result(error_code::OVERLONG, pos); + } + if (0xd7ff < code_point && code_point < 0xe000) { + return result(error_code::SURROGATE, pos); + } + if constexpr (!match_system(big_endian)) { + code_point = uint32_t(u16_swap_bytes(uint16_t(code_point))); + } + *utf16_output++ = char16_t(code_point); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 3]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + + // range check + uint32_t code_point = (leading_byte & 0b00000111) << 18 | + (uint8_t(data[pos + 1]) & 0b00111111) << 12 | + (uint8_t(data[pos + 2]) & 0b00111111) << 6 | + (uint8_t(data[pos + 3]) & 0b00111111); + if (code_point <= 0xffff) { + return result(error_code::OVERLONG, pos); + } + if (0x10ffff < code_point) { + return result(error_code::TOO_LARGE, pos); + } + code_point -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (code_point >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (code_point & 0x3FF)); + if constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + pos += 4; + } else { + // we either have too many continuation bytes or an invalid leading byte + if ((leading_byte & 0b11000000) == 0b10000000) { + return result(error_code::TOO_LONG, pos); + } else { + return result(error_code::HEADER_BITS, pos); + } + } + } + return result(error_code::SUCCESS, utf16_output - start); +} + +/** + * When rewind_and_convert_with_errors is called, we are pointing at 'buf' and + * we have up to len input bytes left, and we encountered some error. It is + * possible that the error is at 'buf' exactly, but it could also be in the + * previous bytes (up to 3 bytes back). + * + * prior_bytes indicates how many bytes, prior to 'buf' may belong to the + * current memory section and can be safely accessed. We prior_bytes to access + * safely up to three bytes before 'buf'. + * + * The caller is responsible to ensure that len > 0. + * + * If the error is believed to have occurred prior to 'buf', the count value + * contain in the result will be SIZE_T - 1, SIZE_T - 2, or SIZE_T - 3. + */ +template +inline result rewind_and_convert_with_errors(size_t prior_bytes, + const char *buf, size_t len, + char16_t *utf16_output) { + size_t extra_len{0}; + // We potentially need to go back in time and find a leading byte. + // In theory '3' would be sufficient, but sometimes the error can go back + // quite far. + size_t how_far_back = prior_bytes; + // size_t how_far_back = 3; // 3 bytes in the past + current position + // if(how_far_back >= prior_bytes) { how_far_back = prior_bytes; } + bool found_leading_bytes{false}; + // important: it is i <= how_far_back and not 'i < how_far_back'. + for (size_t i = 0; i <= how_far_back; i++) { + unsigned char byte = buf[-static_cast(i)]; + found_leading_bytes = ((byte & 0b11000000) != 0b10000000); + if (found_leading_bytes) { + if (i > 0 && byte < 128) { + // If we had to go back and the leading byte is ascii + // then we can stop right away. + return result(error_code::TOO_LONG, 0 - i + 1); + } + buf -= i; + extra_len = i; + break; + } + } + // + // It is possible for this function to return a negative count in its result. + // C++ Standard Section 18.1 defines size_t is in which is described + // in C Standard as . C Standard Section 4.1.5 defines size_t as an + // unsigned integral type of the result of the sizeof operator + // + // An unsigned type will simply wrap round arithmetically (well defined). + // + if (!found_leading_bytes) { + // If how_far_back == 3, we may have four consecutive continuation bytes!!! + // [....] [continuation] [continuation] [continuation] | [buf is + // continuation] Or we possibly have a stream that does not start with a + // leading byte. + return result(error_code::TOO_LONG, 0 - how_far_back); + } + result res = convert_with_errors(buf, len + extra_len, utf16_output); + if (res.error) { + res.count -= extra_len; + } + return res; +} + +} // namespace utf8_to_utf16 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_utf16/utf8_to_utf16.h */ +/* begin file include/simdutf/scalar/utf8_to_utf16/valid_utf8_to_utf16.h */ +#ifndef SIMDUTF_VALID_UTF8_TO_UTF16_H +#define SIMDUTF_VALID_UTF8_TO_UTF16_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_utf16 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { // try to convert the next block of 8 ASCII bytes + if (pos + 8 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 8; + while (pos < final_pos) { + const char16_t byte = uint8_t(data[pos]); + *utf16_output++ = + !match_system(big_endian) ? u16_swap_bytes(byte) : byte; + pos++; + } + continue; + } + } + } + + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(leading_byte)) + : char16_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 1 >= len) { + break; + } // minimal bound checking + uint16_t code_point = uint16_t(((leading_byte & 0b00011111) << 6) | + (uint8_t(data[pos + 1]) & 0b00111111)); + if constexpr (!match_system(big_endian)) { + code_point = u16_swap_bytes(uint16_t(code_point)); + } + *utf16_output++ = char16_t(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 2 >= len) { + break; + } // minimal bound checking + uint16_t code_point = + uint16_t(((leading_byte & 0b00001111) << 12) | + ((uint8_t(data[pos + 1]) & 0b00111111) << 6) | + (uint8_t(data[pos + 2]) & 0b00111111)); + if constexpr (!match_system(big_endian)) { + code_point = u16_swap_bytes(uint16_t(code_point)); + } + *utf16_output++ = char16_t(code_point); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + break; + } // minimal bound checking + uint32_t code_point = ((leading_byte & 0b00000111) << 18) | + ((uint8_t(data[pos + 1]) & 0b00111111) << 12) | + ((uint8_t(data[pos + 2]) & 0b00111111) << 6) | + (uint8_t(data[pos + 3]) & 0b00111111); + code_point -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (code_point >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (code_point & 0x3FF)); + if constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + pos += 4; + } else { + // we may have a continuation but we do not do error checking + return 0; + } + } + return utf16_output - start; +} + +} // namespace utf8_to_utf16 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_utf16/valid_utf8_to_utf16.h */ +/* begin file include/simdutf/scalar/utf8_to_utf32/utf8_to_utf32.h */ +#ifndef SIMDUTF_UTF8_TO_UTF32_H +#define SIMDUTF_UTF8_TO_UTF32_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_utf32 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 16; + while (pos < final_pos) { + *utf32_output++ = uint8_t(data[pos]); + pos++; + } + continue; + } + } + } + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf32_output++ = char32_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return 0; + } + // range check + uint32_t code_point = (leading_byte & 0b00011111) << 6 | + (uint8_t(data[pos + 1]) & 0b00111111); + if (code_point < 0x80 || 0x7ff < code_point) { + return 0; + } + *utf32_output++ = char32_t(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8 + if (pos + 2 >= len) { + return 0; + } // minimal bound checking + + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return 0; + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return 0; + } + // range check + uint32_t code_point = (leading_byte & 0b00001111) << 12 | + (uint8_t(data[pos + 1]) & 0b00111111) << 6 | + (uint8_t(data[pos + 2]) & 0b00111111); + if (code_point < 0x800 || 0xffff < code_point || + (0xd7ff < code_point && code_point < 0xe000)) { + return 0; + } + *utf32_output++ = char32_t(code_point); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + return 0; + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return 0; + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return 0; + } + if ((uint8_t(data[pos + 3]) & 0b11000000) != 0b10000000) { + return 0; + } + + // range check + uint32_t code_point = (leading_byte & 0b00000111) << 18 | + (uint8_t(data[pos + 1]) & 0b00111111) << 12 | + (uint8_t(data[pos + 2]) & 0b00111111) << 6 | + (uint8_t(data[pos + 3]) & 0b00111111); + if (code_point <= 0xffff || 0x10ffff < code_point) { + return 0; + } + *utf32_output++ = char32_t(code_point); + pos += 4; + } else { + return 0; + } + } + return utf32_output - start; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 16; + while (pos < final_pos) { + *utf32_output++ = uint8_t(data[pos]); + pos++; + } + continue; + } + } + } + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf32_output++ = char32_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + uint32_t code_point = (leading_byte & 0b00011111) << 6 | + (uint8_t(data[pos + 1]) & 0b00111111); + if (code_point < 0x80 || 0x7ff < code_point) { + return result(error_code::OVERLONG, pos); + } + *utf32_output++ = char32_t(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8 + if (pos + 2 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + uint32_t code_point = (leading_byte & 0b00001111) << 12 | + (uint8_t(data[pos + 1]) & 0b00111111) << 6 | + (uint8_t(data[pos + 2]) & 0b00111111); + if (code_point < 0x800 || 0xffff < code_point) { + return result(error_code::OVERLONG, pos); + } + if (0xd7ff < code_point && code_point < 0xe000) { + return result(error_code::SURROGATE, pos); + } + *utf32_output++ = char32_t(code_point); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 3]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + + // range check + uint32_t code_point = (leading_byte & 0b00000111) << 18 | + (uint8_t(data[pos + 1]) & 0b00111111) << 12 | + (uint8_t(data[pos + 2]) & 0b00111111) << 6 | + (uint8_t(data[pos + 3]) & 0b00111111); + if (code_point <= 0xffff) { + return result(error_code::OVERLONG, pos); + } + if (0x10ffff < code_point) { + return result(error_code::TOO_LARGE, pos); + } + *utf32_output++ = char32_t(code_point); + pos += 4; + } else { + // we either have too many continuation bytes or an invalid leading byte + if ((leading_byte & 0b11000000) == 0b10000000) { + return result(error_code::TOO_LONG, pos); + } else { + return result(error_code::HEADER_BITS, pos); + } + } + } + return result(error_code::SUCCESS, utf32_output - start); +} + +/** + * When rewind_and_convert_with_errors is called, we are pointing at 'buf' and + * we have up to len input bytes left, and we encountered some error. It is + * possible that the error is at 'buf' exactly, but it could also be in the + * previous bytes location (up to 3 bytes back). + * + * prior_bytes indicates how many bytes, prior to 'buf' may belong to the + * current memory section and can be safely accessed. We prior_bytes to access + * safely up to three bytes before 'buf'. + * + * The caller is responsible to ensure that len > 0. + * + * If the error is believed to have occurred prior to 'buf', the count value + * contain in the result will be SIZE_T - 1, SIZE_T - 2, or SIZE_T - 3. + */ +inline result rewind_and_convert_with_errors(size_t prior_bytes, + const char *buf, size_t len, + char32_t *utf32_output) { + size_t extra_len{0}; + // We potentially need to go back in time and find a leading byte. + size_t how_far_back = 3; // 3 bytes in the past + current position + if (how_far_back > prior_bytes) { + how_far_back = prior_bytes; + } + bool found_leading_bytes{false}; + // important: it is i <= how_far_back and not 'i < how_far_back'. + for (size_t i = 0; i <= how_far_back; i++) { + unsigned char byte = buf[-static_cast(i)]; + found_leading_bytes = ((byte & 0b11000000) != 0b10000000); + if (found_leading_bytes) { + if (i > 0 && byte < 128) { + // If we had to go back and the leading byte is ascii + // then we can stop right away. + return result(error_code::TOO_LONG, 0 - i + 1); + } + buf -= i; + extra_len = i; + break; + } + } + // + // It is possible for this function to return a negative count in its result. + // C++ Standard Section 18.1 defines size_t is in which is described + // in C Standard as . C Standard Section 4.1.5 defines size_t as an + // unsigned integral type of the result of the sizeof operator + // + // An unsigned type will simply wrap round arithmetically (well defined). + // + if (!found_leading_bytes) { + // If how_far_back == 3, we may have four consecutive continuation bytes!!! + // [....] [continuation] [continuation] [continuation] | [buf is + // continuation] Or we possibly have a stream that does not start with a + // leading byte. + return result(error_code::TOO_LONG, 0 - how_far_back); + } + + result res = convert_with_errors(buf, len + extra_len, utf32_output); + if (res.error) { + res.count -= extra_len; + } + return res; +} + +} // namespace utf8_to_utf32 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_utf32/utf8_to_utf32.h */ +/* begin file include/simdutf/scalar/utf8_to_utf32/valid_utf8_to_utf32.h */ +#ifndef SIMDUTF_VALID_UTF8_TO_UTF32_H +#define SIMDUTF_VALID_UTF8_TO_UTF32_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_utf32 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 8 ASCII bytes + if (pos + 8 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 8; + while (pos < final_pos) { + *utf32_output++ = uint8_t(data[pos]); + pos++; + } + continue; + } + } + } + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf32_output++ = char32_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + break; + } // minimal bound checking + *utf32_output++ = char32_t(((leading_byte & 0b00011111) << 6) | + (uint8_t(data[pos + 1]) & 0b00111111)); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8 + if (pos + 2 >= len) { + break; + } // minimal bound checking + *utf32_output++ = char32_t(((leading_byte & 0b00001111) << 12) | + ((uint8_t(data[pos + 1]) & 0b00111111) << 6) | + (uint8_t(data[pos + 2]) & 0b00111111)); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + break; + } // minimal bound checking + uint32_t code_word = ((leading_byte & 0b00000111) << 18) | + ((uint8_t(data[pos + 1]) & 0b00111111) << 12) | + ((uint8_t(data[pos + 2]) & 0b00111111) << 6) | + (uint8_t(data[pos + 3]) & 0b00111111); + *utf32_output++ = char32_t(code_word); + pos += 4; + } else { + // we may have a continuation but we do not do error checking + return 0; + } + } + return utf32_output - start; +} + +} // namespace utf8_to_utf32 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_utf32/valid_utf8_to_utf32.h */ + +namespace simdutf { + +constexpr size_t default_line_length = + 76; ///< default line length for base64 encoding with lines + + #ifndef SIMDUTF_NEED_TRAILING_ZEROES + #define SIMDUTF_NEED_TRAILING_ZEROES 1 + #endif + +// base64_options are used to specify the base64 encoding options. +// ASCII spaces are ' ', '\t', '\n', '\r', '\f' +// garbage characters are characters that are not part of the base64 alphabet +// nor ASCII spaces. +constexpr uint64_t base64_reverse_padding = + 2; /* modifier for base64_default and base64_url */ +enum base64_options : uint64_t { + base64_default = 0, /* standard base64 format (with padding) */ + base64_url = 1, /* base64url format (no padding) */ + base64_default_no_padding = + base64_default | + base64_reverse_padding, /* standard base64 format without padding */ + base64_url_with_padding = + base64_url | base64_reverse_padding, /* base64url with padding */ + base64_default_accept_garbage = + 4, /* standard base64 format accepting garbage characters, the input stops + with the first '=' if any */ + base64_url_accept_garbage = + 5, /* base64url format accepting garbage characters, the input stops with + the first '=' if any */ + base64_default_or_url = + 8, /* standard/base64url hybrid format (only meaningful for decoding!) */ + base64_default_or_url_accept_garbage = + 12, /* standard/base64url hybrid format accepting garbage characters + (only meaningful for decoding!), the input stops with the first '=' + if any */ +}; + +// last_chunk_handling_options are used to specify the handling of the last +// chunk in base64 decoding. +// https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 +enum last_chunk_handling_options : uint64_t { + loose = 0, /* standard base64 format, decode partial final chunk */ + strict = 1, /* error when the last chunk is partial, 2 or 3 chars, and + unpadded, or non-zero bit padding */ + stop_before_partial = + 2, /* if the last chunk is partial, ignore it (no error) */ + only_full_chunks = + 3 /* only decode full blocks (4 base64 characters, no padding) */ +}; + +inline simdutf_constexpr23 bool +is_partial(last_chunk_handling_options options) { + return (options == stop_before_partial) || (options == only_full_chunks); +} + +namespace detail { +simdutf_warn_unused const char *find(const char *start, const char *end, + char character) noexcept; +simdutf_warn_unused const char16_t * +find(const char16_t *start, const char16_t *end, char16_t character) noexcept; +} // namespace detail + +/** + * Find the first occurrence of a character in a string. If the character is + * not found, return a pointer to the end of the string. + * @param start the start of the string + * @param end the end of the string + * @param character the character to find + * @return a pointer to the first occurrence of the character in the string, + * or a pointer to the end of the string if the character is not found. + * + */ +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 const char * +find(const char *start, const char *end, char character) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + for (; start != end; ++start) + if (*start == character) + return start; + return end; + } else + #endif + { + return detail::find(start, end, character); + } +} +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 const char16_t * +find(const char16_t *start, const char16_t *end, char16_t character) noexcept { + // implementation note: this is repeated instead of a template, to ensure + // the api is still a function and compiles without concepts + #if SIMDUTF_CPLUSPLUS23 + if consteval { + for (; start != end; ++start) + if (*start == character) + return start; + return end; + } else + #endif + { + return detail::find(start, end, character); + } +} +} + // We include base64_tables once. +/* begin file include/simdutf/base64_tables.h */ +#ifndef SIMDUTF_BASE64_TABLES_H +#define SIMDUTF_BASE64_TABLES_H +#include + +namespace simdutf { +namespace { +namespace tables { +namespace base64 { +namespace base64_default { + +constexpr char e0[256] = { + 'A', 'A', 'A', 'A', 'B', 'B', 'B', 'B', 'C', 'C', 'C', 'C', 'D', 'D', 'D', + 'D', 'E', 'E', 'E', 'E', 'F', 'F', 'F', 'F', 'G', 'G', 'G', 'G', 'H', 'H', + 'H', 'H', 'I', 'I', 'I', 'I', 'J', 'J', 'J', 'J', 'K', 'K', 'K', 'K', 'L', + 'L', 'L', 'L', 'M', 'M', 'M', 'M', 'N', 'N', 'N', 'N', 'O', 'O', 'O', 'O', + 'P', 'P', 'P', 'P', 'Q', 'Q', 'Q', 'Q', 'R', 'R', 'R', 'R', 'S', 'S', 'S', + 'S', 'T', 'T', 'T', 'T', 'U', 'U', 'U', 'U', 'V', 'V', 'V', 'V', 'W', 'W', + 'W', 'W', 'X', 'X', 'X', 'X', 'Y', 'Y', 'Y', 'Y', 'Z', 'Z', 'Z', 'Z', 'a', + 'a', 'a', 'a', 'b', 'b', 'b', 'b', 'c', 'c', 'c', 'c', 'd', 'd', 'd', 'd', + 'e', 'e', 'e', 'e', 'f', 'f', 'f', 'f', 'g', 'g', 'g', 'g', 'h', 'h', 'h', + 'h', 'i', 'i', 'i', 'i', 'j', 'j', 'j', 'j', 'k', 'k', 'k', 'k', 'l', 'l', + 'l', 'l', 'm', 'm', 'm', 'm', 'n', 'n', 'n', 'n', 'o', 'o', 'o', 'o', 'p', + 'p', 'p', 'p', 'q', 'q', 'q', 'q', 'r', 'r', 'r', 'r', 's', 's', 's', 's', + 't', 't', 't', 't', 'u', 'u', 'u', 'u', 'v', 'v', 'v', 'v', 'w', 'w', 'w', + 'w', 'x', 'x', 'x', 'x', 'y', 'y', 'y', 'y', 'z', 'z', 'z', 'z', '0', '0', + '0', '0', '1', '1', '1', '1', '2', '2', '2', '2', '3', '3', '3', '3', '4', + '4', '4', '4', '5', '5', '5', '5', '6', '6', '6', '6', '7', '7', '7', '7', + '8', '8', '8', '8', '9', '9', '9', '9', '+', '+', '+', '+', '/', '/', '/', + '/'}; + +constexpr char e1[256] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', + 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', + 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', + 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', + 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', + 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', + 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', + 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', + 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', + '/'}; + +constexpr char e2[256] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', + 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', + 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', + 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', + 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', + 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', + 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', + 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', + 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', + '/'}; + +constexpr uint32_t d0[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x000000f8, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000fc, + 0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4, + 0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018, + 0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030, + 0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048, + 0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060, + 0x00000064, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078, + 0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090, + 0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8, + 0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0, + 0x000000c4, 0x000000c8, 0x000000cc, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; + +constexpr uint32_t d1[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x0000e003, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000f003, + 0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003, + 0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000, + 0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000, + 0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001, + 0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001, + 0x00009001, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001, + 0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002, + 0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002, + 0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003, + 0x00001003, 0x00002003, 0x00003003, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; + +constexpr uint32_t d2[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x00800f00, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00c00f00, + 0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00, + 0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100, + 0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300, + 0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400, + 0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600, + 0x00400600, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700, + 0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900, + 0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00, + 0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00, + 0x00400c00, 0x00800c00, 0x00c00c00, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; + +constexpr uint32_t d3[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x003e0000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003f0000, + 0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000, + 0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000, + 0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000, + 0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000, + 0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000, + 0x00190000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000, + 0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000, + 0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000, + 0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000, + 0x00310000, 0x00320000, 0x00330000, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +} // namespace base64_default + +namespace base64_url { + +constexpr char e0[256] = { + 'A', 'A', 'A', 'A', 'B', 'B', 'B', 'B', 'C', 'C', 'C', 'C', 'D', 'D', 'D', + 'D', 'E', 'E', 'E', 'E', 'F', 'F', 'F', 'F', 'G', 'G', 'G', 'G', 'H', 'H', + 'H', 'H', 'I', 'I', 'I', 'I', 'J', 'J', 'J', 'J', 'K', 'K', 'K', 'K', 'L', + 'L', 'L', 'L', 'M', 'M', 'M', 'M', 'N', 'N', 'N', 'N', 'O', 'O', 'O', 'O', + 'P', 'P', 'P', 'P', 'Q', 'Q', 'Q', 'Q', 'R', 'R', 'R', 'R', 'S', 'S', 'S', + 'S', 'T', 'T', 'T', 'T', 'U', 'U', 'U', 'U', 'V', 'V', 'V', 'V', 'W', 'W', + 'W', 'W', 'X', 'X', 'X', 'X', 'Y', 'Y', 'Y', 'Y', 'Z', 'Z', 'Z', 'Z', 'a', + 'a', 'a', 'a', 'b', 'b', 'b', 'b', 'c', 'c', 'c', 'c', 'd', 'd', 'd', 'd', + 'e', 'e', 'e', 'e', 'f', 'f', 'f', 'f', 'g', 'g', 'g', 'g', 'h', 'h', 'h', + 'h', 'i', 'i', 'i', 'i', 'j', 'j', 'j', 'j', 'k', 'k', 'k', 'k', 'l', 'l', + 'l', 'l', 'm', 'm', 'm', 'm', 'n', 'n', 'n', 'n', 'o', 'o', 'o', 'o', 'p', + 'p', 'p', 'p', 'q', 'q', 'q', 'q', 'r', 'r', 'r', 'r', 's', 's', 's', 's', + 't', 't', 't', 't', 'u', 'u', 'u', 'u', 'v', 'v', 'v', 'v', 'w', 'w', 'w', + 'w', 'x', 'x', 'x', 'x', 'y', 'y', 'y', 'y', 'z', 'z', 'z', 'z', '0', '0', + '0', '0', '1', '1', '1', '1', '2', '2', '2', '2', '3', '3', '3', '3', '4', + '4', '4', '4', '5', '5', '5', '5', '6', '6', '6', '6', '7', '7', '7', '7', + '8', '8', '8', '8', '9', '9', '9', '9', '-', '-', '-', '-', '_', '_', '_', + '_'}; + +constexpr char e1[256] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', + 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', + 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', + 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', + 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', + 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', + 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', + 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', + 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', + '_'}; + +constexpr char e2[256] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', + 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', + 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', + 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', + 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', + 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', + 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', + 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', + 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', + '_'}; + +constexpr uint32_t d0[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000f8, 0x01ffffff, 0x01ffffff, + 0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4, + 0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018, + 0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030, + 0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048, + 0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060, + 0x00000064, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000fc, + 0x01ffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078, + 0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090, + 0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8, + 0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0, + 0x000000c4, 0x000000c8, 0x000000cc, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d1[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000e003, 0x01ffffff, 0x01ffffff, + 0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003, + 0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000, + 0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000, + 0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001, + 0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001, + 0x00009001, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000f003, + 0x01ffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001, + 0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002, + 0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002, + 0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003, + 0x00001003, 0x00002003, 0x00003003, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d2[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00800f00, 0x01ffffff, 0x01ffffff, + 0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00, + 0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100, + 0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300, + 0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400, + 0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600, + 0x00400600, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00c00f00, + 0x01ffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700, + 0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900, + 0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00, + 0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00, + 0x00400c00, 0x00800c00, 0x00c00c00, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d3[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003e0000, 0x01ffffff, 0x01ffffff, + 0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000, + 0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000, + 0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000, + 0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000, + 0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000, + 0x00190000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003f0000, + 0x01ffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000, + 0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000, + 0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000, + 0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000, + 0x00310000, 0x00320000, 0x00330000, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +} // namespace base64_url + +namespace base64_default_or_url { +constexpr uint32_t d0[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x000000f8, 0x01ffffff, 0x000000f8, 0x01ffffff, 0x000000fc, + 0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4, + 0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018, + 0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030, + 0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048, + 0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060, + 0x00000064, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000fc, + 0x01ffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078, + 0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090, + 0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8, + 0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0, + 0x000000c4, 0x000000c8, 0x000000cc, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d1[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x0000e003, 0x01ffffff, 0x0000e003, 0x01ffffff, 0x0000f003, + 0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003, + 0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000, + 0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000, + 0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001, + 0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001, + 0x00009001, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000f003, + 0x01ffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001, + 0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002, + 0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002, + 0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003, + 0x00001003, 0x00002003, 0x00003003, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d2[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x00800f00, 0x01ffffff, 0x00800f00, 0x01ffffff, 0x00c00f00, + 0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00, + 0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100, + 0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300, + 0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400, + 0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600, + 0x00400600, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00c00f00, + 0x01ffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700, + 0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900, + 0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00, + 0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00, + 0x00400c00, 0x00800c00, 0x00c00c00, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d3[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x003e0000, 0x01ffffff, 0x003e0000, 0x01ffffff, 0x003f0000, + 0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000, + 0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000, + 0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000, + 0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000, + 0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000, + 0x00190000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003f0000, + 0x01ffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000, + 0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000, + 0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000, + 0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000, + 0x00310000, 0x00320000, 0x00330000, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +} // namespace base64_default_or_url +constexpr uint64_t thintable_epi8[256] = { + 0x0706050403020100, 0x0007060504030201, 0x0007060504030200, + 0x0000070605040302, 0x0007060504030100, 0x0000070605040301, + 0x0000070605040300, 0x0000000706050403, 0x0007060504020100, + 0x0000070605040201, 0x0000070605040200, 0x0000000706050402, + 0x0000070605040100, 0x0000000706050401, 0x0000000706050400, + 0x0000000007060504, 0x0007060503020100, 0x0000070605030201, + 0x0000070605030200, 0x0000000706050302, 0x0000070605030100, + 0x0000000706050301, 0x0000000706050300, 0x0000000007060503, + 0x0000070605020100, 0x0000000706050201, 0x0000000706050200, + 0x0000000007060502, 0x0000000706050100, 0x0000000007060501, + 0x0000000007060500, 0x0000000000070605, 0x0007060403020100, + 0x0000070604030201, 0x0000070604030200, 0x0000000706040302, + 0x0000070604030100, 0x0000000706040301, 0x0000000706040300, + 0x0000000007060403, 0x0000070604020100, 0x0000000706040201, + 0x0000000706040200, 0x0000000007060402, 0x0000000706040100, + 0x0000000007060401, 0x0000000007060400, 0x0000000000070604, + 0x0000070603020100, 0x0000000706030201, 0x0000000706030200, + 0x0000000007060302, 0x0000000706030100, 0x0000000007060301, + 0x0000000007060300, 0x0000000000070603, 0x0000000706020100, + 0x0000000007060201, 0x0000000007060200, 0x0000000000070602, + 0x0000000007060100, 0x0000000000070601, 0x0000000000070600, + 0x0000000000000706, 0x0007050403020100, 0x0000070504030201, + 0x0000070504030200, 0x0000000705040302, 0x0000070504030100, + 0x0000000705040301, 0x0000000705040300, 0x0000000007050403, + 0x0000070504020100, 0x0000000705040201, 0x0000000705040200, + 0x0000000007050402, 0x0000000705040100, 0x0000000007050401, + 0x0000000007050400, 0x0000000000070504, 0x0000070503020100, + 0x0000000705030201, 0x0000000705030200, 0x0000000007050302, + 0x0000000705030100, 0x0000000007050301, 0x0000000007050300, + 0x0000000000070503, 0x0000000705020100, 0x0000000007050201, + 0x0000000007050200, 0x0000000000070502, 0x0000000007050100, + 0x0000000000070501, 0x0000000000070500, 0x0000000000000705, + 0x0000070403020100, 0x0000000704030201, 0x0000000704030200, + 0x0000000007040302, 0x0000000704030100, 0x0000000007040301, + 0x0000000007040300, 0x0000000000070403, 0x0000000704020100, + 0x0000000007040201, 0x0000000007040200, 0x0000000000070402, + 0x0000000007040100, 0x0000000000070401, 0x0000000000070400, + 0x0000000000000704, 0x0000000703020100, 0x0000000007030201, + 0x0000000007030200, 0x0000000000070302, 0x0000000007030100, + 0x0000000000070301, 0x0000000000070300, 0x0000000000000703, + 0x0000000007020100, 0x0000000000070201, 0x0000000000070200, + 0x0000000000000702, 0x0000000000070100, 0x0000000000000701, + 0x0000000000000700, 0x0000000000000007, 0x0006050403020100, + 0x0000060504030201, 0x0000060504030200, 0x0000000605040302, + 0x0000060504030100, 0x0000000605040301, 0x0000000605040300, + 0x0000000006050403, 0x0000060504020100, 0x0000000605040201, + 0x0000000605040200, 0x0000000006050402, 0x0000000605040100, + 0x0000000006050401, 0x0000000006050400, 0x0000000000060504, + 0x0000060503020100, 0x0000000605030201, 0x0000000605030200, + 0x0000000006050302, 0x0000000605030100, 0x0000000006050301, + 0x0000000006050300, 0x0000000000060503, 0x0000000605020100, + 0x0000000006050201, 0x0000000006050200, 0x0000000000060502, + 0x0000000006050100, 0x0000000000060501, 0x0000000000060500, + 0x0000000000000605, 0x0000060403020100, 0x0000000604030201, + 0x0000000604030200, 0x0000000006040302, 0x0000000604030100, + 0x0000000006040301, 0x0000000006040300, 0x0000000000060403, + 0x0000000604020100, 0x0000000006040201, 0x0000000006040200, + 0x0000000000060402, 0x0000000006040100, 0x0000000000060401, + 0x0000000000060400, 0x0000000000000604, 0x0000000603020100, + 0x0000000006030201, 0x0000000006030200, 0x0000000000060302, + 0x0000000006030100, 0x0000000000060301, 0x0000000000060300, + 0x0000000000000603, 0x0000000006020100, 0x0000000000060201, + 0x0000000000060200, 0x0000000000000602, 0x0000000000060100, + 0x0000000000000601, 0x0000000000000600, 0x0000000000000006, + 0x0000050403020100, 0x0000000504030201, 0x0000000504030200, + 0x0000000005040302, 0x0000000504030100, 0x0000000005040301, + 0x0000000005040300, 0x0000000000050403, 0x0000000504020100, + 0x0000000005040201, 0x0000000005040200, 0x0000000000050402, + 0x0000000005040100, 0x0000000000050401, 0x0000000000050400, + 0x0000000000000504, 0x0000000503020100, 0x0000000005030201, + 0x0000000005030200, 0x0000000000050302, 0x0000000005030100, + 0x0000000000050301, 0x0000000000050300, 0x0000000000000503, + 0x0000000005020100, 0x0000000000050201, 0x0000000000050200, + 0x0000000000000502, 0x0000000000050100, 0x0000000000000501, + 0x0000000000000500, 0x0000000000000005, 0x0000000403020100, + 0x0000000004030201, 0x0000000004030200, 0x0000000000040302, + 0x0000000004030100, 0x0000000000040301, 0x0000000000040300, + 0x0000000000000403, 0x0000000004020100, 0x0000000000040201, + 0x0000000000040200, 0x0000000000000402, 0x0000000000040100, + 0x0000000000000401, 0x0000000000000400, 0x0000000000000004, + 0x0000000003020100, 0x0000000000030201, 0x0000000000030200, + 0x0000000000000302, 0x0000000000030100, 0x0000000000000301, + 0x0000000000000300, 0x0000000000000003, 0x0000000000020100, + 0x0000000000000201, 0x0000000000000200, 0x0000000000000002, + 0x0000000000000100, 0x0000000000000001, 0x0000000000000000, + 0x0000000000000000, +}; + +constexpr uint8_t pshufb_combine_table[272] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0x00, 0x01, 0x02, 0x03, + 0x04, 0x05, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, + 0x0f, 0xff, 0xff, 0xff, 0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, 0x00, 0x01, 0x02, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, + 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +}; + +constexpr unsigned char BitsSetTable256mul2[256] = { + 0, 2, 2, 4, 2, 4, 4, 6, 2, 4, 4, 6, 4, 6, 6, 8, 2, 4, 4, + 6, 4, 6, 6, 8, 4, 6, 6, 8, 6, 8, 8, 10, 2, 4, 4, 6, 4, 6, + 6, 8, 4, 6, 6, 8, 6, 8, 8, 10, 4, 6, 6, 8, 6, 8, 8, 10, 6, + 8, 8, 10, 8, 10, 10, 12, 2, 4, 4, 6, 4, 6, 6, 8, 4, 6, 6, 8, + 6, 8, 8, 10, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, 10, 8, 10, 10, + 12, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, 10, 8, 10, 10, 12, 6, 8, + 8, 10, 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, 12, 14, 2, 4, 4, 6, 4, + 6, 6, 8, 4, 6, 6, 8, 6, 8, 8, 10, 4, 6, 6, 8, 6, 8, 8, 10, + 6, 8, 8, 10, 8, 10, 10, 12, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, + 10, 8, 10, 10, 12, 6, 8, 8, 10, 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, + 12, 14, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, 10, 8, 10, 10, 12, 6, + 8, 8, 10, 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, 12, 14, 6, 8, 8, 10, + 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, 12, 14, 8, 10, 10, 12, 10, 12, 12, + 14, 10, 12, 12, 14, 12, 14, 14, 16}; + +constexpr uint8_t to_base64_value[] = { + 255, 255, 255, 255, 255, 255, 255, 255, 255, 64, 64, 255, 64, 64, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 64, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, + 255, 255, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, + 255, 255, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 255, 255, 255, 255, 255, 255, 26, 27, 28, 29, 30, 31, 32, 33, + 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255}; + +constexpr uint8_t to_base64_url_value[] = { + 255, 255, 255, 255, 255, 255, 255, 255, 255, 64, 64, 255, 64, 64, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 64, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 62, 255, 255, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, + 255, 255, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 255, 255, 255, 255, 63, 255, 26, 27, 28, 29, 30, 31, 32, 33, + 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255}; + +constexpr uint8_t to_base64_default_or_url_value[] = { + 255, 255, 255, 255, 255, 255, 255, 255, 255, 64, 64, 255, 64, 64, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 64, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, + 62, 255, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, + 255, 255, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 255, 255, 255, 255, 63, 255, 26, 27, 28, 29, 30, 31, 32, 33, + 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255}; + +static_assert(sizeof(to_base64_value) == 256, + "to_base64_value must have 256 elements"); +static_assert(sizeof(to_base64_url_value) == 256, + "to_base64_url_value must have 256 elements"); +static_assert(to_base64_value[uint8_t(' ')] == 64, + "space must be == 64 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t(' ')] == 64, + "space must be == 64 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('\t')] == 64, + "tab must be == 64 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('\t')] == 64, + "tab must be == 64 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('\r')] == 64, + "cr must be == 64 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('\r')] == 64, + "cr must be == 64 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('\n')] == 64, + "lf must be == 64 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('\n')] == 64, + "lf must be == 64 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('\f')] == 64, + "ff must be == 64 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('\f')] == 64, + "ff must be == 64 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('+')] == 62, + "+ must be == 62 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('-')] == 62, + "- must be == 62 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('/')] == 63, + "/ must be == 63 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('_')] == 63, + "_ must be == 63 in to_base64_url_value"); +} // namespace base64 +} // namespace tables +} // unnamed namespace +} // namespace simdutf + +#endif // SIMDUTF_BASE64_TABLES_H +/* end file include/simdutf/base64_tables.h */ +/* begin file include/simdutf/scalar/base64.h */ +#ifndef SIMDUTF_BASE64_H +#define SIMDUTF_BASE64_H + +#include +#include +#include +#include + +namespace simdutf { +namespace scalar { +namespace { +namespace base64 { + +// This function is not expected to be fast. Do not use in long loops. +// In most instances you should be using is_ignorable. +template bool is_ascii_white_space(char_type c) { + return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f'; +} + +template simdutf_constexpr23 bool is_eight_byte(char_type c) { + if constexpr (sizeof(char_type) == 1) { + return true; + } + return uint8_t(c) == c; +} + +template +simdutf_constexpr23 bool is_ignorable(char_type c, + simdutf::base64_options options) { + const uint8_t *to_base64 = + (options & base64_default_or_url) + ? tables::base64::to_base64_default_or_url_value + : ((options & base64_url) ? tables::base64::to_base64_url_value + : tables::base64::to_base64_value); + const bool ignore_garbage = + (options == base64_options::base64_url_accept_garbage) || + (options == base64_options::base64_default_accept_garbage) || + (options == base64_options::base64_default_or_url_accept_garbage); + uint8_t code = to_base64[uint8_t(c)]; + if (is_eight_byte(c) && code <= 63) { + return false; + } + if (is_eight_byte(c) && code == 64) { + return true; + } + return ignore_garbage; +} +template +simdutf_constexpr23 bool is_base64(char_type c, + simdutf::base64_options options) { + const uint8_t *to_base64 = + (options & base64_default_or_url) + ? tables::base64::to_base64_default_or_url_value + : ((options & base64_url) ? tables::base64::to_base64_url_value + : tables::base64::to_base64_value); + uint8_t code = to_base64[uint8_t(c)]; + if (is_eight_byte(c) && code <= 63) { + return true; + } + return false; +} + +template +simdutf_constexpr23 bool is_base64_or_padding(char_type c, + simdutf::base64_options options) { + const uint8_t *to_base64 = + (options & base64_default_or_url) + ? tables::base64::to_base64_default_or_url_value + : ((options & base64_url) ? tables::base64::to_base64_url_value + : tables::base64::to_base64_value); + if (c == '=') { + return true; + } + uint8_t code = to_base64[uint8_t(c)]; + if (is_eight_byte(c) && code <= 63) { + return true; + } + return false; +} + +template +bool is_ignorable_or_padding(char_type c, simdutf::base64_options options) { + return is_ignorable(c, options) || c == '='; +} + +struct reduced_input { + size_t equalsigns; // number of padding characters '=', typically 0, 1, 2. + size_t equallocation; // location of the first padding character if any + size_t srclen; // length of the input buffer before padding + size_t full_input_length; // length of the input buffer with padding but + // without ignorable characters +}; + +// find the end of the base64 input buffer +// It returns the number of padding characters, the location of the first +// padding character if any, the length of the input buffer before padding +// and the length of the input buffer with padding. The input buffer is not +// modified. The function assumes that there are at most two padding characters. +template +simdutf_constexpr23 reduced_input find_end(const char_type *src, size_t srclen, + simdutf::base64_options options) { + const uint8_t *to_base64 = + (options & base64_default_or_url) + ? tables::base64::to_base64_default_or_url_value + : ((options & base64_url) ? tables::base64::to_base64_url_value + : tables::base64::to_base64_value); + const bool ignore_garbage = + (options == base64_options::base64_url_accept_garbage) || + (options == base64_options::base64_default_accept_garbage) || + (options == base64_options::base64_default_or_url_accept_garbage); + + size_t equalsigns = 0; + // We intentionally include trailing spaces in the full input length. + // See https://github.com/simdutf/simdutf/issues/824 + size_t full_input_length = srclen; + // skip trailing spaces + while (!ignore_garbage && srclen > 0 && + scalar::base64::is_eight_byte(src[srclen - 1]) && + to_base64[uint8_t(src[srclen - 1])] == 64) { + srclen--; + } + size_t equallocation = + srclen; // location of the first padding character if any + if (ignore_garbage) { + // Technically, we don't need to find the first padding character, we can + // just change our algorithms, but it adds substantial complexity. + auto it = simdutf::find(src, src + srclen, '='); + if (it != src + srclen) { + equallocation = it - src; + equalsigns = 1; + srclen = equallocation; + full_input_length = equallocation + 1; + } + return {equalsigns, equallocation, srclen, full_input_length}; + } + if (!ignore_garbage && srclen > 0 && src[srclen - 1] == '=') { + // This is the last '=' sign. + equallocation = srclen - 1; + srclen--; + equalsigns = 1; + // skip trailing spaces + while (srclen > 0 && scalar::base64::is_eight_byte(src[srclen - 1]) && + to_base64[uint8_t(src[srclen - 1])] == 64) { + srclen--; + } + if (srclen > 0 && src[srclen - 1] == '=') { + // This is the second '=' sign. + equallocation = srclen - 1; + srclen--; + equalsigns = 2; + } + } + return {equalsigns, equallocation, srclen, full_input_length}; +} + +// Returns true upon success. The destination buffer must be large enough. +// This functions assumes that the padding (=) has been removed. +// if check_capacity is true, it will check that the destination buffer is +// large enough. If it is not, it will return OUTPUT_BUFFER_TOO_SMALL. +template +simdutf_constexpr23 full_result base64_tail_decode_impl( + char *dst, size_t outlen, const char_type *src, size_t length, + size_t padding_characters, // number of padding characters + // '=', typically 0, 1, 2. + base64_options options, last_chunk_handling_options last_chunk_options) { + char *dstend = dst + outlen; + (void)dstend; + // This looks like 10 branches, but we expect the compiler to resolve this to + // two branches (easily predicted): + const uint8_t *to_base64 = + (options & base64_default_or_url) + ? tables::base64::to_base64_default_or_url_value + : ((options & base64_url) ? tables::base64::to_base64_url_value + : tables::base64::to_base64_value); + const uint32_t *d0 = + (options & base64_default_or_url) + ? tables::base64::base64_default_or_url::d0 + : ((options & base64_url) ? tables::base64::base64_url::d0 + : tables::base64::base64_default::d0); + const uint32_t *d1 = + (options & base64_default_or_url) + ? tables::base64::base64_default_or_url::d1 + : ((options & base64_url) ? tables::base64::base64_url::d1 + : tables::base64::base64_default::d1); + const uint32_t *d2 = + (options & base64_default_or_url) + ? tables::base64::base64_default_or_url::d2 + : ((options & base64_url) ? tables::base64::base64_url::d2 + : tables::base64::base64_default::d2); + const uint32_t *d3 = + (options & base64_default_or_url) + ? tables::base64::base64_default_or_url::d3 + : ((options & base64_url) ? tables::base64::base64_url::d3 + : tables::base64::base64_default::d3); + const bool ignore_garbage = + (options == base64_options::base64_url_accept_garbage) || + (options == base64_options::base64_default_accept_garbage) || + (options == base64_options::base64_default_or_url_accept_garbage); + + const char_type *srcend = src + length; + const char_type *srcinit = src; + const char *dstinit = dst; + + uint32_t x; + size_t idx; + uint8_t buffer[4]; + while (true) { + while (srcend - src >= 4 && is_eight_byte(src[0]) && + is_eight_byte(src[1]) && is_eight_byte(src[2]) && + is_eight_byte(src[3]) && + (x = d0[uint8_t(src[0])] | d1[uint8_t(src[1])] | + d2[uint8_t(src[2])] | d3[uint8_t(src[3])]) < 0x01FFFFFF) { + if (check_capacity && dstend - dst < 3) { + return {OUTPUT_BUFFER_TOO_SMALL, size_t(src - srcinit), + size_t(dst - dstinit)}; + } + *dst++ = static_cast(x & 0xFF); + *dst++ = static_cast((x >> 8) & 0xFF); + *dst++ = static_cast((x >> 16) & 0xFF); + src += 4; + } + const char_type *srccur = src; + idx = 0; + // we need at least four characters. +#ifdef __clang__ + // If possible, we read four characters at a time. (It is an optimization.) + if (ignore_garbage && src + 4 <= srcend) { + char_type c0 = src[0]; + char_type c1 = src[1]; + char_type c2 = src[2]; + char_type c3 = src[3]; + + uint8_t code0 = to_base64[uint8_t(c0)]; + uint8_t code1 = to_base64[uint8_t(c1)]; + uint8_t code2 = to_base64[uint8_t(c2)]; + uint8_t code3 = to_base64[uint8_t(c3)]; + + buffer[idx] = code0; + idx += (is_eight_byte(c0) && code0 <= 63); + buffer[idx] = code1; + idx += (is_eight_byte(c1) && code1 <= 63); + buffer[idx] = code2; + idx += (is_eight_byte(c2) && code2 <= 63); + buffer[idx] = code3; + idx += (is_eight_byte(c3) && code3 <= 63); + src += 4; + } +#endif + while ((idx < 4) && (src < srcend)) { + char_type c = *src; + + uint8_t code = to_base64[uint8_t(c)]; + buffer[idx] = uint8_t(code); + if (is_eight_byte(c) && code <= 63) { + idx++; + } else if (!ignore_garbage && + (code > 64 || !scalar::base64::is_eight_byte(c))) { + return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), + size_t(dst - dstinit)}; + } else { + // We have a space or a newline or garbage. We ignore it. + } + src++; + } + if (idx != 4) { + simdutf_log_assert(idx < 4, "idx should be less than 4"); + // We never should have that the number of base64 characters + the + // number of padding characters is more than 4. + if (!ignore_garbage && (idx + padding_characters > 4)) { + return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), + size_t(dst - dstinit), true}; + } + + // The idea here is that in loose mode, + // if there is padding at all, it must be used + // to form 4-wise chunk. However, in loose mode, + // we do accept no padding at all. + if (!ignore_garbage && + last_chunk_options == last_chunk_handling_options::loose && + (idx >= 2) && padding_characters > 0 && + ((idx + padding_characters) & 3) != 0) { + return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), + size_t(dst - dstinit), true}; + } else + + // The idea here is that in strict mode, we do not want to accept + // incomplete base64 chunks. So if the chunk was otherwise valid, we + // return BASE64_INPUT_REMAINDER. + if (!ignore_garbage && + last_chunk_options == last_chunk_handling_options::strict && + (idx >= 2) && ((idx + padding_characters) & 3) != 0) { + // The partial chunk was at src - idx + return {BASE64_INPUT_REMAINDER, size_t(src - srcinit), + size_t(dst - dstinit), true}; + } else + // If there is a partial chunk with insufficient padding, with + // stop_before_partial, we need to just ignore it. In "only full" + // mode, skip the minute there are padding characters. + if ((last_chunk_options == + last_chunk_handling_options::stop_before_partial && + (padding_characters + idx < 4) && (idx != 0) && + (idx >= 2 || padding_characters == 0)) || + (last_chunk_options == + last_chunk_handling_options::only_full_chunks && + (idx >= 2 || padding_characters == 0))) { + // partial means that we are *not* going to consume the read + // characters. We need to rewind the src pointer. + src = srccur; + return {SUCCESS, size_t(src - srcinit), size_t(dst - dstinit)}; + } else { + if (idx == 2) { + uint32_t triple = (uint32_t(buffer[0]) << 3 * 6) + + (uint32_t(buffer[1]) << 2 * 6); + if (!ignore_garbage && + (last_chunk_options == last_chunk_handling_options::strict) && + (triple & 0xffff)) { + return {BASE64_EXTRA_BITS, size_t(src - srcinit), + size_t(dst - dstinit)}; + } + if (check_capacity && dstend - dst < 1) { + return {OUTPUT_BUFFER_TOO_SMALL, size_t(srccur - srcinit), + size_t(dst - dstinit)}; + } + *dst++ = static_cast((triple >> 16) & 0xFF); + } else if (idx == 3) { + uint32_t triple = (uint32_t(buffer[0]) << 3 * 6) + + (uint32_t(buffer[1]) << 2 * 6) + + (uint32_t(buffer[2]) << 1 * 6); + if (!ignore_garbage && + (last_chunk_options == last_chunk_handling_options::strict) && + (triple & 0xff)) { + return {BASE64_EXTRA_BITS, size_t(src - srcinit), + size_t(dst - dstinit)}; + } + if (check_capacity && dstend - dst < 2) { + return {OUTPUT_BUFFER_TOO_SMALL, size_t(srccur - srcinit), + size_t(dst - dstinit)}; + } + *dst++ = static_cast((triple >> 16) & 0xFF); + *dst++ = static_cast((triple >> 8) & 0xFF); + } else if (!ignore_garbage && idx == 1 && + (!is_partial(last_chunk_options) || + (is_partial(last_chunk_options) && + padding_characters > 0))) { + return {BASE64_INPUT_REMAINDER, size_t(src - srcinit), + size_t(dst - dstinit)}; + } else if (!ignore_garbage && idx == 0 && padding_characters > 0) { + return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), + size_t(dst - dstinit), true}; + } + return {SUCCESS, size_t(src - srcinit), size_t(dst - dstinit)}; + } + } + if (check_capacity && dstend - dst < 3) { + return {OUTPUT_BUFFER_TOO_SMALL, size_t(srccur - srcinit), + size_t(dst - dstinit)}; + } + uint32_t triple = + (uint32_t(buffer[0]) << 3 * 6) + (uint32_t(buffer[1]) << 2 * 6) + + (uint32_t(buffer[2]) << 1 * 6) + (uint32_t(buffer[3]) << 0 * 6); + *dst++ = static_cast((triple >> 16) & 0xFF); + *dst++ = static_cast((triple >> 8) & 0xFF); + *dst++ = static_cast(triple & 0xFF); + } +} + +template +simdutf_constexpr23 full_result base64_tail_decode( + char *dst, const char_type *src, size_t length, + size_t padding_characters, // number of padding characters + // '=', typically 0, 1, 2. + base64_options options, last_chunk_handling_options last_chunk_options) { + return base64_tail_decode_impl(dst, 0, src, length, padding_characters, + options, last_chunk_options); +} + +// like base64_tail_decode, but it will not write past the end of the output +// buffer. The outlen parameter is modified to reflect the number of bytes +// written. This functions assumes that the padding (=) has been removed. +// +template +simdutf_constexpr23 full_result base64_tail_decode_safe( + char *dst, size_t outlen, const char_type *src, size_t length, + size_t padding_characters, // number of padding characters + // '=', typically 0, 1, 2. + base64_options options, last_chunk_handling_options last_chunk_options) { + return base64_tail_decode_impl(dst, outlen, src, length, + padding_characters, options, + last_chunk_options); +} + +inline simdutf_constexpr23 full_result +patch_tail_result(full_result r, size_t previous_input, size_t previous_output, + size_t equallocation, size_t full_input_length, + last_chunk_handling_options last_chunk_options) { + r.input_count += previous_input; + r.output_count += previous_output; + if (r.padding_error) { + r.input_count = equallocation; + } + + if (r.error == error_code::SUCCESS) { + if (!is_partial(last_chunk_options)) { + // A success when we are not in stop_before_partial mode. + // means that we have consumed the whole input buffer. + r.input_count = full_input_length; + } else if (r.output_count % 3 != 0) { + r.input_count = full_input_length; + } + } + return r; +} + +// Returns the number of bytes written. The destination buffer must be large +// enough. It will add padding (=) if needed. +template +simdutf_constexpr23 size_t tail_encode_base64_impl( + char *dst, const char *src, size_t srclen, base64_options options, + size_t line_length = simdutf::default_line_length, size_t line_offset = 0) { + if constexpr (use_lines) { + // sanitize line_length and starting_line_offset. + // line_length must be greater than 3. + if (line_length < 4) { + line_length = 4; + } + simdutf_log_assert(line_offset <= line_length, + "line_offset should be less than line_length"); + } + // By default, we use padding if we are not using the URL variant. + // This is check with ((options & base64_url) == 0) which returns true if we + // are not using the URL variant. However, we also allow 'inversion' of the + // convention with the base64_reverse_padding option. If the + // base64_reverse_padding option is set, we use padding if we are using the + // URL variant, and we omit it if we are not using the URL variant. This is + // checked with + // ((options & base64_reverse_padding) == base64_reverse_padding). + bool use_padding = + ((options & base64_url) == 0) ^ + ((options & base64_reverse_padding) == base64_reverse_padding); + // This looks like 3 branches, but we expect the compiler to resolve this to + // a single branch: + const char *e0 = (options & base64_url) ? tables::base64::base64_url::e0 + : tables::base64::base64_default::e0; + const char *e1 = (options & base64_url) ? tables::base64::base64_url::e1 + : tables::base64::base64_default::e1; + const char *e2 = (options & base64_url) ? tables::base64::base64_url::e2 + : tables::base64::base64_default::e2; + char *out = dst; + size_t i = 0; + uint8_t t1, t2, t3; + for (; i + 2 < srclen; i += 3) { + t1 = uint8_t(src[i]); + t2 = uint8_t(src[i + 1]); + t3 = uint8_t(src[i + 2]); + if constexpr (use_lines) { + if (line_offset + 3 >= line_length) { + if (line_offset == line_length) { + *out++ = '\n'; + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = e2[t3]; + line_offset = 4; + } else if (line_offset + 1 == line_length) { + *out++ = e0[t1]; + *out++ = '\n'; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = e2[t3]; + line_offset = 3; + } else if (line_offset + 2 == line_length) { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = '\n'; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = e2[t3]; + line_offset = 2; + } else if (line_offset + 3 == line_length) { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = '\n'; + *out++ = e2[t3]; + line_offset = 1; + } + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = e2[t3]; + line_offset += 4; + } + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = e2[t3]; + } + } + switch (srclen - i) { + case 0: + break; + case 1: + t1 = uint8_t(src[i]); + if constexpr (use_lines) { + if (use_padding) { + if (line_offset + 3 >= line_length) { + if (line_offset == line_length) { + *out++ = '\n'; + *out++ = e0[t1]; + *out++ = e1[(t1 & 0x03) << 4]; + *out++ = '='; + *out++ = '='; + } else if (line_offset + 1 == line_length) { + *out++ = e0[t1]; + *out++ = '\n'; + *out++ = e1[(t1 & 0x03) << 4]; + *out++ = '='; + *out++ = '='; + } else if (line_offset + 2 == line_length) { + *out++ = e0[t1]; + *out++ = e1[(t1 & 0x03) << 4]; + *out++ = '\n'; + *out++ = '='; + *out++ = '='; + } else if (line_offset + 3 == line_length) { + *out++ = e0[t1]; + *out++ = e1[(t1 & 0x03) << 4]; + *out++ = '='; + *out++ = '\n'; + *out++ = '='; + } + } else { + *out++ = e0[t1]; + *out++ = e1[(t1 & 0x03) << 4]; + *out++ = '='; + *out++ = '='; + } + } else { + if (line_offset + 2 >= line_length) { + if (line_offset == line_length) { + *out++ = '\n'; + *out++ = e0[uint8_t(src[i])]; + *out++ = e1[(uint8_t(src[i]) & 0x03) << 4]; + } else if (line_offset + 1 == line_length) { + *out++ = e0[uint8_t(src[i])]; + *out++ = '\n'; + *out++ = e1[(uint8_t(src[i]) & 0x03) << 4]; + } else { + *out++ = e0[uint8_t(src[i])]; + *out++ = e1[(uint8_t(src[i]) & 0x03) << 4]; + // *out++ = '\n'; ==> no newline at the end of the output + } + } else { + *out++ = e0[uint8_t(src[i])]; + *out++ = e1[(uint8_t(src[i]) & 0x03) << 4]; + } + } + } else { + *out++ = e0[t1]; + *out++ = e1[(t1 & 0x03) << 4]; + if (use_padding) { + *out++ = '='; + *out++ = '='; + } + } + break; + default: /* case 2 */ + t1 = uint8_t(src[i]); + t2 = uint8_t(src[i + 1]); + if constexpr (use_lines) { + if (use_padding) { + if (line_offset + 3 >= line_length) { + if (line_offset == line_length) { + *out++ = '\n'; + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + *out++ = '='; + } else if (line_offset + 1 == line_length) { + *out++ = e0[t1]; + *out++ = '\n'; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + *out++ = '='; + } else if (line_offset + 2 == line_length) { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = '\n'; + *out++ = e2[(t2 & 0x0F) << 2]; + *out++ = '='; + } else if (line_offset + 3 == line_length) { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + *out++ = '\n'; + *out++ = '='; + } + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + *out++ = '='; + } + } else { + if (line_offset + 3 >= line_length) { + if (line_offset == line_length) { + *out++ = '\n'; + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + } else if (line_offset + 1 == line_length) { + *out++ = e0[t1]; + *out++ = '\n'; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + } else if (line_offset + 2 == line_length) { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = '\n'; + *out++ = e2[(t2 & 0x0F) << 2]; + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + // *out++ = '\n'; ==> no newline at the end of the output + } + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + } + } + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + if (use_padding) { + *out++ = '='; + } + } + } + return (size_t)(out - dst); +} + +// Returns the number of bytes written. The destination buffer must be large +// enough. It will add padding (=) if needed. +inline simdutf_constexpr23 size_t tail_encode_base64(char *dst, const char *src, + size_t srclen, + base64_options options) { + return tail_encode_base64_impl(dst, src, srclen, options); +} + +template +simdutf_warn_unused simdutf_constexpr23 size_t +maximal_binary_length_from_base64(InputPtr input, size_t length) noexcept { + // We process the padding characters ('=') at the end to make sure + // that we return an exact result when the input has no ignorable characters + // (e.g., spaces). + size_t padding = 0; + if (length > 0) { + if (input[length - 1] == '=') { + padding++; + if (length > 1 && input[length - 2] == '=') { + padding++; + } + } + } + // The input is not otherwise processed for ignorable characters or + // validation, so that the function runs in constant time (very fast). In + // practice, base64 inputs without ignorable characters are common and the + // common case are line separated inputs with relatively long lines (e.g., 76 + // characters) which leads this function to a slight (1%) overestimation of + // the output size. + // + // Of course, some inputs might contain an arbitrary number of spaces or + // newlines, which would make this function return a very pessimistic output + // size but systems that produce base64 outputs typically do not do that and + // if they do, they do not care much about minimizing memory usage. + // + // In specialized applications, users may know that their input is line + // separated, which can be checked very quickly by by iterating (e.g., over 76 + // character chunks, looking for the linefeed characters only). We could + // provide a specialized function for that, but it is not clear that the added + // complexity is worth it for us. + // + size_t actual_length = length - padding; + if (actual_length % 4 <= 1) { + return actual_length / 4 * 3; + } + // if we have a valid input, then the remainder must be 2 or 3 adding one or + // two extra bytes. + return actual_length / 4 * 3 + (actual_length % 4) - 1; +} + +// This function computes the binary length by iterating through the input +// and counting non-whitespace characters (excluding padding characters). +// We use a simple check (c > ' ') which is easy to parallelize and matches +// SIMD behavior. Only the last few characters are checked for padding '='. +template +simdutf_warn_unused simdutf_constexpr23 size_t +binary_length_from_base64(const char_type *input, size_t length) noexcept { + // Count non-whitespace characters (c > ' ') with loop unrolling + size_t count = 0; + for (size_t i = 0; i < length; i++) { + count += (input[i] > ' '); + } + + // Check for padding '=' at the end (at most 2 padding characters) + // Scan backwards, skipping whitespace, to find padding + size_t padding = 0; + size_t pos = length; + // Skip trailing whitespace + while (pos > 0 && padding < 2) { + char_type c = input[--pos]; + if (c == '=') { + padding++; + } else if (c > ' ') { + break; + } + } + return ((count - padding) * 3) / 4; +} + +template +simdutf_warn_unused simdutf_constexpr23 full_result +base64_to_binary_details_impl( + const char_type *input, size_t length, char *output, base64_options options, + last_chunk_handling_options last_chunk_options) noexcept { + const bool ignore_garbage = + (options == base64_options::base64_url_accept_garbage) || + (options == base64_options::base64_default_accept_garbage) || + (options == base64_options::base64_default_or_url_accept_garbage); + auto ri = simdutf::scalar::base64::find_end(input, length, options); + size_t equallocation = ri.equallocation; + size_t equalsigns = ri.equalsigns; + length = ri.srclen; + size_t full_input_length = ri.full_input_length; + if (length == 0) { + if (!ignore_garbage && equalsigns > 0) { + return {INVALID_BASE64_CHARACTER, equallocation, 0, true}; + } + return {SUCCESS, full_input_length, 0}; + } + full_result r = scalar::base64::base64_tail_decode( + output, input, length, equalsigns, options, last_chunk_options); + r = scalar::base64::patch_tail_result(r, 0, 0, equallocation, + full_input_length, last_chunk_options); + if (!is_partial(last_chunk_options) && r.error == error_code::SUCCESS && + equalsigns > 0 && !ignore_garbage) { + // additional checks + if ((r.output_count % 3 == 0) || + ((r.output_count % 3) + 1 + equalsigns != 4)) { + return {INVALID_BASE64_CHARACTER, equallocation, r.output_count, true}; + } + } + // When is_partial(last_chunk_options) is true, we must either end with + // the end of the stream (beyond whitespace) or right after a non-ignorable + // character or at the very beginning of the stream. + // See https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 + if (is_partial(last_chunk_options) && r.error == error_code::SUCCESS && + r.input_count < full_input_length) { + // First check if we can extend the input to the end of the stream + while (r.input_count < full_input_length && + base64_ignorable(*(input + r.input_count), options)) { + r.input_count++; + } + // If we are still not at the end of the stream, then we must backtrack + // to the last non-ignorable character. + if (r.input_count < full_input_length) { + while (r.input_count > 0 && + base64_ignorable(*(input + r.input_count - 1), options)) { + r.input_count--; + } + } + } + return r; +} + +template +simdutf_constexpr23 simdutf_warn_unused full_result +base64_to_binary_details_safe_impl( + const char_type *input, size_t length, char *output, size_t outlen, + base64_options options, + last_chunk_handling_options last_chunk_options) noexcept { + const bool ignore_garbage = + (options == base64_options::base64_url_accept_garbage) || + (options == base64_options::base64_default_accept_garbage) || + (options == base64_options::base64_default_or_url_accept_garbage); + auto ri = simdutf::scalar::base64::find_end(input, length, options); + size_t equallocation = ri.equallocation; + size_t equalsigns = ri.equalsigns; + length = ri.srclen; + size_t full_input_length = ri.full_input_length; + if (length == 0) { + if (!ignore_garbage && equalsigns > 0) { + return {INVALID_BASE64_CHARACTER, equallocation, 0}; + } + return {SUCCESS, full_input_length, 0}; + } + full_result r = scalar::base64::base64_tail_decode_safe( + output, outlen, input, length, equalsigns, options, last_chunk_options); + r = scalar::base64::patch_tail_result(r, 0, 0, equallocation, + full_input_length, last_chunk_options); + if (!is_partial(last_chunk_options) && r.error == error_code::SUCCESS && + equalsigns > 0 && !ignore_garbage) { + // additional checks + if ((r.output_count % 3 == 0) || + ((r.output_count % 3) + 1 + equalsigns != 4)) { + return {INVALID_BASE64_CHARACTER, equallocation, r.output_count}; + } + } + + // When is_partial(last_chunk_options) is true, we must either end with + // the end of the stream (beyond whitespace) or right after a non-ignorable + // character or at the very beginning of the stream. + // See https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 + if (is_partial(last_chunk_options) && r.error == error_code::SUCCESS && + r.input_count < full_input_length) { + // First check if we can extend the input to the end of the stream + while (r.input_count < full_input_length && + base64_ignorable(*(input + r.input_count), options)) { + r.input_count++; + } + // If we are still not at the end of the stream, then we must backtrack + // to the last non-ignorable character. + if (r.input_count < full_input_length) { + while (r.input_count > 0 && + base64_ignorable(*(input + r.input_count - 1), options)) { + r.input_count--; + } + } + } + return r; +} + +simdutf_warn_unused simdutf_constexpr23 size_t +base64_length_from_binary(size_t length, base64_options options) noexcept { + // By default, we use padding if we are not using the URL variant. + // This is check with ((options & base64_url) == 0) which returns true if we + // are not using the URL variant. However, we also allow 'inversion' of the + // convention with the base64_reverse_padding option. If the + // base64_reverse_padding option is set, we use padding if we are using the + // URL variant, and we omit it if we are not using the URL variant. This is + // checked with + // ((options & base64_reverse_padding) == base64_reverse_padding). + bool use_padding = + ((options & base64_url) == 0) ^ + ((options & base64_reverse_padding) == base64_reverse_padding); + if (!use_padding) { + return length / 3 * 4 + ((length % 3) ? (length % 3) + 1 : 0); + } + return (length + 2) / 3 * + 4; // We use padding to make the length a multiple of 4. +} + +simdutf_warn_unused simdutf_constexpr23 size_t +base64_length_from_binary_with_lines(size_t length, base64_options options, + size_t line_length) noexcept { + if (length == 0) { + return 0; + } + size_t base64_length = + scalar::base64::base64_length_from_binary(length, options); + if (line_length < 4) { + line_length = 4; + } + size_t lines = + (base64_length + line_length - 1) / line_length; // number of lines + return base64_length + lines - 1; +} + +// Return the length of the prefix that contains count base64 characters. +// Thus, if count is 3, the function returns the length of the prefix +// that contains 3 base64 characters. +// The function returns (size_t)-1 if there is not enough base64 characters in +// the input. +template +simdutf_warn_unused size_t prefix_length(size_t count, + simdutf::base64_options options, + const char_type *input, + size_t length) noexcept { + size_t i = 0; + while (i < length && is_ignorable(input[i], options)) { + i++; + } + if (count == 0) { + return i; // duh! + } + for (; i < length; i++) { + if (is_ignorable(input[i], options)) { + continue; + } + // We have a base64 character or a padding character. + count--; + if (count == 0) { + return i + 1; + } + } + simdutf_log_assert(false, "You never get here"); + + return -1; // should never happen +} + +} // namespace base64 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/base64.h */ + +namespace simdutf { + +inline std::string_view to_string(base64_options options) { + switch (options) { + case base64_default: + return "base64_default"; + case base64_url: + return "base64_url"; + case base64_reverse_padding: + return "base64_reverse_padding"; + case base64_url_with_padding: + return "base64_url_with_padding"; + case base64_default_accept_garbage: + return "base64_default_accept_garbage"; + case base64_url_accept_garbage: + return "base64_url_accept_garbage"; + case base64_default_or_url: + return "base64_default_or_url"; + case base64_default_or_url_accept_garbage: + return "base64_default_or_url_accept_garbage"; + } + return ""; +} + +inline std::string_view to_string(last_chunk_handling_options options) { + switch (options) { + case loose: + return "loose"; + case strict: + return "strict"; + case stop_before_partial: + return "stop_before_partial"; + case only_full_chunks: + return "only_full_chunks"; + } + return ""; +} + +/** + * Provide the maximal binary length in bytes given the base64 input. + * As long as the input does not contain ignorable characters (e.g., ASCII + * spaces or linefeed characters), the result is exact. In particular, the + * function checks for padding characters. + * + * The function is fast (constant time). It checks up to two characters at + * the end of the string. The input is not otherwise validated or read. + * + * @param input the base64 input to process + * @param length the length of the base64 input in bytes + * @return maximum number of binary bytes + */ +simdutf_warn_unused size_t +maximal_binary_length_from_base64(const char *input, size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +maximal_binary_length_from_base64( + const detail::input_span_of_byte_like auto &input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::maximal_binary_length_from_base64( + detail::constexpr_cast_ptr(input.data()), input.size()); + } else + #endif + { + return maximal_binary_length_from_base64( + reinterpret_cast(input.data()), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Provide the maximal binary length in bytes given the base64 input. + * As long as the input does not contain ignorable characters (e.g., ASCII + * spaces or linefeed characters), the result is exact. In particular, the + * function checks for padding characters. + * + * The function is fast (constant time). It checks up to two characters at + * the end of the string. The input is not otherwise validated or read. + * + * @param input the base64 input to process, in ASCII stored as 16-bit + * units + * @param length the length of the base64 input in 16-bit units + * @return maximal number of binary bytes + */ +simdutf_warn_unused size_t maximal_binary_length_from_base64( + const char16_t *input, size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +maximal_binary_length_from_base64(std::span input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::maximal_binary_length_from_base64(input.data(), + input.size()); + } else + #endif + { + return maximal_binary_length_from_base64(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the binary length from a base64 input. + * This function is useful for base64 inputs that may contain ASCII whitespaces + * (such as line breaks). For such inputs, the result is exact, and for any + * inputs the result can be used to size the output buffer passed to + * `base64_to_binary`. + * + * The function ignores whitespace and does not require padding characters + * ('='). + * + * @param input the base64 input to process + * @param length the length of the base64 input in bytes + * @return number of binary bytes + */ +simdutf_warn_unused size_t binary_length_from_base64(const char *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +binary_length_from_base64( + const detail::input_span_of_byte_like auto &input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::binary_length_from_base64(input.data(), + input.size()); + } else + #endif + { + return binary_length_from_base64( + reinterpret_cast(input.data()), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the binary length from a base64 input. + * This function is useful for base64 inputs that may contain ASCII whitespaces + * (such as line breaks). For such inputs, the result is exact, and for any + * inputs the result can be used to size the output buffer passed to + * `base64_to_binary`. + * + * The function ignores whitespace and does not require padding characters + * ('='). + * + * @param input the base64 input to process, in ASCII stored as 16-bit + * units + * @param length the length of the base64 input in 16-bit units + * @return number of binary bytes + */ +simdutf_warn_unused size_t binary_length_from_base64(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +binary_length_from_base64(std::span input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::binary_length_from_base64(input.data(), + input.size()); + } else + #endif + { + return binary_length_from_base64(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert a base64 input to a binary output. + * + * This function follows the WHATWG forgiving-base64 format, which means that it + * will ignore any ASCII spaces in the input. You may provide a padded input + * (with one or two equal signs at the end) or an unpadded input (without any + * equal signs at the end). + * + * See https://infra.spec.whatwg.org/#forgiving-base64-decode * * This function will fail in case of invalid input. When last_chunk_options = * loose, there are two possible reasons for failure: the input contains a @@ -1388,15 +7167,25 @@ simdutf_warn_unused result base64_to_binary( base64_options options = base64_default, last_chunk_handling_options last_chunk_options = loose) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused result base64_to_binary( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +base64_to_binary( const detail::input_span_of_byte_like auto &input, detail::output_span_of_byte_like auto &&binary_output, base64_options options = base64_default, last_chunk_handling_options last_chunk_options = loose) noexcept { - return base64_to_binary(reinterpret_cast(input.data()), - input.size(), - reinterpret_cast(binary_output.data()), - options, last_chunk_options); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::base64_to_binary_details_impl( + input.data(), input.size(), binary_output.data(), options, + last_chunk_options); + } else + #endif + { + return base64_to_binary(reinterpret_cast(input.data()), + input.size(), + reinterpret_cast(binary_output.data()), + options, last_chunk_options); + } } #endif // SIMDUTF_SPAN @@ -1406,8 +7195,27 @@ simdutf_really_inline simdutf_warn_unused result base64_to_binary( * @param length the length of the input in bytes * @return number of base64 bytes */ -simdutf_warn_unused size_t base64_length_from_binary( - size_t length, base64_options options = base64_default) noexcept; +inline simdutf_warn_unused simdutf_constexpr23 size_t base64_length_from_binary( + size_t length, base64_options options = base64_default) noexcept { + return scalar::base64::base64_length_from_binary(length, options); +} + +/** + * Provide the base64 length in bytes given the length of a binary input, + * taking into account line breaks. + * + * @param length the length of the input in bytes + * @param line_length the length of lines, must be at least 4 (otherwise it is + * interpreted as 4), + * @return number of base64 bytes + */ +inline simdutf_warn_unused simdutf_constexpr23 size_t +base64_length_from_binary_with_lines( + size_t length, base64_options options = base64_default, + size_t line_length = default_line_length) noexcept { + return scalar::base64::base64_length_from_binary_with_lines(length, options, + line_length); +} /** * Convert a binary input to a base64 output. @@ -1433,13 +7241,70 @@ simdutf_warn_unused size_t base64_length_from_binary( size_t binary_to_base64(const char *input, size_t length, char *output, base64_options options = base64_default) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t binary_to_base64(const detail::input_span_of_byte_like auto &input, detail::output_span_of_byte_like auto &&binary_output, base64_options options = base64_default) noexcept { - return binary_to_base64( - reinterpret_cast(input.data()), input.size(), - reinterpret_cast(binary_output.data()), options); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::tail_encode_base64( + binary_output.data(), input.data(), input.size(), options); + } else + #endif + { + return binary_to_base64( + reinterpret_cast(input.data()), input.size(), + reinterpret_cast(binary_output.data()), options); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert a binary input to a base64 output with line breaks. + * + * The default option (simdutf::base64_default) uses the characters `+` and `/` + * as part of its alphabet. Further, it adds padding (`=`) at the end of the + * output to ensure that the output length is a multiple of four. + * + * The URL option (simdutf::base64_url) uses the characters `-` and `_` as part + * of its alphabet. No padding is added at the end of the output. + * + * This function always succeeds. + * + * @param input the binary to process + * @param length the length of the input in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least base64_length_from_binary_with_lines(length, + * options, line_length) bytes long) + * @param line_length the length of lines, must be at least 4 (otherwise it is + * interpreted as 4), + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return number of written bytes, will be equal to + * base64_length_from_binary_with_lines(length, options) + */ +size_t +binary_to_base64_with_lines(const char *input, size_t length, char *output, + size_t line_length = simdutf::default_line_length, + base64_options options = base64_default) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +binary_to_base64_with_lines( + const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&binary_output, + size_t line_length = simdutf::default_line_length, + base64_options options = base64_default) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::tail_encode_base64_impl( + binary_output.data(), input.data(), input.size(), options, line_length); + } else + #endif + { + return binary_to_base64_with_lines( + reinterpret_cast(input.data()), input.size(), + reinterpret_cast(binary_output.data()), line_length, options); + } } #endif // SIMDUTF_SPAN @@ -1562,32 +7427,197 @@ base64_to_binary(const char16_t *input, size_t length, char *output, last_chunk_handling_options last_chunk_options = last_chunk_handling_options::loose) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused result base64_to_binary( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +base64_to_binary( + std::span input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::base64_to_binary_details_impl( + input.data(), input.size(), binary_output.data(), options, + last_chunk_options); + } else + #endif + { + return base64_to_binary(input.data(), input.size(), + reinterpret_cast(binary_output.data()), + options, last_chunk_options); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert a base64 input to a binary output while returning more details + * than base64_to_binary. + * + * This function follows the WHATWG forgiving-base64 format, which means that it + * will ignore any ASCII spaces in the input. You may provide a padded input + * (with one or two equal signs at the end) or an unpadded input (without any + * equal signs at the end). + * + * See https://infra.spec.whatwg.org/#forgiving-base64-decode + * + * Unlike base64_to_binary, this function returns a full_result with both + * input_count and output_count, so you always know how much input was consumed + * and how much output was written. There are three cases where the input may + * not be fully consumed: + * + * 1. stop_before_partial: When last_chunk_options is set to + * stop_before_partial, any incomplete 4-character group at the end of the + * input is left unconsumed. This is useful for streaming/chunked decoding + * where you can carry over the unconsumed input to the next chunk. + * + * 2. INVALID_BASE64_CHARACTER: The input contains a character that is not a + * valid base64 character. In this case, input_count indicates where the + * invalid character was found. + * + * 3. BASE64_INPUT_REMAINDER: When last_chunk_options is loose, the input + * contains a number of base64 characters that, when divided by 4, leaves + * a single remainder character (which cannot encode any bytes). + * + * You should call this function with a buffer that is at least + * maximal_binary_length_from_base64(input, length) bytes long. If you fail to + * provide that much space, the function may cause a buffer overflow. + * + * @param input the base64 string to process + * @param length the length of the string in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least maximal_binary_length_from_base64(input, length) + * bytes long). + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @param last_chunk_options the last chunk handling options, + * last_chunk_handling_options::loose by default + * but can also be last_chunk_handling_options::strict or + * last_chunk_handling_options::stop_before_partial. + * @return a full_result struct (of type simdutf::full_result containing the + * three fields error, input_count and output_count). + */ +simdutf_warn_unused full_result +base64_to_binary_details(const char *input, size_t length, char *output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 full_result +base64_to_binary_details( + const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::base64_to_binary_details_impl( + input.data(), input.size(), binary_output.data(), options, + last_chunk_options); + } else + #endif + { + return base64_to_binary_details( + reinterpret_cast(input.data()), input.size(), + reinterpret_cast(binary_output.data()), options, + last_chunk_options); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert a base64 input to a binary output while returning more details + * than base64_to_binary. + * + * This function follows the WHATWG forgiving-base64 format, which means that it + * will ignore any ASCII spaces in the input. You may provide a padded input + * (with one or two equal signs at the end) or an unpadded input (without any + * equal signs at the end). + * + * See https://infra.spec.whatwg.org/#forgiving-base64-decode + * + * Unlike base64_to_binary, this function returns a full_result with both + * input_count and output_count, so you always know how much input was consumed + * and how much output was written. There are three cases where the input may + * not be fully consumed: + * + * 1. stop_before_partial: When last_chunk_options is set to + * stop_before_partial, any incomplete 4-character group at the end of the + * input is left unconsumed. This is useful for streaming/chunked decoding + * where you can carry over the unconsumed input to the next chunk. + * + * 2. INVALID_BASE64_CHARACTER: The input contains a character that is not a + * valid base64 character. In this case, input_count indicates where the + * invalid character was found. + * + * 3. BASE64_INPUT_REMAINDER: When last_chunk_options is loose, the input + * contains a number of base64 characters that, when divided by 4, leaves + * a single remainder character (which cannot encode any bytes). + * + * You should call this function with a buffer that is at least + * maximal_binary_length_from_base64(input, length) bytes long. If you fail to + * provide that much space, the function may cause a buffer overflow. + * + * @param input the base64 string to process, in ASCII stored as 16-bit + * units + * @param length the length of the string in 16-bit units + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least maximal_binary_length_from_base64(input, length) + * bytes long). + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @param last_chunk_options the last chunk handling options, + * last_chunk_handling_options::loose by default + * but can also be last_chunk_handling_options::strict or + * last_chunk_handling_options::stop_before_partial. + * @return a full_result struct (of type simdutf::full_result containing the + * three fields error, input_count and output_count). + */ +simdutf_warn_unused full_result +base64_to_binary_details(const char16_t *input, size_t length, char *output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 full_result +base64_to_binary_details( std::span input, detail::output_span_of_byte_like auto &&binary_output, base64_options options = base64_default, last_chunk_handling_options last_chunk_options = loose) noexcept { - return base64_to_binary(input.data(), input.size(), - reinterpret_cast(binary_output.data()), - options, last_chunk_options); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::base64_to_binary_details_impl( + input.data(), input.size(), binary_output.data(), options, + last_chunk_options); + } else + #endif + { + return base64_to_binary_details( + input.data(), input.size(), + reinterpret_cast(binary_output.data()), options, + last_chunk_options); + } } #endif // SIMDUTF_SPAN /** - * Check if a character is an ignorabl base64 character. + * Check if a character is an ignorable base64 character. * Checking a large input, character by character, is not computationally * efficient. * * @param input the character to check * @param options the base64 options to use, is base64_default by default. - * @return true if the character is an ignorablee base64 character, false + * @return true if the character is an ignorable base64 character, false * otherwise. */ -simdutf_warn_unused bool -base64_ignorable(char input, base64_options options = base64_default) noexcept; -simdutf_warn_unused bool +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool +base64_ignorable(char input, base64_options options = base64_default) noexcept { + return scalar::base64::is_ignorable(input, options); +} +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool base64_ignorable(char16_t input, - base64_options options = base64_default) noexcept; + base64_options options = base64_default) noexcept { + return scalar::base64::is_ignorable(input, options); +} /** * Check if a character is a valid base64 character. @@ -1600,10 +7630,14 @@ base64_ignorable(char16_t input, * @param options the base64 options to use, is base64_default by default. * @return true if the character is a base64 character, false otherwise. */ -simdutf_warn_unused bool -base64_valid(char input, base64_options options = base64_default) noexcept; -simdutf_warn_unused bool -base64_valid(char16_t input, base64_options options = base64_default) noexcept; +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool +base64_valid(char input, base64_options options = base64_default) noexcept { + return scalar::base64::is_base64(input, options); +} +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool +base64_valid(char16_t input, base64_options options = base64_default) noexcept { + return scalar::base64::is_base64(input, options); +} /** * Check if a character is a valid base64 character or the padding character @@ -1614,12 +7648,16 @@ base64_valid(char16_t input, base64_options options = base64_default) noexcept; * @param options the base64 options to use, is base64_default by default. * @return true if the character is a base64 character, false otherwise. */ -simdutf_warn_unused bool +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool base64_valid_or_padding(char input, - base64_options options = base64_default) noexcept; -simdutf_warn_unused bool + base64_options options = base64_default) noexcept { + return scalar::base64::is_base64_or_padding(input, options); +} +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool base64_valid_or_padding(char16_t input, - base64_options options = base64_default) noexcept; + base64_options options = base64_default) noexcept { + return scalar::base64::is_base64_or_padding(input, options); +} /** * Convert a base64 input to a binary output. @@ -1694,25 +7732,7 @@ base64_to_binary_safe(const char *input, size_t length, char *output, last_chunk_handling_options last_chunk_options = last_chunk_handling_options::loose, bool decode_up_to_bad_char = false) noexcept; - #if SIMDUTF_SPAN -/** - * @brief span overload - * @return a tuple of result and outlen - */ -simdutf_really_inline simdutf_warn_unused std::tuple -base64_to_binary_safe(const detail::input_span_of_byte_like auto &input, - detail::output_span_of_byte_like auto &&binary_output, - base64_options options = base64_default, - last_chunk_handling_options last_chunk_options = loose, - bool decode_up_to_bad_char = false) noexcept { - size_t outlen = binary_output.size(); - auto r = base64_to_binary_safe( - reinterpret_cast(input.data()), input.size(), - reinterpret_cast(binary_output.data()), outlen, options, - last_chunk_options, decode_up_to_bad_char); - return {r, outlen}; -} - #endif // SIMDUTF_SPAN +// the span overload has moved to the bottom of the file simdutf_warn_unused result base64_to_binary_safe(const char16_t *input, size_t length, char *output, @@ -1720,25 +7740,7 @@ base64_to_binary_safe(const char16_t *input, size_t length, char *output, last_chunk_handling_options last_chunk_options = last_chunk_handling_options::loose, bool decode_up_to_bad_char = false) noexcept; - #if SIMDUTF_SPAN -/** - * @brief span overload - * @return a tuple of result and outlen - */ -simdutf_really_inline simdutf_warn_unused std::tuple -base64_to_binary_safe(std::span input, - detail::output_span_of_byte_like auto &&binary_output, - base64_options options = base64_default, - last_chunk_handling_options last_chunk_options = loose, - bool decode_up_to_bad_char = false) noexcept { - size_t outlen = binary_output.size(); - auto r = base64_to_binary_safe(input.data(), input.size(), - reinterpret_cast(binary_output.data()), - outlen, options, last_chunk_options, - decode_up_to_bad_char); - return {r, outlen}; -} - #endif // SIMDUTF_SPAN + // span overload moved to bottom of file #if SIMDUTF_ATOMIC_REF /** @@ -1832,21 +7834,6 @@ atomic_base64_to_binary_safe( #endif // SIMDUTF_SPAN #endif // SIMDUTF_ATOMIC_REF -/** - * Find the first occurrence of a character in a string. If the character is - * not found, return a pointer to the end of the string. - * @param start the start of the string - * @param end the end of the string - * @param character the character to find - * @return a pointer to the first occurrence of the character in the string, - * or a pointer to the end of the string if the character is not found. - * - */ -simdutf_warn_unused const char *find(const char *start, const char *end, - char character) noexcept; -simdutf_warn_unused const char16_t * -find(const char16_t *start, const char16_t *end, char16_t character) noexcept; - /** * An implementation of simdutf for a particular CPU architecture. * @@ -1865,7 +7852,7 @@ class implementation { * * @return the name of the implementation, e.g. "haswell", "westmere", "arm64" */ - virtual std::string name() const { return std::string(_name); } + virtual std::string_view name() const noexcept { return _name; } /** * The description of this implementation. @@ -1876,7 +7863,7 @@ class implementation { * * @return the name of the implementation, e.g. "haswell", "westmere", "arm64" */ - virtual std::string description() const { return std::string(_description); } + virtual std::string_view description() const noexcept { return _description; } /** * The instruction sets this implementation is compiled against @@ -1902,9 +7889,12 @@ class implementation { /** * Provide the maximal binary length in bytes given the base64 input. - * In general, if the input contains ASCII spaces, the result will be less - * than the maximum length. It is acceptable to pass invalid base64 strings - * but in such cases the result is implementation defined. + * As long as the input does not contain ignorable characters (e.g., ASCII + * spaces or linefeed characters), the result is exact. In particular, the + * function checks for padding characters. + * + * The function is fast (constant time). It checks up to two characters at + * the end of the string. The input is not otherwise validated or read.. * * @param input the base64 input to process * @param length the length of the base64 input in bytes @@ -1915,9 +7905,12 @@ class implementation { /** * Provide the maximal binary length in bytes given the base64 input. - * In general, if the input contains ASCII spaces, the result will be less - * than the maximum length. It is acceptable to pass invalid base64 strings - * but in such cases the result is implementation defined. + * As long as the input does not contain ignorable characters (e.g., ASCII + * spaces or linefeed characters), the result is exact. In particular, the + * function checks for padding characters. + * + * The function is fast (constant time). It checks up to two characters at + * the end of the string. The input is not otherwise validated or read. * * @param input the base64 input to process, in ASCII stored as 16-bit * units @@ -1927,6 +7920,38 @@ class implementation { simdutf_warn_unused size_t maximal_binary_length_from_base64( const char16_t *input, size_t length) const noexcept; + /** + * Compute the binary length from a base64 input with ASCII spaces. + * This function is useful for well-formed base64 inputs that may contain + * ASCII spaces (such as line breaks). For such inputs, the result is exact. + * + * The function counts non-whitespace characters (ASCII value > 0x20) and + * subtracts padding characters ('=') found at the end. + * + * @param input the base64 input to process + * @param length the length of the base64 input in bytes + * @return number of binary bytes + */ + simdutf_warn_unused virtual size_t + binary_length_from_base64(const char *input, size_t length) const noexcept; + + /** + * Compute the binary length from a base64 input with ASCII spaces. + * This function is useful for well-formed base64 inputs that may contain + * ASCII spaces (such as line breaks). For such inputs, the result is exact. + * + * The function counts non-whitespace characters (ASCII value > 0x20) and + * subtracts padding characters ('=') found at the end. + * + * @param input the base64 input to process, in ASCII stored as 16-bit + * units + * @param length the length of the base64 input in 16-bit units + * @return number of binary bytes + */ + simdutf_warn_unused virtual size_t + binary_length_from_base64(const char16_t *input, + size_t length) const noexcept; + /** * Convert a base64 input to a binary output. * @@ -2001,6 +8026,7 @@ class implementation { base64_options options = base64_default, last_chunk_handling_options last_chunk_options = last_chunk_handling_options::loose) const noexcept = 0; + /** * Convert a base64 input to a binary output. * @@ -2076,11 +8102,12 @@ class implementation { base64_options options = base64_default, last_chunk_handling_options last_chunk_options = last_chunk_handling_options::loose) const noexcept = 0; + /** * Provide the base64 length in bytes given the length of a binary input. * * @param length the length of the input in bytes - * @parem options the base64 options to use, can be base64_default or + * @param options the base64 options to use, can be base64_default or * base64_url, is base64_default by default. * @return number of base64 bytes */ @@ -2111,6 +8138,37 @@ class implementation { virtual size_t binary_to_base64(const char *input, size_t length, char *output, base64_options options = base64_default) const noexcept = 0; + + /** + * Convert a binary input to a base64 output with lines of given length. + * Lines are separated by a single linefeed character. + * + * The default option (simdutf::base64_default) uses the characters `+` and + * `/` as part of its alphabet. Further, it adds padding (`=`) at the end of + * the output to ensure that the output length is a multiple of four. + * + * The URL option (simdutf::base64_url) uses the characters `-` and `_` as + * part of its alphabet. No padding is added at the end of the output. + * + * This function always succeeds. + * + * @param input the binary to process + * @param length the length of the input in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least base64_length_from_binary_with_lines(length, + * options, line_length) bytes long) + * @param line_length the length of each line, values smaller than 4 are + * interpreted as 4 + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return number of written bytes, will be equal to + * base64_length_from_binary_with_lines(length, options, line_length) + */ + virtual size_t binary_to_base64_with_lines( + const char *input, size_t length, char *output, + size_t line_length = simdutf::default_line_length, + base64_options options = base64_default) const noexcept = 0; + /** * Find the first occurrence of a character in a string. If the character is * not found, return a pointer to the end of the string. @@ -2137,7 +8195,7 @@ class implementation { struct TestProcedure { // display name - std::string name; + std::string_view name; // procedure should return whether given test pass or not void (*procedure)(const implementation &); @@ -2205,7 +8263,7 @@ class available_implementation_list { * @param name the implementation to find, e.g. "westmere", "haswell", "arm64" * @return the implementation, or nullptr if the parse failed. */ - const implementation *operator[](const std::string &name) const noexcept { + const implementation *operator[](std::string_view name) const noexcept { for (const implementation *impl : *this) { if (impl->name() == name) { return impl; @@ -2291,6 +8349,316 @@ get_active_implementation(); } // namespace simdutf + // this header is not part of the public api +/* begin file include/simdutf/base64_implementation.h */ +#ifndef SIMDUTF_BASE64_IMPLEMENTATION_H +#define SIMDUTF_BASE64_IMPLEMENTATION_H + +// this is not part of the public api + +namespace simdutf { + +template +simdutf_warn_unused simdutf_constexpr23 result slow_base64_to_binary_safe_impl( + const chartype *input, size_t length, char *output, size_t &outlen, + base64_options options, + last_chunk_handling_options last_chunk_options) noexcept { + const bool ignore_garbage = (options & base64_default_accept_garbage) != 0; + auto ri = simdutf::scalar::base64::find_end(input, length, options); + size_t equallocation = ri.equallocation; + size_t equalsigns = ri.equalsigns; + length = ri.srclen; + size_t full_input_length = ri.full_input_length; + (void)full_input_length; + if (length == 0) { + outlen = 0; + if (!ignore_garbage && equalsigns > 0) { + return {INVALID_BASE64_CHARACTER, equallocation}; + } + return {SUCCESS, 0}; + } + + // The parameters of base64_tail_decode_safe are: + // - dst: the output buffer + // - outlen: the size of the output buffer + // - srcr: the input buffer + // - length: the size of the input buffer + // - padded_characters: the number of padding characters + // - options: the options for the base64 decoder + // - last_chunk_options: the options for the last chunk + // The function will return the number of bytes written to the output buffer + // and the number of bytes read from the input buffer. + // The function will also return an error code if the input buffer is not + // valid base64. + full_result r = scalar::base64::base64_tail_decode_safe( + output, outlen, input, length, equalsigns, options, last_chunk_options); + r = scalar::base64::patch_tail_result(r, 0, 0, equallocation, + full_input_length, last_chunk_options); + outlen = r.output_count; + if (!is_partial(last_chunk_options) && r.error == error_code::SUCCESS && + equalsigns > 0) { + // additional checks + if ((outlen % 3 == 0) || ((outlen % 3) + 1 + equalsigns != 4)) { + r.error = error_code::INVALID_BASE64_CHARACTER; + } + } + return {r.error, r.input_count}; // we cannot return r itself because it gets + // converted to error/output_count +} + +template +simdutf_warn_unused simdutf_constexpr23 result base64_to_binary_safe_impl( + const chartype *input, size_t length, char *output, size_t &outlen, + base64_options options, + last_chunk_handling_options last_chunk_handling_options, + bool decode_up_to_bad_char) noexcept { + static_assert(std::is_same::value || + std::is_same::value, + "Only char and char16_t are supported."); + size_t remaining_input_length = length; + size_t remaining_output_length = outlen; + size_t input_position = 0; + size_t output_position = 0; + + // We also do a first pass using the fast path to decode as much as possible + size_t safe_input = (std::min)( + remaining_input_length, + base64_length_from_binary(remaining_output_length / 3 * 3, options)); + bool done_with_partial = (safe_input == remaining_input_length); + simdutf::full_result r; + +#if SIMDUTF_CPLUSPLUS23 + if consteval { + r = scalar::base64::base64_to_binary_details_impl( + input + input_position, safe_input, output + output_position, options, + done_with_partial + ? last_chunk_handling_options + : simdutf::last_chunk_handling_options::only_full_chunks); + } else +#endif + { + r = get_active_implementation()->base64_to_binary_details( + input + input_position, safe_input, output + output_position, options, + done_with_partial + ? last_chunk_handling_options + : simdutf::last_chunk_handling_options::only_full_chunks); + } + simdutf_log_assert(r.input_count <= safe_input, + "You should not read more than safe_input"); + simdutf_log_assert(r.output_count <= remaining_output_length, + "You should not write more than remaining_output_length"); + // Technically redundant, but we want to be explicit about it. + input_position += r.input_count; + output_position += r.output_count; + remaining_input_length -= r.input_count; + remaining_output_length -= r.output_count; + if (r.error != simdutf::error_code::SUCCESS) { + // There is an error. We return. + if (decode_up_to_bad_char && + r.error == error_code::INVALID_BASE64_CHARACTER) { + return slow_base64_to_binary_safe_impl( + input, length, output, outlen, options, last_chunk_handling_options); + } + outlen = output_position; + return {r.error, input_position}; + } + + if (done_with_partial) { + // We are done. We have decoded everything. + outlen = output_position; + return {simdutf::error_code::SUCCESS, input_position}; + } + // We have decoded some data, but we still have some data to decode. + // We need to decode the rest of the input buffer. + r = simdutf::scalar::base64::base64_to_binary_details_safe_impl( + input + input_position, remaining_input_length, output + output_position, + remaining_output_length, options, last_chunk_handling_options); + input_position += r.input_count; + output_position += r.output_count; + remaining_input_length -= r.input_count; + remaining_output_length -= r.output_count; + + if (r.error != simdutf::error_code::SUCCESS) { + // There is an error. We return. + if (decode_up_to_bad_char && + r.error == error_code::INVALID_BASE64_CHARACTER) { + return slow_base64_to_binary_safe_impl( + input, length, output, outlen, options, last_chunk_handling_options); + } + outlen = output_position; + return {r.error, input_position}; + } + if (input_position < length) { + // We cannot process the entire input in one go, so we need to + // process it in two steps: first the fast path, then the slow path. + // In some cases, the processing might 'eat up' trailing ignorable + // characters in the fast path, but that can be a problem. + // suppose we have just white space followed by a single base64 character. + // If we first process the white space with the fast path, it will + // eat all of it. But, by the JavaScript standard, we should consume + // no character. See + // https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 + while (input_position > 0 && + base64_ignorable(input[input_position - 1], options)) { + input_position--; + } + } + outlen = output_position; + return {simdutf::error_code::SUCCESS, input_position}; +} + +} // namespace simdutf +#endif // SIMDUTF_BASE64_IMPLEMENTATION_H +/* end file include/simdutf/base64_implementation.h */ + +namespace simdutf { + #if SIMDUTF_SPAN +/** + * @brief span overload + * @return a tuple of result and outlen + */ +simdutf_really_inline + simdutf_constexpr23 simdutf_warn_unused std::tuple + base64_to_binary_safe( + const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose, + bool decode_up_to_bad_char = false) noexcept { + size_t outlen = binary_output.size(); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + using CInput = std::decay_t; + static_assert(std::is_same_v, + "sorry, the constexpr implementation is for now limited to " + "input of type char"); + using COutput = std::decay_t; + static_assert(std::is_same_v, + "sorry, the constexpr implementation is for now limited to " + "output of type char"); + auto r = base64_to_binary_safe_impl( + input.data(), input.size(), binary_output.data(), outlen, options, + last_chunk_options, decode_up_to_bad_char); + return {r, outlen}; + } else + #endif + { + auto r = base64_to_binary_safe_impl( + reinterpret_cast(input.data()), input.size(), + reinterpret_cast(binary_output.data()), outlen, options, + last_chunk_options, decode_up_to_bad_char); + return {r, outlen}; + } +} + + #if SIMDUTF_SPAN +/** + * @brief span overload + * @return a tuple of result and outlen + */ +simdutf_really_inline + simdutf_warn_unused simdutf_constexpr23 std::tuple + base64_to_binary_safe( + std::span input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose, + bool decode_up_to_bad_char = false) noexcept { + size_t outlen = binary_output.size(); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + auto r = base64_to_binary_safe_impl( + input.data(), input.size(), binary_output.data(), outlen, options, + last_chunk_options, decode_up_to_bad_char); + return {r, outlen}; + } else + #endif + { + auto r = base64_to_binary_safe( + input.data(), input.size(), + reinterpret_cast(binary_output.data()), outlen, options, + last_chunk_options, decode_up_to_bad_char); + return {r, outlen}; + } +} + #endif // SIMDUTF_SPAN + + #endif // SIMDUTF_SPAN +} // namespace simdutf + +#if SIMDUTF_CPLUSPLUS23 && SIMDUTF_FEATURE_BASE64 + +namespace simdutf { +namespace literals { + +namespace detail { + +// the detail namespace is not part of the public api + +template struct base64_literal_helper { + std::array storage{}; + static constexpr std::size_t size() noexcept { return N - 1; } + consteval base64_literal_helper(const char (&str)[N]) { + for (std::size_t i = 0; i < size(); i++) { + storage[i] = str[i]; + } + } +}; + +template struct base64_decode_result { + static constexpr std::size_t max_out = (InputLen + 3) / 4 * 3; + std::array buffer{}; + std::size_t output_count{}; +}; + +template +consteval auto base64_decode_literal(const char *str) { + base64_decode_result result{}; + auto r = scalar::base64::base64_to_binary_details_impl( + str, InputLen, result.buffer.data(), base64_default, loose); + if (r.error != error_code::SUCCESS) { + #if __cpp_lib_unreachable >= 202202L + std::unreachable(); // invalid base64 input in _base64 literal + #else + // workaround for older stdlib + throw "invalid base64 input in _base64 literal"; + #endif + } + result.output_count = r.output_count; + return result; +} + +template consteval auto base64_make_array() { + constexpr auto decoded = base64_decode_literal(a.storage.data()); + std::array ret{}; + for (std::size_t i = 0; i < decoded.output_count; i++) { + ret[i] = decoded.buffer[i]; + } + return ret; +} + +} // namespace detail + +/** + * User-defined literal for compile-time base64 decoding. + * + * Usage: + * using namespace simdutf::literals; + * constexpr auto decoded = "SGVsbG8gV29ybGQh"_base64; + * // decoded is a std::array containing "Hello World!" + * + * The input must be valid base64. Whitepace is allowed and ignored. + * A compilation error occurs if the input is invalid. + */ +template consteval auto operator""_base64() { + return detail::base64_make_array(); +} + +} // namespace literals +} // namespace simdutf + +#endif // SIMDUTF_CPLUSPLUS23 && SIMDUTF_FEATURE_BASE64 + #endif // SIMDUTF_IMPLEMENTATION_H /* end file include/simdutf/implementation.h */ diff --git a/docs/android.png b/docs/android.png index 120011b..2cb121b 100644 Binary files a/docs/android.png and b/docs/android.png differ diff --git a/docs/ios.png b/docs/ios.png index 0bc2da2..501e167 100644 Binary files a/docs/ios.png and b/docs/ios.png differ diff --git a/example/Gemfile.lock b/example/Gemfile.lock index f5fa9e6..4ab57d2 100644 --- a/example/Gemfile.lock +++ b/example/Gemfile.lock @@ -5,18 +5,12 @@ GEM base64 nkf rexml - activesupport (7.2.2.2) - base64 - benchmark (>= 0.3) - bigdecimal - concurrent-ruby (~> 1.0, >= 1.3.1) - connection_pool (>= 2.2.5) - drb + activesupport (6.1.7.10) + concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) - logger (>= 1.4.2) minitest (>= 5.1) - securerandom (>= 0.3) - tzinfo (~> 2.0, >= 2.0.5) + tzinfo (~> 2.0) + zeitwerk (~> 2.3) addressable (2.8.7) public_suffix (>= 2.0.2, < 7.0) algoliasearch (1.27.5) @@ -66,8 +60,6 @@ GEM cocoapods-try (1.2.0) colored2 (3.1.2) concurrent-ruby (1.3.3) - connection_pool (2.5.4) - drb (2.2.3) escape (0.0.4) ethon (0.15.0) ffi (>= 1.15.0) @@ -79,9 +71,9 @@ GEM mutex_m i18n (1.14.7) concurrent-ruby (~> 1.0) - json (2.15.0) + json (2.7.6) logger (1.7.0) - minitest (5.25.5) + minitest (5.25.4) molinillo (0.8.0) mutex_m (0.3.0) nanaimo (0.3.0) @@ -91,7 +83,6 @@ GEM public_suffix (4.0.7) rexml (3.4.4) ruby-macho (2.5.1) - securerandom (0.4.1) typhoeus (1.5.0) ethon (>= 0.9.0, < 0.16.0) tzinfo (2.0.6) @@ -103,6 +94,7 @@ GEM colored2 (~> 3.1) nanaimo (~> 0.3.0) rexml (>= 3.3.6, < 4.0) + zeitwerk (2.6.18) PLATFORMS ruby diff --git a/example/index.js b/example/index.js index 3ddfc58..4ae8f67 100644 --- a/example/index.js +++ b/example/index.js @@ -2,6 +2,6 @@ import { AppRegistry } from 'react-native'; import { name as appName } from './app.json'; import App from './src/App'; import { install } from 'react-native-nitro-base64'; - +import 'react-native-quick-base64'; install(); AppRegistry.registerComponent(appName, () => App); diff --git a/example/ios/NitroBase64Example.xcodeproj/project.pbxproj b/example/ios/NitroBase64Example.xcodeproj/project.pbxproj index 0444ca0..363fef2 100644 --- a/example/ios/NitroBase64Example.xcodeproj/project.pbxproj +++ b/example/ios/NitroBase64Example.xcodeproj/project.pbxproj @@ -7,11 +7,11 @@ objects = { /* Begin PBXBuildFile section */ - 0C80B921A6F3F58F76C31292 /* libPods-NitroBase64Example.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 5DCACB8F33CDC322A6C60F78 /* libPods-NitroBase64Example.a */; }; 13B07FBF1A68108700A75B9A /* Images.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 13B07FB51A68108700A75B9A /* Images.xcassets */; }; 761780ED2CA45674006654EE /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 761780EC2CA45674006654EE /* AppDelegate.swift */; }; 81AB9BB82411601600AC10FF /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 81AB9BB72411601600AC10FF /* LaunchScreen.storyboard */; }; BA8731FA8F4AE30FD34E383A /* PrivacyInfo.xcprivacy in Resources */ = {isa = PBXBuildFile; fileRef = 13B07FB81A68108700A75B9A /* PrivacyInfo.xcprivacy */; }; + FF36CB20B19F64C38348A6E0 /* libPods-NitroBase64Example.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 9AE440FAA83DDAAECFBFDE91 /* libPods-NitroBase64Example.a */; }; /* End PBXBuildFile section */ /* Begin PBXFileReference section */ @@ -19,11 +19,11 @@ 13B07FB51A68108700A75B9A /* Images.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; name = Images.xcassets; path = NitroBase64Example/Images.xcassets; sourceTree = ""; }; 13B07FB61A68108700A75B9A /* Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; name = Info.plist; path = NitroBase64Example/Info.plist; sourceTree = ""; }; 13B07FB81A68108700A75B9A /* PrivacyInfo.xcprivacy */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; name = PrivacyInfo.xcprivacy; path = NitroBase64Example/PrivacyInfo.xcprivacy; sourceTree = ""; }; - 3B4392A12AC88292D35C810B /* Pods-NitroBase64Example.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-NitroBase64Example.debug.xcconfig"; path = "Target Support Files/Pods-NitroBase64Example/Pods-NitroBase64Example.debug.xcconfig"; sourceTree = ""; }; - 5709B34CF0A7D63546082F79 /* Pods-NitroBase64Example.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-NitroBase64Example.release.xcconfig"; path = "Target Support Files/Pods-NitroBase64Example/Pods-NitroBase64Example.release.xcconfig"; sourceTree = ""; }; - 5DCACB8F33CDC322A6C60F78 /* libPods-NitroBase64Example.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-NitroBase64Example.a"; sourceTree = BUILT_PRODUCTS_DIR; }; + 5970CB6C7B61052C670099BF /* Pods-NitroBase64Example.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-NitroBase64Example.debug.xcconfig"; path = "Target Support Files/Pods-NitroBase64Example/Pods-NitroBase64Example.debug.xcconfig"; sourceTree = ""; }; 761780EC2CA45674006654EE /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; name = AppDelegate.swift; path = NitroBase64Example/AppDelegate.swift; sourceTree = ""; }; 81AB9BB72411601600AC10FF /* LaunchScreen.storyboard */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = file.storyboard; name = LaunchScreen.storyboard; path = NitroBase64Example/LaunchScreen.storyboard; sourceTree = ""; }; + 9AE440FAA83DDAAECFBFDE91 /* libPods-NitroBase64Example.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-NitroBase64Example.a"; sourceTree = BUILT_PRODUCTS_DIR; }; + 9E8DAE6BEF6EB943E3A5C42E /* Pods-NitroBase64Example.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-NitroBase64Example.release.xcconfig"; path = "Target Support Files/Pods-NitroBase64Example/Pods-NitroBase64Example.release.xcconfig"; sourceTree = ""; }; ED297162215061F000B7C4FE /* JavaScriptCore.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = JavaScriptCore.framework; path = System/Library/Frameworks/JavaScriptCore.framework; sourceTree = SDKROOT; }; /* End PBXFileReference section */ @@ -32,7 +32,7 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( - 0C80B921A6F3F58F76C31292 /* libPods-NitroBase64Example.a in Frameworks */, + FF36CB20B19F64C38348A6E0 /* libPods-NitroBase64Example.a in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -55,7 +55,7 @@ isa = PBXGroup; children = ( ED297162215061F000B7C4FE /* JavaScriptCore.framework */, - 5DCACB8F33CDC322A6C60F78 /* libPods-NitroBase64Example.a */, + 9AE440FAA83DDAAECFBFDE91 /* libPods-NitroBase64Example.a */, ); name = Frameworks; sourceTree = ""; @@ -92,8 +92,8 @@ BBD78D7AC51CEA395F1C20DB /* Pods */ = { isa = PBXGroup; children = ( - 3B4392A12AC88292D35C810B /* Pods-NitroBase64Example.debug.xcconfig */, - 5709B34CF0A7D63546082F79 /* Pods-NitroBase64Example.release.xcconfig */, + 5970CB6C7B61052C670099BF /* Pods-NitroBase64Example.debug.xcconfig */, + 9E8DAE6BEF6EB943E3A5C42E /* Pods-NitroBase64Example.release.xcconfig */, ); path = Pods; sourceTree = ""; @@ -105,13 +105,13 @@ isa = PBXNativeTarget; buildConfigurationList = 13B07F931A680F5B00A75B9A /* Build configuration list for PBXNativeTarget "NitroBase64Example" */; buildPhases = ( - C38B50BA6285516D6DCD4F65 /* [CP] Check Pods Manifest.lock */, + B60AA6539ECF2D1C459B7F53 /* [CP] Check Pods Manifest.lock */, 13B07F871A680F5B00A75B9A /* Sources */, 13B07F8C1A680F5B00A75B9A /* Frameworks */, 13B07F8E1A680F5B00A75B9A /* Resources */, 00DD1BFF1BD5951E006B06BC /* Bundle React Native code and images */, - 00EEFC60759A1932668264C0 /* [CP] Embed Pods Frameworks */, - E235C05ADACE081382539298 /* [CP] Copy Pods Resources */, + B25FA5EDB160A78A757DC68E /* [CP] Embed Pods Frameworks */, + 7EF932ECA7B9FEC381831A14 /* [CP] Copy Pods Resources */, ); buildRules = ( ); @@ -183,7 +183,24 @@ shellPath = /bin/sh; shellScript = "set -e\n\nWITH_ENVIRONMENT=\"$REACT_NATIVE_PATH/scripts/xcode/with-environment.sh\"\nREACT_NATIVE_XCODE=\"$REACT_NATIVE_PATH/scripts/react-native-xcode.sh\"\n\n/bin/sh -c \"$WITH_ENVIRONMENT $REACT_NATIVE_XCODE\"\n"; }; - 00EEFC60759A1932668264C0 /* [CP] Embed Pods Frameworks */ = { + 7EF932ECA7B9FEC381831A14 /* [CP] Copy Pods Resources */ = { + isa = PBXShellScriptBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + "${PODS_ROOT}/Target Support Files/Pods-NitroBase64Example/Pods-NitroBase64Example-resources-${CONFIGURATION}-input-files.xcfilelist", + ); + name = "[CP] Copy Pods Resources"; + outputFileListPaths = ( + "${PODS_ROOT}/Target Support Files/Pods-NitroBase64Example/Pods-NitroBase64Example-resources-${CONFIGURATION}-output-files.xcfilelist", + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "\"${PODS_ROOT}/Target Support Files/Pods-NitroBase64Example/Pods-NitroBase64Example-resources.sh\"\n"; + showEnvVarsInLog = 0; + }; + B25FA5EDB160A78A757DC68E /* [CP] Embed Pods Frameworks */ = { isa = PBXShellScriptBuildPhase; buildActionMask = 2147483647; files = ( @@ -200,7 +217,7 @@ shellScript = "\"${PODS_ROOT}/Target Support Files/Pods-NitroBase64Example/Pods-NitroBase64Example-frameworks.sh\"\n"; showEnvVarsInLog = 0; }; - C38B50BA6285516D6DCD4F65 /* [CP] Check Pods Manifest.lock */ = { + B60AA6539ECF2D1C459B7F53 /* [CP] Check Pods Manifest.lock */ = { isa = PBXShellScriptBuildPhase; buildActionMask = 2147483647; files = ( @@ -222,23 +239,6 @@ shellScript = "diff \"${PODS_PODFILE_DIR_PATH}/Podfile.lock\" \"${PODS_ROOT}/Manifest.lock\" > /dev/null\nif [ $? != 0 ] ; then\n # print error to STDERR\n echo \"error: The sandbox is not in sync with the Podfile.lock. Run 'pod install' or update your CocoaPods installation.\" >&2\n exit 1\nfi\n# This output is used by Xcode 'outputs' to avoid re-running this script phase.\necho \"SUCCESS\" > \"${SCRIPT_OUTPUT_FILE_0}\"\n"; showEnvVarsInLog = 0; }; - E235C05ADACE081382539298 /* [CP] Copy Pods Resources */ = { - isa = PBXShellScriptBuildPhase; - buildActionMask = 2147483647; - files = ( - ); - inputFileListPaths = ( - "${PODS_ROOT}/Target Support Files/Pods-NitroBase64Example/Pods-NitroBase64Example-resources-${CONFIGURATION}-input-files.xcfilelist", - ); - name = "[CP] Copy Pods Resources"; - outputFileListPaths = ( - "${PODS_ROOT}/Target Support Files/Pods-NitroBase64Example/Pods-NitroBase64Example-resources-${CONFIGURATION}-output-files.xcfilelist", - ); - runOnlyForDeploymentPostprocessing = 0; - shellPath = /bin/sh; - shellScript = "\"${PODS_ROOT}/Target Support Files/Pods-NitroBase64Example/Pods-NitroBase64Example-resources.sh\"\n"; - showEnvVarsInLog = 0; - }; /* End PBXShellScriptBuildPhase section */ /* Begin PBXSourcesBuildPhase section */ @@ -255,11 +255,12 @@ /* Begin XCBuildConfiguration section */ 13B07F941A680F5B00A75B9A /* Debug */ = { isa = XCBuildConfiguration; - baseConfigurationReference = 3B4392A12AC88292D35C810B /* Pods-NitroBase64Example.debug.xcconfig */; + baseConfigurationReference = 5970CB6C7B61052C670099BF /* Pods-NitroBase64Example.debug.xcconfig */; buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CLANG_ENABLE_MODULES = YES; CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = 4QDQZA9V92; ENABLE_BITCODE = NO; INFOPLIST_FILE = NitroBase64Example/Info.plist; IPHONEOS_DEPLOYMENT_TARGET = 16.6; @@ -283,11 +284,12 @@ }; 13B07F951A680F5B00A75B9A /* Release */ = { isa = XCBuildConfiguration; - baseConfigurationReference = 5709B34CF0A7D63546082F79 /* Pods-NitroBase64Example.release.xcconfig */; + baseConfigurationReference = 9E8DAE6BEF6EB943E3A5C42E /* Pods-NitroBase64Example.release.xcconfig */; buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CLANG_ENABLE_MODULES = YES; CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = 4QDQZA9V92; INFOPLIST_FILE = NitroBase64Example/Info.plist; IPHONEOS_DEPLOYMENT_TARGET = 16.6; LD_RUNPATH_SEARCH_PATHS = ( diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock index df8655d..1175979 100644 --- a/example/ios/Podfile.lock +++ b/example/ios/Podfile.lock @@ -8,7 +8,7 @@ PODS: - hermes-engine (0.81.1): - hermes-engine/Pre-built (= 0.81.1) - hermes-engine/Pre-built (0.81.1) - - NitroBase64 (0.8.0): + - NitroBase64 (1.1.0): - boost - DoubleConversion - fast_float @@ -38,7 +38,7 @@ PODS: - ReactCommon/turbomodule/core - SocketRocket - Yoga - - NitroModules (0.29.8): + - NitroModules (0.35.6): - boost - DoubleConversion - fast_float @@ -1807,8 +1807,34 @@ PODS: - React-RCTFBReactNativeSpec - ReactCommon/turbomodule/core - SocketRocket - - react-native-quick-base64 (2.2.2): + - react-native-quick-base64 (3.0.0): + - boost + - DoubleConversion + - fast_float + - fmt + - glog + - hermes-engine + - RCT-Folly + - RCT-Folly/Fabric + - RCTRequired + - RCTTypeSafety - React-Core + - React-debug + - React-Fabric + - React-featureflags + - React-graphics + - React-ImageManager + - React-jsi + - React-NativeModulesApple + - React-RCTFabric + - React-renderercss + - React-rendererdebug + - React-utils + - ReactCodegen + - ReactCommon/turbomodule/bridging + - ReactCommon/turbomodule/core + - SocketRocket + - Yoga - react-native-safe-area-context (5.6.1): - boost - DoubleConversion @@ -2648,8 +2674,8 @@ SPEC CHECKSUMS: fmt: a40bb5bd0294ea969aaaba240a927bd33d878cdd glog: 5683914934d5b6e4240e497e0f4a3b42d1854183 hermes-engine: 4f8246b1f6d79f625e0d99472d1f3a71da4d28ca - NitroBase64: a902e6a58f53c5854d5fcfe1e8db4a29f89c2468 - NitroModules: 808eee1dfaf75b2c5a63ea3608cb651cc7d13687 + NitroBase64: f35489547b9de21ebce93bfd2f5f39f42fe83d1e + NitroModules: 70e2ff77a0b718e47d5bccd837fefe8a9aa20f50 RCT-Folly: 846fda9475e61ec7bcbf8a3fe81edfcaeb090669 RCTDeprecation: c4b9e2fd0ab200e3af72b013ed6113187c607077 RCTRequired: e97dd5dafc1db8094e63bc5031e0371f092ae92a @@ -2683,7 +2709,7 @@ SPEC CHECKSUMS: React-logger: 7aef4d74123e5e3d267e5af1fbf5135b5a0d8381 React-Mapbuffer: 91e0eab42a6ae7f3e34091a126d70fc53bd3823e React-microtasksnativemodule: 1ead4fe154df3b1ba34b5a9e35ef3c4bdfa72ccb - react-native-quick-base64: 6568199bb2ac8e72ecdfdc73a230fbc5c1d3aac4 + react-native-quick-base64: e5eea0a73474108a7b80e33ce2c004a414649da2 react-native-safe-area-context: c6e2edd1c1da07bdce287fa9d9e60c5f7b514616 React-NativeModulesApple: eff2eba56030eb0d107b1642b8f853bc36a833ac React-oscompat: b12c633e9c00f1f99467b1e0e0b8038895dae436 diff --git a/example/package.json b/example/package.json index 31912f2..30bc9ad 100644 --- a/example/package.json +++ b/example/package.json @@ -13,8 +13,8 @@ "@react-native/new-app-screen": "0.81.1", "react": "19.1.0", "react-native": "0.81.1", - "react-native-nitro-modules": "^0.29.8", - "react-native-quick-base64": "^2.2.2", + "react-native-nitro-modules": "^0.35.6", + "react-native-quick-base64": "^3.0.0", "react-native-safe-area-context": "^5.5.2" }, "devDependencies": { diff --git a/example/src/Benchmark.tsx b/example/src/Benchmark.tsx index 0bfe45d..764e943 100644 --- a/example/src/Benchmark.tsx +++ b/example/src/Benchmark.tsx @@ -38,7 +38,7 @@ const NitroBase64Demo = () => { try { // First encode to get valid base64 const encoded = encode(inputText, urlSafeMode); - const result = decode(encoded, urlSafeMode); + const result = decode(encoded); logResult(`Nitro Decode (urlSafe: ${urlSafeMode})`, result); logResult('Round Trip Success', `${result === inputText ? 'YES' : 'NO'}`); Alert.alert( @@ -57,14 +57,14 @@ const NitroBase64Demo = () => { const nitroEncoded = encode(inputText, false); logResult('Nitro encode()', nitroEncoded); - const nitroDecoded = decode(nitroEncoded, false); + const nitroDecoded = decode(nitroEncoded); logResult('Nitro decode()', nitroDecoded); // Test URL-safe version const nitroUrlSafe = encode(inputText, true); logResult('Nitro URL-safe encode()', nitroUrlSafe); - const nitroUrlSafeDecoded = decode(nitroUrlSafe, true); + const nitroUrlSafeDecoded = decode(nitroUrlSafe); logResult('Nitro URL-safe decode()', nitroUrlSafeDecoded); // Test if they match @@ -101,7 +101,7 @@ const NitroBase64Demo = () => { testCases.forEach((testCase) => { try { const encoded = encode(testCase.data, urlSafeMode); - const decoded = decode(encoded, urlSafeMode); + const decoded = decode(encoded); const success = decoded === testCase.data; if (success) passCount++; logResult( @@ -128,7 +128,7 @@ const NitroBase64Demo = () => { try { const encoded = encode(largeText, urlSafeMode); - const decoded = decode(encoded, urlSafeMode); + const decoded = decode(encoded); const endTime = Date.now(); const duration = endTime - startTime; diff --git a/example/src/Benchmark2.tsx b/example/src/Benchmark2.tsx index 9d4a7e3..9c330b4 100644 --- a/example/src/Benchmark2.tsx +++ b/example/src/Benchmark2.tsx @@ -1,258 +1,292 @@ -/* global performance, atob, btoa */ -import jsBase64 from 'base64-js'; +import { + fromByteArray as quickFromByteArray, + toByteArray as quickToByteArray, +} from 'react-native-quick-base64'; import { useState } from 'react'; import { Platform, Pressable, StyleSheet, Text, View } from 'react-native'; -import { decode, encode } from 'react-native-nitro-base64'; +import { + decode, + decodeBuffer, + encode, + encodeBuffer, + fromByteArray, + toByteArray, +} from 'react-native-nitro-base64'; import { data as largeData } from './image-large.json'; import { data } from './image.json'; -const sleep = (t: number) => new Promise((resolve) => setTimeout(resolve, t)); -const round = (num: number, decimalPlaces = 0): string => { - return num.toFixed(decimalPlaces); +declare const performance: { now(): number }; +declare function atob(data: string): string; +declare function btoa(data: string): string; + +const sleep = (t: number) => + new Promise((resolve) => setTimeout(resolve, t)); +const round = (num: number, decimalPlaces = 0): string => + num.toFixed(decimalPlaces); +const speedup = (ms: number, base: number) => { + if (!ms || !base || base <= 0) return ''; + const ratio = base / ms; + if (ratio >= 1) return `(${round(ratio, 2)}x faster)`; + return `(${round(1 / ratio, 2)}x slower)`; }; const Benchmarks2 = () => { - const [processingJSBase64, setProcessingJSBase64] = useState(false); - const [jsBase64Result, setJSBase64Result] = useState(0); - const [processingNativeBase64, setProcessingNativeBase64] = - useState(false); - const [nativeBase64Result, setNativeBase64Result] = useState(0); - const [processingObjcBase64, setProcessingObjcBase64] = - useState(false); - const [objcBase64Result, setObjcBase64Result] = useState(0); - const [processingAtobBtoa, setProcessingAtobBtoa] = useState(false); - const [atobBtoaResult, setAtobBtoaResult] = useState(0); - const [processingAtobBtoaLarge, setProcessingAtobBtoaLarge] = - useState(false); - const [atobBtoaLargeResult, setAtobBtoaLargeResult] = useState(0); - const [processingJSBase64Large, setProcessingJSBase64Large] = - useState(false); - const [jsBase64LargeResult, setJSBase64LargeResult] = useState(0); - - const handleNativeBase64Press = async () => { - setProcessingNativeBase64(true); - let dataToProcess = data; - await sleep(1); - const startTime = performance.now(); + // Nitro string (small / large) + const [nitroResult, setNitroResult] = useState(0); + const [nitroLargeResult, setNitroLargeResult] = useState(0); - for (let iter = 0; iter < 30; iter++) { - const decoded = encode(dataToProcess); - dataToProcess = decode(decoded); - if (dataToProcess !== data) { - throw new Error('Data does not match'); - } - } - const finishedTime = performance.now(); - console.log( - 'Nitro C++ done! took', - finishedTime - startTime, - 'milliseconds' - ); - setNativeBase64Result(finishedTime - startTime); - setProcessingNativeBase64(false); - }; + // quick-base64 (small / large) + const [quickResult, setQuickResult] = useState(0); + const [quickLargeResult, setQuickLargeResult] = useState(0); - const handleJSBase64Press = async () => { - setProcessingJSBase64(true); - let dataToProcess = data; - await sleep(1); - const startTime = performance.now(); + // atob/btoa (small / large) + const [atobResult, setAtobResult] = useState(0); + const [atobLargeResult, setAtobLargeResult] = useState(0); + + // decodeBuffer (small / large) + const [decodeBufferResult, setDecodeBufferResult] = useState(0); + const [decodeBufferLargeResult, setDecodeBufferLargeResult] = useState(0); + + // encodeBuffer (small / large) + const [encodeBufferResult, setEncodeBufferResult] = useState(0); + const [encodeBufferLargeResult, setEncodeBufferLargeResult] = useState(0); - for (let iter = 0; iter < 30; iter++) { - const decoded = jsBase64.toByteArray(dataToProcess); - dataToProcess = jsBase64.fromByteArray(decoded); + // fromByteArray / toByteArray (small / large) + const [fromByteArrayResult, setFromByteArrayResult] = useState(0); + const [fromByteArrayLargeResult, setFromByteArrayLargeResult] = useState(0); + const [toByteArrayResult, setToByteArrayResult] = useState(0); + const [toByteArrayLargeResult, setToByteArrayLargeResult] = useState(0); + + const [processing, setProcessing] = useState(false); + + // ── Nitro string ────────────────────────────────────────────────────────── + + const runNitro = async (src: string): Promise => { + let d = src; + await sleep(1); + const t = performance.now(); + for (let i = 0; i < 30; i++) { + const enc = encode(d); + d = decode(enc); } - const finishedTime = performance.now(); - console.log( - 'base64-js done! took', - finishedTime - startTime, - 'milliseconds' - ); - setJSBase64Result(finishedTime - startTime); - setProcessingJSBase64(false); + return performance.now() - t; }; - // atob/btoa benchmark (native browser API) - Small image - const handleAtobBtoaPress = async () => { - setProcessingAtobBtoa(true); - let dataToProcess = data; - await sleep(1); - const startTime = performance.now(); + // ── quick-base64 ────────────────────────────────────────────────────────── - try { - for (let iter = 0; iter < 30; iter++) { - // btoa expects a binary string, atob returns a binary string - const decoded = atob(dataToProcess); - dataToProcess = btoa(decoded); - } - const finishedTime = performance.now(); - console.log( - 'atob/btoa done! took', - finishedTime - startTime, - 'milliseconds' - ); - setAtobBtoaResult(finishedTime - startTime); - } catch (error) { - console.error('atob/btoa error:', error); - setAtobBtoaResult(-1); // Indicate error + const runQuick = async (src: string): Promise => { + let d = src; + await sleep(1); + const t = performance.now(); + for (let i = 0; i < 30; i++) { + const bytes = quickToByteArray(d); + d = quickFromByteArray(bytes); } - setProcessingAtobBtoa(false); + return performance.now() - t; }; - // atob/btoa benchmark (native browser API) - Large image - const handleAtobBtoaLargePress = async () => { - setProcessingAtobBtoaLarge(true); - let dataToProcess = largeData; - await sleep(1); - const startTime = performance.now(); + // ── atob/btoa ───────────────────────────────────────────────────────────── + const runAtob = async (src: string): Promise => { + let d = src; + await sleep(1); + const t = performance.now(); try { - for (let iter = 0; iter < 30; iter++) { - const decoded = atob(dataToProcess); - dataToProcess = btoa(decoded); + for (let i = 0; i < 30; i++) { + const bin = atob(d); + d = btoa(bin); } - const finishedTime = performance.now(); - console.log( - 'atob/btoa Large done! took', - finishedTime - startTime, - 'milliseconds' - ); - setAtobBtoaLargeResult(finishedTime - startTime); - } catch (error) { - console.error('atob/btoa Large error:', error); - setAtobBtoaLargeResult(-1); // Indicate error + } catch { + return -1; } - setProcessingAtobBtoaLarge(false); + return performance.now() - t; }; - // Large image base64-js benchmark - const handleJSBase64LargePress = async () => { - setProcessingJSBase64Large(true); - let dataToProcess = largeData; + // ── decodeBuffer ────────────────────────────────────────────────────────── + + const runDecodeBuffer = async (src: string): Promise => { + const encoded = encode(src); await sleep(1); - const startTime = performance.now(); + const t = performance.now(); + for (let i = 0; i < 30; i++) decodeBuffer(encoded); + return performance.now() - t; + }; - for (let iter = 0; iter < 30; iter++) { - const decoded = jsBase64.toByteArray(dataToProcess); - dataToProcess = jsBase64.fromByteArray(decoded); - } - const finishedTime = performance.now(); - console.log( - 'base64-js Large done! took', - finishedTime - startTime, - 'milliseconds' - ); - setJSBase64LargeResult(finishedTime - startTime); - setProcessingJSBase64Large(false); + // ── encodeBuffer ────────────────────────────────────────────────────────── + + const runEncodeBuffer = async (src: string): Promise => { + const binary = decodeBuffer(src); + await sleep(1); + const t = performance.now(); + for (let i = 0; i < 30; i++) encodeBuffer(binary); + return performance.now() - t; }; - // Large Nitro C++ base64 benchmark - const handleObjcBase64Press = async () => { - setProcessingObjcBase64(true); - let dataToProcess = largeData; + // ── fromByteArray ───────────────────────────────────────────────────────── + + const runFromByteArray = async (src: string): Promise => { + const bytes = new Uint8Array(decodeBuffer(src)); await sleep(1); - const startTime = performance.now(); - for (let iter = 0; iter < 30; iter++) { - const decoded = encode(dataToProcess); - dataToProcess = decode(decoded); - } - const finishedTime = performance.now(); - console.log( - 'Nitro C++ Large done! took', - finishedTime - startTime, - 'milliseconds' - ); - setObjcBase64Result(finishedTime - startTime); - setProcessingObjcBase64(false); + const t = performance.now(); + for (let i = 0; i < 30; i++) fromByteArray(bytes); + return performance.now() - t; }; - const calculateSpeedup = (baseTime: number, compareTime: number) => { - if (!baseTime || !compareTime || compareTime <= 0) return ''; - return `(${round(baseTime / compareTime, 2)}x faster)`; + // ── toByteArray ─────────────────────────────────────────────────────────── + + const runToByteArray = async (src: string): Promise => { + const encoded = encode(src); + await sleep(1); + const t = performance.now(); + for (let i = 0; i < 30; i++) toByteArray(encoded); + return performance.now() - t; }; - return ( - - Small Image (7KB) - 30 iterations + // ── Run all ─────────────────────────────────────────────────────────────── - - Nitro Simdutf C++ - - {nativeBase64Result > 0 ? `${round(nativeBase64Result, 2)}ms` : ''} - - + const handleRunAll = async () => { + setProcessing(true); - - base64-js (JS) - - {jsBase64Result > 0 - ? `${round(jsBase64Result, 2)}ms ${calculateSpeedup(jsBase64Result, nativeBase64Result)}` - : ''} - - + const [n, nL, q, qL, a, aL, db, dbL, eb, ebL, fba, fbaL, tba, tbaL] = + await Promise.all([ + runNitro(data), + runNitro(largeData), + runQuick(data), + runQuick(largeData), + runAtob(data), + runAtob(largeData), + runDecodeBuffer(data), + runDecodeBuffer(largeData), + runEncodeBuffer(data), + runEncodeBuffer(largeData), + runFromByteArray(data), + runFromByteArray(largeData), + runToByteArray(data), + runToByteArray(largeData), + ]); - - atob/btoa (Native) - - {atobBtoaResult > 0 - ? `${round(atobBtoaResult, 2)}ms ${calculateSpeedup(atobBtoaResult, nativeBase64Result)}` - : atobBtoaResult === -1 - ? 'Error/Not supported' - : ''} - - + setNitroResult(n); + setNitroLargeResult(nL); + setQuickResult(q); + setQuickLargeResult(qL); + setAtobResult(a); + setAtobLargeResult(aL); + setDecodeBufferResult(db); + setDecodeBufferLargeResult(dbL); + setEncodeBufferResult(eb); + setEncodeBufferLargeResult(ebL); + setFromByteArrayResult(fba); + setFromByteArrayLargeResult(fbaL); + setToByteArrayResult(tba); + setToByteArrayLargeResult(tbaL); - - Large Image (1.3MB) - 30 iterations + setProcessing(false); + }; + + // ── Render ──────────────────────────────────────────────────────────────── + + const Row = ({ + label, + ms, + base, + }: { + label: string; + ms: number; + base: number; + }) => ( + + {label} + + {ms > 0 + ? `${round(ms, 2)}ms ${speedup(ms, base)}` + : ms === -1 + ? 'Not supported' + : ''} + + ); - - Nitro Simdutf C++ - - {objcBase64Result > 0 ? `${round(objcBase64Result, 2)}ms` : ''} - - + return ( + + Small Image (7KB) — 30 iterations - base64-js (JS) + Nitro simdutf C++ (string) - {jsBase64LargeResult > 0 - ? `${round(jsBase64LargeResult, 2)}ms ${calculateSpeedup(jsBase64LargeResult, objcBase64Result)}` - : ''} + {nitroResult > 0 ? `${round(nitroResult, 2)}ms` : ''} + + + + + + + + + Large Image (1.3MB) — 30 iterations + - atob/btoa (Native) + Nitro simdutf C++ (string) - {atobBtoaLargeResult > 0 - ? `${round(atobBtoaLargeResult, 2)}ms ${calculateSpeedup(atobBtoaLargeResult, objcBase64Result)}` - : atobBtoaLargeResult === -1 - ? 'Error/Not supported' - : ''} + {nitroLargeResult > 0 ? `${round(nitroLargeResult, 2)}ms` : ''} + + + + + + - { - handleNativeBase64Press(); - handleJSBase64Press(); - handleAtobBtoaPress(); - handleObjcBase64Press(); - handleJSBase64LargePress(); - handleAtobBtoaLargePress(); - }} - style={styles.button} - > + - {processingNativeBase64 || - processingJSBase64 || - processingObjcBase64 || - processingAtobBtoa || - processingAtobBtoaLarge || - processingJSBase64Large - ? 'Processing...' - : 'Run Benchmarks'} + {processing ? 'Processing...' : 'Run Benchmarks'} @@ -262,11 +296,6 @@ const Benchmarks2 = () => { export default Benchmarks2; const styles = StyleSheet.create({ - box: { - width: 60, - height: 60, - marginVertical: 20, - }, lib: { flexDirection: 'row', justifyContent: 'space-between', @@ -292,11 +321,6 @@ const styles = StyleSheet.create({ marginTop: 20, borderRadius: 8, }, - speedup: { - marginVertical: 5, - fontSize: 18, - textAlign: 'center', - }, sectionTitle: { fontSize: 16, fontWeight: 'bold', diff --git a/nitro.json b/nitro.json index f896413..26f1b00 100644 --- a/nitro.json +++ b/nitro.json @@ -9,7 +9,10 @@ }, "autolinking": { "NitroBase64": { - "cpp": "NitroBase64" + "all": { + "language": "c++", + "implementationClassName": "NitroBase64" + } } }, "ignorePaths": ["node_modules"] diff --git a/nitrogen/generated/android/NitroBase64+autolinking.cmake b/nitrogen/generated/android/NitroBase64+autolinking.cmake index 0602aee..2c51da3 100644 --- a/nitrogen/generated/android/NitroBase64+autolinking.cmake +++ b/nitrogen/generated/android/NitroBase64+autolinking.cmake @@ -2,7 +2,7 @@ # NitroBase64+autolinking.cmake # This file was generated by nitrogen. DO NOT MODIFY THIS FILE. # https://github.com/mrousavy/nitro -# Copyright © 2025 Marc Rousavy @ Margelo +# Copyright © Marc Rousavy @ Margelo # # This is a CMake file that adds all files generated by Nitrogen diff --git a/nitrogen/generated/android/NitroBase64+autolinking.gradle b/nitrogen/generated/android/NitroBase64+autolinking.gradle index 1cef298..4e22201 100644 --- a/nitrogen/generated/android/NitroBase64+autolinking.gradle +++ b/nitrogen/generated/android/NitroBase64+autolinking.gradle @@ -2,7 +2,7 @@ /// NitroBase64+autolinking.gradle /// This file was generated by nitrogen. DO NOT MODIFY THIS FILE. /// https://github.com/mrousavy/nitro -/// Copyright © 2025 Marc Rousavy @ Margelo +/// Copyright © Marc Rousavy @ Margelo /// /// This is a Gradle file that adds all files generated by Nitrogen diff --git a/nitrogen/generated/android/NitroBase64OnLoad.cpp b/nitrogen/generated/android/NitroBase64OnLoad.cpp index 1a14934..9173907 100644 --- a/nitrogen/generated/android/NitroBase64OnLoad.cpp +++ b/nitrogen/generated/android/NitroBase64OnLoad.cpp @@ -2,7 +2,7 @@ /// NitroBase64OnLoad.cpp /// This file was generated by nitrogen. DO NOT MODIFY THIS FILE. /// https://github.com/mrousavy/nitro -/// Copyright © 2025 Marc Rousavy @ Margelo +/// Copyright © Marc Rousavy @ Margelo /// #ifndef BUILDING_NITROBASE64_WITH_GENERATED_CMAKE_PROJECT @@ -20,25 +20,30 @@ namespace margelo::nitro::nitrobase64 { int initialize(JavaVM* vm) { + return facebook::jni::initialize(vm, []() { + ::margelo::nitro::nitrobase64::registerAllNatives(); + }); +} + + + +void registerAllNatives() { using namespace margelo::nitro; using namespace margelo::nitro::nitrobase64; - using namespace facebook; - - return facebook::jni::initialize(vm, [] { - // Register native JNI methods - - - // Register Nitro Hybrid Objects - HybridObjectRegistry::registerHybridObjectConstructor( - "NitroBase64", - []() -> std::shared_ptr { - static_assert(std::is_default_constructible_v, - "The HybridObject \"NitroBase64\" is not default-constructible! " - "Create a public constructor that takes zero arguments to be able to autolink this HybridObject."); - return std::make_shared(); - } - ); - }); + + // Register native JNI methods + + + // Register Nitro Hybrid Objects + HybridObjectRegistry::registerHybridObjectConstructor( + "NitroBase64", + []() -> std::shared_ptr { + static_assert(std::is_default_constructible_v, + "The HybridObject \"NitroBase64\" is not default-constructible! " + "Create a public constructor that takes zero arguments to be able to autolink this HybridObject."); + return std::make_shared(); + } + ); } } // namespace margelo::nitro::nitrobase64 diff --git a/nitrogen/generated/android/NitroBase64OnLoad.hpp b/nitrogen/generated/android/NitroBase64OnLoad.hpp index c4ac6d1..9208fde 100644 --- a/nitrogen/generated/android/NitroBase64OnLoad.hpp +++ b/nitrogen/generated/android/NitroBase64OnLoad.hpp @@ -2,24 +2,33 @@ /// NitroBase64OnLoad.hpp /// This file was generated by nitrogen. DO NOT MODIFY THIS FILE. /// https://github.com/mrousavy/nitro -/// Copyright © 2025 Marc Rousavy @ Margelo +/// Copyright © Marc Rousavy @ Margelo /// #include +#include #include namespace margelo::nitro::nitrobase64 { + [[deprecated("Use registerNatives() instead.")]] + int initialize(JavaVM* vm); + /** - * Initializes the native (C++) part of NitroBase64, and autolinks all Hybrid Objects. - * Call this in your `JNI_OnLoad` function (probably inside `cpp-adapter.cpp`). + * Register the native (C++) part of NitroBase64, and autolinks all Hybrid Objects. + * Call this in your `JNI_OnLoad` function (probably inside `cpp-adapter.cpp`), + * inside a `facebook::jni::initialize(vm, ...)` call. * Example: * ```cpp (cpp-adapter.cpp) * JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void*) { - * return margelo::nitro::nitrobase64::initialize(vm); + * return facebook::jni::initialize(vm, []() { + * // register all NitroBase64 HybridObjects + * margelo::nitro::nitrobase64::registerNatives(); + * // any other custom registrations go here. + * }); * } * ``` */ - int initialize(JavaVM* vm); + void registerAllNatives(); } // namespace margelo::nitro::nitrobase64 diff --git a/nitrogen/generated/android/kotlin/com/margelo/nitro/nitrobase64/NitroBase64OnLoad.kt b/nitrogen/generated/android/kotlin/com/margelo/nitro/nitrobase64/NitroBase64OnLoad.kt index dc4eb11..7418548 100644 --- a/nitrogen/generated/android/kotlin/com/margelo/nitro/nitrobase64/NitroBase64OnLoad.kt +++ b/nitrogen/generated/android/kotlin/com/margelo/nitro/nitrobase64/NitroBase64OnLoad.kt @@ -2,7 +2,7 @@ /// NitroBase64OnLoad.kt /// This file was generated by nitrogen. DO NOT MODIFY THIS FILE. /// https://github.com/mrousavy/nitro -/// Copyright © 2025 Marc Rousavy @ Margelo +/// Copyright © Marc Rousavy @ Margelo /// package com.margelo.nitro.nitrobase64 diff --git a/nitrogen/generated/ios/NitroBase64+autolinking.rb b/nitrogen/generated/ios/NitroBase64+autolinking.rb index 02f41ce..656ede5 100644 --- a/nitrogen/generated/ios/NitroBase64+autolinking.rb +++ b/nitrogen/generated/ios/NitroBase64+autolinking.rb @@ -2,7 +2,7 @@ # NitroBase64+autolinking.rb # This file was generated by nitrogen. DO NOT MODIFY THIS FILE. # https://github.com/mrousavy/nitro -# Copyright © 2025 Marc Rousavy @ Margelo +# Copyright © Marc Rousavy @ Margelo # # This is a Ruby script that adds all files generated by Nitrogen @@ -52,9 +52,11 @@ def add_nitrogen_files(spec) spec.pod_target_xcconfig = current_pod_target_xcconfig.merge({ # Use C++ 20 "CLANG_CXX_LANGUAGE_STANDARD" => "c++20", - # Enables C++ <-> Swift interop (by default it's only C) + # Enables C++ <-> Swift interop (by default it's only ObjC) "SWIFT_OBJC_INTEROP_MODE" => "objcxx", # Enables stricter modular headers "DEFINES_MODULE" => "YES", + # Disable auto-generated ObjC header for Swift (Static linkage on Xcode 26.4 breaks here) + "SWIFT_INSTALL_OBJC_HEADER" => "NO", }) end diff --git a/nitrogen/generated/ios/NitroBase64-Swift-Cxx-Bridge.cpp b/nitrogen/generated/ios/NitroBase64-Swift-Cxx-Bridge.cpp index 8d77468..ea1cf66 100644 --- a/nitrogen/generated/ios/NitroBase64-Swift-Cxx-Bridge.cpp +++ b/nitrogen/generated/ios/NitroBase64-Swift-Cxx-Bridge.cpp @@ -2,7 +2,7 @@ /// NitroBase64-Swift-Cxx-Bridge.cpp /// This file was generated by nitrogen. DO NOT MODIFY THIS FILE. /// https://github.com/mrousavy/nitro -/// Copyright © 2025 Marc Rousavy @ Margelo +/// Copyright © Marc Rousavy @ Margelo /// #include "NitroBase64-Swift-Cxx-Bridge.hpp" diff --git a/nitrogen/generated/ios/NitroBase64-Swift-Cxx-Bridge.hpp b/nitrogen/generated/ios/NitroBase64-Swift-Cxx-Bridge.hpp index 0868b04..c06a4d2 100644 --- a/nitrogen/generated/ios/NitroBase64-Swift-Cxx-Bridge.hpp +++ b/nitrogen/generated/ios/NitroBase64-Swift-Cxx-Bridge.hpp @@ -2,7 +2,7 @@ /// NitroBase64-Swift-Cxx-Bridge.hpp /// This file was generated by nitrogen. DO NOT MODIFY THIS FILE. /// https://github.com/mrousavy/nitro -/// Copyright © 2025 Marc Rousavy @ Margelo +/// Copyright © Marc Rousavy @ Margelo /// #pragma once diff --git a/nitrogen/generated/ios/NitroBase64-Swift-Cxx-Umbrella.hpp b/nitrogen/generated/ios/NitroBase64-Swift-Cxx-Umbrella.hpp index b38ad5e..e986634 100644 --- a/nitrogen/generated/ios/NitroBase64-Swift-Cxx-Umbrella.hpp +++ b/nitrogen/generated/ios/NitroBase64-Swift-Cxx-Umbrella.hpp @@ -2,7 +2,7 @@ /// NitroBase64-Swift-Cxx-Umbrella.hpp /// This file was generated by nitrogen. DO NOT MODIFY THIS FILE. /// https://github.com/mrousavy/nitro -/// Copyright © 2025 Marc Rousavy @ Margelo +/// Copyright © Marc Rousavy @ Margelo /// #pragma once diff --git a/nitrogen/generated/ios/NitroBase64Autolinking.mm b/nitrogen/generated/ios/NitroBase64Autolinking.mm index b703394..68c6450 100644 --- a/nitrogen/generated/ios/NitroBase64Autolinking.mm +++ b/nitrogen/generated/ios/NitroBase64Autolinking.mm @@ -2,7 +2,7 @@ /// NitroBase64Autolinking.mm /// This file was generated by nitrogen. DO NOT MODIFY THIS FILE. /// https://github.com/mrousavy/nitro -/// Copyright © 2025 Marc Rousavy @ Margelo +/// Copyright © Marc Rousavy @ Margelo /// #import diff --git a/nitrogen/generated/ios/NitroBase64Autolinking.swift b/nitrogen/generated/ios/NitroBase64Autolinking.swift index 3dfbabd..5f773f4 100644 --- a/nitrogen/generated/ios/NitroBase64Autolinking.swift +++ b/nitrogen/generated/ios/NitroBase64Autolinking.swift @@ -2,9 +2,13 @@ /// NitroBase64Autolinking.swift /// This file was generated by nitrogen. DO NOT MODIFY THIS FILE. /// https://github.com/mrousavy/nitro -/// Copyright © 2025 Marc Rousavy @ Margelo +/// Copyright © Marc Rousavy @ Margelo /// +import NitroModules + +// TODO: Use empty enums once Swift supports exporting them as namespaces +// See: https://github.com/swiftlang/swift/pull/83616 public final class NitroBase64Autolinking { public typealias bridge = margelo.nitro.nitrobase64.bridge.swift diff --git a/nitrogen/generated/shared/c++/HybridNitroBase64Spec.cpp b/nitrogen/generated/shared/c++/HybridNitroBase64Spec.cpp index f8c1765..6dbcbb9 100644 --- a/nitrogen/generated/shared/c++/HybridNitroBase64Spec.cpp +++ b/nitrogen/generated/shared/c++/HybridNitroBase64Spec.cpp @@ -2,7 +2,7 @@ /// HybridNitroBase64Spec.cpp /// This file was generated by nitrogen. DO NOT MODIFY THIS FILE. /// https://github.com/mrousavy/nitro -/// Copyright © 2025 Marc Rousavy @ Margelo +/// Copyright © Marc Rousavy @ Margelo /// #include "HybridNitroBase64Spec.hpp" @@ -17,6 +17,8 @@ namespace margelo::nitro::nitrobase64 { prototype.registerHybridMethod("install", &HybridNitroBase64Spec::install); prototype.registerHybridMethod("encode", &HybridNitroBase64Spec::encode); prototype.registerHybridMethod("decode", &HybridNitroBase64Spec::decode); + prototype.registerHybridMethod("decodeBuffer", &HybridNitroBase64Spec::decodeBuffer); + prototype.registerHybridMethod("encodeBuffer", &HybridNitroBase64Spec::encodeBuffer); }); } diff --git a/nitrogen/generated/shared/c++/HybridNitroBase64Spec.hpp b/nitrogen/generated/shared/c++/HybridNitroBase64Spec.hpp index a707954..a00b5ab 100644 --- a/nitrogen/generated/shared/c++/HybridNitroBase64Spec.hpp +++ b/nitrogen/generated/shared/c++/HybridNitroBase64Spec.hpp @@ -2,7 +2,7 @@ /// HybridNitroBase64Spec.hpp /// This file was generated by nitrogen. DO NOT MODIFY THIS FILE. /// https://github.com/mrousavy/nitro -/// Copyright © 2025 Marc Rousavy @ Margelo +/// Copyright © Marc Rousavy @ Margelo /// #pragma once @@ -16,6 +16,7 @@ #include +#include namespace margelo::nitro::nitrobase64 { @@ -50,7 +51,9 @@ namespace margelo::nitro::nitrobase64 { // Methods virtual void install() = 0; virtual std::string encode(const std::string& input, bool urlSafe) = 0; - virtual std::string decode(const std::string& base64, bool urlSafe) = 0; + virtual std::string decode(const std::string& base64) = 0; + virtual std::shared_ptr decodeBuffer(const std::string& base64) = 0; + virtual std::string encodeBuffer(const std::shared_ptr& input, bool urlSafe) = 0; protected: // Hybrid Setup diff --git a/package.json b/package.json index 4a568aa..4a4f7cb 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "react-native-nitro-base64", - "version": "0.8.2", - "description": "NitroModule Base64", + "version": "1.1.2", + "description": "The fastest Base64 library for React Native, powered by simdutf SIMD C++ and Nitro Modules", "main": "./lib/module/index.js", "types": "./lib/typescript/src/index.d.ts", "exports": { @@ -47,7 +47,24 @@ "keywords": [ "react-native", "ios", - "android" + "android", + "base64", + "base64url", + "encoding", + "decoding", + "simdutf", + "simd", + "performance", + "fast", + "nitro", + "nitro-modules", + "jsi", + "arraybuffer", + "uint8array", + "binary", + "image", + "crypto", + "jwt" ], "repository": { "type": "git", @@ -79,12 +96,12 @@ "eslint-config-prettier": "^10.1.8", "eslint-plugin-prettier": "^5.5.4", "jest": "^29.7.0", - "nitrogen": "^0.29.8", + "nitrogen": "^0.35.6", "prettier": "^3.6.2", "react": "19.1.0", "react-native": "0.81.1", "react-native-builder-bob": "^0.40.13", - "react-native-nitro-modules": "^0.29.8", + "react-native-nitro-modules": "^0.35.6", "release-it": "^19.0.4", "turbo": "^2.5.6", "typescript": "^5.9.2" diff --git a/src/NitroBase64.nitro.ts b/src/NitroBase64.nitro.ts index 48f47d4..a509caf 100644 --- a/src/NitroBase64.nitro.ts +++ b/src/NitroBase64.nitro.ts @@ -4,7 +4,9 @@ export interface NitroBase64 extends HybridObject<{ ios: 'c++'; android: 'c++' }> { install(): void; encode(input: string, urlSafe: boolean): string; - decode(base64: string, urlSafe: boolean): string; + decode(base64: string): string; + decodeBuffer(base64: string): ArrayBuffer; + encodeBuffer(input: ArrayBuffer, urlSafe: boolean): string; } // export interface NitroBase64Module extends HybridObject<{ android: 'kotlin' }> { diff --git a/src/index.tsx b/src/index.tsx index f827427..ec45442 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -6,14 +6,33 @@ const NitroBase64HybridObject = // const NitroBase64ModuleHybridObject = // NitroModules.createHybridObject('NitroBase64Module'); -export function decode(base64: string, urlSafe = false): string { - return NitroBase64HybridObject.decode(base64, urlSafe); +export function decode(base64: string): string { + return NitroBase64HybridObject.decode(base64); } export function encode(input: string, urlSafe = false): string { return NitroBase64HybridObject.encode(input, urlSafe); } +export function decodeBuffer(base64: string): ArrayBuffer { + return NitroBase64HybridObject.decodeBuffer(base64); +} + +export function encodeBuffer(input: ArrayBuffer, urlSafe = false): string { + return NitroBase64HybridObject.encodeBuffer(input, urlSafe); +} + +export function fromByteArray(input: Uint8Array, urlSafe = false): string { + return NitroBase64HybridObject.encodeBuffer( + input.buffer as ArrayBuffer, + urlSafe + ); +} + +export function toByteArray(base64: string): Uint8Array { + return new Uint8Array(NitroBase64HybridObject.decodeBuffer(base64)); +} + export function install(): void { NitroBase64HybridObject.install(); } diff --git a/yarn.lock b/yarn.lock index 18046c3..ec1e0c5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3793,7 +3793,7 @@ __metadata: languageName: node linkType: hard -"ansi-styles@npm:^6.1.0": +"ansi-styles@npm:^6.1.0, ansi-styles@npm:^6.2.1": version: 6.2.3 resolution: "ansi-styles@npm:6.2.3" checksum: f1b0829cf048cce870a305819f65ce2adcebc097b6d6479e12e955fd6225df9b9eb8b497083b764df796d94383ff20016cc4dbbae5b40f36138fb65a9d33c2e2 @@ -4584,6 +4584,17 @@ __metadata: languageName: node linkType: hard +"cliui@npm:^9.0.1": + version: 9.0.1 + resolution: "cliui@npm:9.0.1" + dependencies: + string-width: ^7.2.0 + strip-ansi: ^7.1.0 + wrap-ansi: ^9.0.0 + checksum: 143879ae462bf76822f341bf40979f0225fdba8dde6dfe429018b13396fd0532752cc2a809ac48cecc0ea189406184ad7568c0af44eea73d2ac3b432c4c6431f + languageName: node + linkType: hard + "clone@npm:^1.0.2": version: 1.0.4 resolution: "clone@npm:1.0.4" @@ -5390,6 +5401,13 @@ __metadata: languageName: node linkType: hard +"emoji-regex@npm:^10.3.0": + version: 10.6.0 + resolution: "emoji-regex@npm:10.6.0" + checksum: 8785f6a7ec4559c931bd6640f748fe23791f5af4c743b131d458c5551b4aa7da2a9cd882518723cb3859e8b0b59b0cc08f2ce0f8e65c61a026eed71c2dc407d5 + languageName: node + linkType: hard + "emoji-regex@npm:^8.0.0": version: 8.0.0 resolution: "emoji-regex@npm:8.0.0" @@ -6401,6 +6419,13 @@ __metadata: languageName: node linkType: hard +"get-east-asian-width@npm:^1.0.0": + version: 1.6.0 + resolution: "get-east-asian-width@npm:1.6.0" + checksum: 88faf98aaade2b24ad260be369b026d979ff4d0e6201bfd991654da17f2aa720bbddb812ce5612110eef6b469d8370c863f69a2f9818d61733f1eb2a83d779b8 + languageName: node + linkType: hard + "get-east-asian-width@npm:^1.3.0": version: 1.4.0 resolution: "get-east-asian-width@npm:1.4.0" @@ -9140,18 +9165,18 @@ __metadata: languageName: node linkType: hard -"nitrogen@npm:^0.29.8": - version: 0.29.8 - resolution: "nitrogen@npm:0.29.8" +"nitrogen@npm:^0.35.6": + version: 0.35.6 + resolution: "nitrogen@npm:0.35.6" dependencies: chalk: ^5.3.0 - react-native-nitro-modules: ^0.29.8 + react-native-nitro-modules: ^0.35.6 ts-morph: ^27.0.0 - yargs: ^17.7.2 + yargs: ^18.0.0 zod: ^4.0.5 bin: nitrogen: lib/index.js - checksum: ec56919ee570e1a9475ec95a90ed3779110bf9139811102d63d3b40a9b2747ab54962c081ff4f0cc5f0df83f06719f020d55489ce8d4f2a7a08ea1d0299a1977 + checksum: 105ff9cf455c3c12def88181e37d2b681b5b50dc891a725db9e36e16695809f9c80056b60bc94d19dfaf7bf71c77820660f7ac9f29f9e5b7ed3e5c2e0a5e8d8a languageName: node linkType: hard @@ -10120,8 +10145,8 @@ __metadata: react-native: 0.81.1 react-native-builder-bob: ^0.40.13 react-native-monorepo-config: ^0.1.9 - react-native-nitro-modules: ^0.29.8 - react-native-quick-base64: ^2.2.2 + react-native-nitro-modules: ^0.35.6 + react-native-quick-base64: ^3.0.0 react-native-safe-area-context: ^5.5.2 languageName: unknown linkType: soft @@ -10146,12 +10171,12 @@ __metadata: eslint-config-prettier: ^10.1.8 eslint-plugin-prettier: ^5.5.4 jest: ^29.7.0 - nitrogen: ^0.29.8 + nitrogen: ^0.35.6 prettier: ^3.6.2 react: 19.1.0 react-native: 0.81.1 react-native-builder-bob: ^0.40.13 - react-native-nitro-modules: ^0.29.8 + react-native-nitro-modules: ^0.35.6 release-it: ^19.0.4 turbo: ^2.5.6 typescript: ^5.9.2 @@ -10162,23 +10187,23 @@ __metadata: languageName: unknown linkType: soft -"react-native-nitro-modules@npm:^0.29.8": - version: 0.29.8 - resolution: "react-native-nitro-modules@npm:0.29.8" +"react-native-nitro-modules@npm:^0.35.6": + version: 0.35.6 + resolution: "react-native-nitro-modules@npm:0.35.6" peerDependencies: react: "*" react-native: "*" - checksum: 54e9ddb7d9c28f39000e9ec06c1ae59fde9dc53454bade6697ae842c4996becd388cdfa1bb0cb504a6ddc25873e1409533a141f94d2f36153bdbf4bf51cf97b1 + checksum: b057dc606c4717bff2447cff3efdeebf5cd7342fa11b3f51cfd420ccce1224d0b4be5999ea63c9856721520f1cbb50fba9372109f49a4c8b14202d7ab4d3b37e languageName: node linkType: hard -"react-native-quick-base64@npm:^2.2.2": - version: 2.2.2 - resolution: "react-native-quick-base64@npm:2.2.2" +"react-native-quick-base64@npm:^3.0.0": + version: 3.0.0 + resolution: "react-native-quick-base64@npm:3.0.0" peerDependencies: react: "*" react-native: "*" - checksum: 51cd8df0b7f78d4c10996f157927cc075b8a9e85f7d0e48790a4247a033ca335e440c96572969fb6f6b49135d51dbf87c6d659fba509b51b9237957d41a6c580 + checksum: b9d447a66178e8f72ae3a6d852056bb928256bb9c8703c0b3bcfb9338df12d89ff82617535114f3d04643c908aaf0b48d5c46b0f0fb281bab34855d95c23f448 languageName: node linkType: hard @@ -11098,6 +11123,17 @@ __metadata: languageName: node linkType: hard +"string-width@npm:^7.0.0, string-width@npm:^7.2.0": + version: 7.2.0 + resolution: "string-width@npm:7.2.0" + dependencies: + emoji-regex: ^10.3.0 + get-east-asian-width: ^1.0.0 + strip-ansi: ^7.1.0 + checksum: 42f9e82f61314904a81393f6ef75b832c39f39761797250de68c041d8ba4df2ef80db49ab6cd3a292923a6f0f409b8c9980d120f7d32c820b4a8a84a2598a295 + languageName: node + linkType: hard + "string-width@npm:^8.1.0": version: 8.1.0 resolution: "string-width@npm:8.1.0" @@ -12020,6 +12056,17 @@ __metadata: languageName: node linkType: hard +"wrap-ansi@npm:^9.0.0": + version: 9.0.2 + resolution: "wrap-ansi@npm:9.0.2" + dependencies: + ansi-styles: ^6.2.1 + string-width: ^7.0.0 + strip-ansi: ^7.1.0 + checksum: 9827bf8bbb341d2d15f26d8507d98ca2695279359073422fe089d374b30e233d24ab95beca55cf9ab8dcb89face00e919be4158af50d4b6d8eab5ef4ee399e0c + languageName: node + linkType: hard + "wrappy@npm:1": version: 1.0.2 resolution: "wrappy@npm:1.0.2" @@ -12131,6 +12178,13 @@ __metadata: languageName: node linkType: hard +"yargs-parser@npm:^22.0.0": + version: 22.0.0 + resolution: "yargs-parser@npm:22.0.0" + checksum: 55df0d94f3f9f933f1349f244ddf72a6978a9d5a972b69332965cdfd5ec849ff26386965512f4179065b0573cc6e8df33ca44334958a892c47fedae08a967c99 + languageName: node + linkType: hard + "yargs@npm:^15.1.0": version: 15.4.1 resolution: "yargs@npm:15.4.1" @@ -12150,7 +12204,7 @@ __metadata: languageName: node linkType: hard -"yargs@npm:^17.0.0, yargs@npm:^17.3.1, yargs@npm:^17.5.1, yargs@npm:^17.6.2, yargs@npm:^17.7.2": +"yargs@npm:^17.0.0, yargs@npm:^17.3.1, yargs@npm:^17.5.1, yargs@npm:^17.6.2": version: 17.7.2 resolution: "yargs@npm:17.7.2" dependencies: @@ -12165,6 +12219,20 @@ __metadata: languageName: node linkType: hard +"yargs@npm:^18.0.0": + version: 18.0.0 + resolution: "yargs@npm:18.0.0" + dependencies: + cliui: ^9.0.1 + escalade: ^3.1.1 + get-caller-file: ^2.0.5 + string-width: ^7.2.0 + y18n: ^5.0.5 + yargs-parser: ^22.0.0 + checksum: a7cf1b97cb4e81c059f78fd32a4160505d421ecdce5409f5e3840fdcc4c982885fc645b44af961eab94d673cb46f81207d831aa87862246907ffacf45884976a + languageName: node + linkType: hard + "yocto-queue@npm:^0.1.0": version: 0.1.0 resolution: "yocto-queue@npm:0.1.0"