diff --git a/.gitignore b/.gitignore index 2960135..f67bb6c 100644 --- a/.gitignore +++ b/.gitignore @@ -207,3 +207,6 @@ bin/ ### Mac OS ### .DS_Store **/.bucketbase.fscache*/ + +# Local-only signing secrets for publishing to Maven Central (never commit) +.publish-secrets/ diff --git a/java/PUBLISHING.md b/java/PUBLISHING.md new file mode 100644 index 0000000..dbe0f8d --- /dev/null +++ b/java/PUBLISHING.md @@ -0,0 +1,233 @@ +# Publishing the Java library to Maven Central + +The Gradle project publishes: + +```text +com.esamtrade:bucketbase: +``` + +The default version is `0.1.0`. Override it for a release with +`-PreleaseVersion=`. Maven Central releases are immutable and the version must not end +in `-SNAPSHOT`. + +## Build and publication design + +- Gradle Wrapper: `9.6.1` +- Java toolchain and bytecode release: `25` +- Central Portal integration: `com.vanniktech.maven.publish` `0.37.0` +- Automatic release: enabled +- Deployment validation: waits until `PUBLISHED` +- Signing: in-memory ASCII-armored OpenPGP key + +Every Central upload task depends on the complete `test` task. Publishing cannot bypass the +test suite. + +The generated consumer POM contains: + +- `commons-io` as a runtime dependency; +- AWS SDK v2 `s3` as an optional dependency; +- AWS SDK v1 `aws-java-sdk-s3` as an optional dependency. + +JUnit and Trino are test-only and do not appear in the production JAR or consumer POM. + +## Verified local state + +The following were verified locally on 2026-07-24: + +- the Gradle 9.6.1 distribution SHA-256 matched Gradle's published checksum; +- the Gradle Wrapper JAR SHA-256 matched Gradle's published checksum; +- all 95 Java tests passed, including live MinIO tests through AWS SDK v1 and v2; +- the main, sources and Javadoc JARs were generated; +- the Maven POM contains the required license, developer and SCM metadata; +- all JAR, POM and Gradle-module signatures were generated and verified with GPG; +- the complete `publishAndReleaseToMavenCentral` task graph was dry-run successfully. + +No component was uploaded while performing these checks. + +## Prerequisites + +1. JDK 25 must be installed. +2. The `com.esamtrade` namespace must be verified in the + [Central Portal](https://central.sonatype.com/). +3. Generate a Central Portal user token. Its generated username and password—not the account + password—are the publishing credentials. +4. The signing public key must be available from a public keyserver. +5. The repository-root `.publish-secrets/` directory must contain: + - `private-key.asc` + - `gpg_passphrase.txt` + +The existing `.publish-secrets/` directory is ignored by Git. Never commit or print its contents. + +## Normal build + +From the `java/` directory: + +```bash +./gradlew --no-daemon --non-interactive clean test +``` + +To skip the live MinIO tests during ordinary development: + +```bash +BUCKETBASE_SKIP_FUNCTIONAL_TESTS=1 \ + ./gradlew --no-daemon --non-interactive clean test +``` + +Do not skip functional tests for a release. + +## Validate release artifacts without uploading + +Build the three release JARs and generate the POM: + +```bash +./gradlew --no-daemon --non-interactive \ + clean test assemble generatePomFileForMavenPublication +``` + +Inspect: + +```text +build/libs/bucketbase-.jar +build/libs/bucketbase--sources.jar +build/libs/bucketbase--javadoc.jar +build/publications/maven/pom-default.xml +``` + +Load the local signing files into Gradle environment properties: + +```bash +export ORG_GRADLE_PROJECT_signingInMemoryKey="$(<../.publish-secrets/private-key.asc)" +export ORG_GRADLE_PROJECT_signingInMemoryKeyPassword="$(<../.publish-secrets/gpg_passphrase.txt)" +``` + +Generate signatures locally: + +```bash +./gradlew --no-daemon --non-interactive signMavenPublication +``` + +The signatures are written next to the three JARs and under +`build/publications/maven/`. Verify them with `gpg --verify` before publishing. + +Dry-run the complete upload and automatic-release graph: + +```bash +./gradlew --no-daemon --non-interactive \ + publishAndReleaseToMavenCentral --dry-run +``` + +Unset the signing properties when finished: + +```bash +unset ORG_GRADLE_PROJECT_signingInMemoryKey +unset ORG_GRADLE_PROJECT_signingInMemoryKeyPassword +``` + +## Publish + +Set the Central token without placing it in `gradle.properties` or shell command arguments: + +```bash +read -r -p "Central token username: " CENTRAL_TOKEN_USERNAME +read -r -s -p "Central token password: " CENTRAL_TOKEN_PASSWORD +echo + +export ORG_GRADLE_PROJECT_mavenCentralUsername="$CENTRAL_TOKEN_USERNAME" +export ORG_GRADLE_PROJECT_mavenCentralPassword="$CENTRAL_TOKEN_PASSWORD" +unset CENTRAL_TOKEN_USERNAME CENTRAL_TOKEN_PASSWORD + +export ORG_GRADLE_PROJECT_signingInMemoryKey="$(<../.publish-secrets/private-key.asc)" +export ORG_GRADLE_PROJECT_signingInMemoryKeyPassword="$(<../.publish-secrets/gpg_passphrase.txt)" +``` + +Confirm that the selected version is unused, then publish: + +```bash +./gradlew --no-daemon --non-interactive \ + -PreleaseVersion=0.1.0 \ + publishAndReleaseToMavenCentral +``` + +The task: + +1. runs the full Java test suite; +2. builds the main, sources and Javadoc JARs; +3. generates Maven and Gradle metadata; +4. signs the publication; +5. uploads it to the Central Portal; +6. requests release automatically; +7. waits until the deployment reaches `PUBLISHED`. + +Always remove the credentials from the environment afterwards: + +```bash +unset ORG_GRADLE_PROJECT_mavenCentralUsername +unset ORG_GRADLE_PROJECT_mavenCentralPassword +unset ORG_GRADLE_PROJECT_signingInMemoryKey +unset ORG_GRADLE_PROJECT_signingInMemoryKeyPassword +``` + +Confirm the release at: + +```text +https://central.sonatype.com/artifact/com.esamtrade/bucketbase +``` + +Central search and `repo1.maven.org` can lag behind the `PUBLISHED` state. + +## Consumer dependencies + +Core and in-memory use: + +```groovy +implementation "com.esamtrade:bucketbase:" +``` + +AWS SDK v2 backend: + +```groovy +implementation "com.esamtrade:bucketbase:" +implementation "software.amazon.awssdk:s3:2.30.36" +``` + +AWS SDK v1 backend: + +```groovy +implementation "com.esamtrade:bucketbase:" +implementation "com.amazonaws:aws-java-sdk-s3:1.12.782" +``` + +The AWS SDK dependencies are intentionally optional; a core-only consumer does not download +either SDK. + +## CI secret names + +For a future GitHub Actions publishing workflow, configure: + +- `CENTRAL_TOKEN_USERNAME` +- `CENTRAL_TOKEN_PASSWORD` +- `GPG_PRIVATE_KEY` +- `GPG_PASSPHRASE` +- optional private MinIO credentials + +Use the distinct Java tag namespace `java-v*`; the Python package already uses `v*`. + +Map the secrets to: + +```text +ORG_GRADLE_PROJECT_mavenCentralUsername +ORG_GRADLE_PROJECT_mavenCentralPassword +ORG_GRADLE_PROJECT_signingInMemoryKey +ORG_GRADLE_PROJECT_signingInMemoryKeyPassword +``` + +## Troubleshooting + +| Symptom | Cause | +|---|---| +| `Missing Maven Central credentials` | The generated Central token username/password were not exposed through the two `mavenCentral*` Gradle properties. | +| `Cannot perform signing task ... no configured signatory` | The in-memory private key or its password is missing. | +| Central reports missing sources, Javadocs or signatures | Publish with `publishAndReleaseToMavenCentral`; do not manually upload only `build/libs/*.jar`. | +| Central rejects the POM | Inspect `build/publications/maven/pom-default.xml` and confirm all required metadata is present. | +| `409 Conflict` or an already-existing version | Central releases cannot be overwritten; choose a new version. | +| S3 classes fail to compile in a consumer | Add the matching optional AWS SDK dependency shown above. | diff --git a/java/build.gradle b/java/build.gradle new file mode 100644 index 0000000..de34ddd --- /dev/null +++ b/java/build.gradle @@ -0,0 +1,125 @@ +import com.vanniktech.maven.publish.DeploymentValidation + +plugins { + id 'java-library' + id 'com.vanniktech.maven.publish' version '0.37.0' +} + +group = 'com.esamtrade' +version = '0.1.0' +description = 'BucketBase Java library for abstracting object storage solutions' + +def commonsIoVersion = '2.16.1' +def awsSdkV2Version = '2.30.36' +def awsSdkV1Version = '1.12.782' +def junitVersion = '5.10.0' +def trinoVersion = '483' + +repositories { + mavenCentral() +} + +java { + toolchain { + languageVersion = JavaLanguageVersion.of(25) + } +} + +dependencies { + // Required internally by PurePosixPath and PurePosixPrefix at runtime. + implementation "commons-io:commons-io:${commonsIoVersion}" + + // Optional backends. Consumers add only the SDK for the backend they use. + compileOnly "software.amazon.awssdk:s3:${awsSdkV2Version}" + compileOnly "com.amazonaws:aws-java-sdk-s3:${awsSdkV1Version}" + testImplementation "software.amazon.awssdk:s3:${awsSdkV2Version}" + testImplementation "com.amazonaws:aws-java-sdk-s3:${awsSdkV1Version}" + + testImplementation platform("org.junit:junit-bom:${junitVersion}") + testImplementation 'org.junit.jupiter:junit-jupiter' + testRuntimeOnly 'org.junit.platform:junit-platform-launcher' + + // Used only to prove real Parquet partial reads, projection, predicate pushdown and writes. + testImplementation "io.trino:trino-parquet:${trinoVersion}" + testImplementation "io.trino:trino-spi:${trinoVersion}" +} + +tasks.withType(JavaCompile).configureEach { + options.release = 25 + options.encoding = 'UTF-8' +} + +tasks.withType(Test).configureEach { + useJUnitPlatform() + // Required only by the test-scoped Trino Parquet implementation. + jvmArgs '--add-modules', 'jdk.incubator.vector', + '--enable-native-access=ALL-UNNAMED' +} + +tasks.withType(Javadoc).configureEach { + options.encoding = 'UTF-8' + options.addBooleanOption('Xdoclint:none', true) +} + +mavenPublishing { + coordinates('com.esamtrade', 'bucketbase', project.version.toString()) + publishToMavenCentral(true, DeploymentValidation.PUBLISHED) + signAllPublications() + + pom { + name = 'BucketBase' + description = project.description + url = 'https://github.com/eSAMTrade/bucketbase' + + licenses { + license { + name = 'MIT License' + url = 'https://opensource.org/licenses/MIT' + distribution = 'repo' + } + } + + developers { + developer { + id = 'esamtrade' + name = 'eSAMTrade' + email = 'contact@esamtrade.com' + } + } + + scm { + connection = 'scm:git:git://github.com/eSAMTrade/bucketbase.git' + developerConnection = 'scm:git:ssh://git@github.com/eSAMTrade/bucketbase.git' + url = 'https://github.com/eSAMTrade/bucketbase' + } + } +} + +// Gradle compileOnly dependencies are intentionally absent from a generated Maven POM. +// Publish the two backend SDKs as Maven-optional so consumers can discover their coordinates +// without forcing either SDK onto core-only users. +publishing.publications.withType(MavenPublication).configureEach { + pom.withXml { + Node dependenciesNode = asNode().dependencies.isEmpty() + ? asNode().appendNode('dependencies') + : asNode().dependencies[0] + [ + ['software.amazon.awssdk', 's3', awsSdkV2Version], + ['com.amazonaws', 'aws-java-sdk-s3', awsSdkV1Version] + ].each { coordinates -> + Node dependencyNode = dependenciesNode.appendNode('dependency') + dependencyNode.appendNode('groupId', coordinates[0]) + dependencyNode.appendNode('artifactId', coordinates[1]) + dependencyNode.appendNode('version', coordinates[2]) + dependencyNode.appendNode('scope', 'compile') + dependencyNode.appendNode('optional', 'true') + } + } +} + +// A Central release is immutable. Never allow an upload task to bypass the full test suite. +tasks.withType(PublishToMavenRepository).configureEach { + if (name.endsWith('ToMavenCentralRepository')) { + dependsOn tasks.named('test') + } +} diff --git a/java/gradle/wrapper/gradle-wrapper.jar b/java/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000..00c1545 --- /dev/null +++ b/java/gradle/wrapper/gradle-wrapper.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:497c8c2a7e5031f6aa847f88104aa80a93532ec32ee17bdb8d1d2f67a194a9c7 +size 48462 diff --git a/java/gradle/wrapper/gradle-wrapper.properties b/java/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..a9db115 --- /dev/null +++ b/java/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,9 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-9.6.1-bin.zip +networkTimeout=10000 +retries=0 +retryBackOffMs=500 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/java/gradlew b/java/gradlew new file mode 100644 index 0000000..249efbb --- /dev/null +++ b/java/gradlew @@ -0,0 +1,248 @@ +#!/bin/sh + +# +# Copyright © 2015 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +############################################################################## +# +# gradlew start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh gradlew +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/3d91ce3b8caaf77ad09f381f43615b715b53f72c/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/java/gradlew.bat b/java/gradlew.bat new file mode 100644 index 0000000..a51ec4f --- /dev/null +++ b/java/gradlew.bat @@ -0,0 +1,82 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem gradlew startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables, and ensure extensions are enabled +setlocal EnableExtensions + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +"%COMSPEC%" /c exit 1 + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +"%COMSPEC%" /c exit 1 + +:execute +@rem Setup the command line + + + +@rem Execute gradlew +@rem endlocal doesn't take effect until after the line is parsed and variables are expanded +@rem which allows us to clear the local environment before executing the java command +endlocal & "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* & call :exitWithErrorLevel + +:exitWithErrorLevel +@rem Use "%COMSPEC%" /c exit to allow operators to work properly in scripts +"%COMSPEC%" /c exit %ERRORLEVEL% diff --git a/java/pom.xml b/java/pom.xml deleted file mode 100644 index f8102ef..0000000 --- a/java/pom.xml +++ /dev/null @@ -1,167 +0,0 @@ - - 4.0.0 - - com.esamtrade - bucketbase - 0.1.0-SNAPSHOT - BucketBase Java library for abstracting object storage solutions - BucketBase - https://github.com/esamtrade/bucketbase - - - UTF-8 - 17 - 17 - 2.16.1 - 2.30.36 - 1.12.782 - 5.10.0 - - - - - - commons-io - commons-io - ${commons.io.version} - provided - - - - - software.amazon.awssdk - s3 - ${aws.sdk.version.v2} - true - - - - - com.amazonaws - aws-java-sdk - ${aws.sdk.version.v1} - true - - - - - org.junit.jupiter - junit-jupiter - ${junit.version} - test - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.11.0 - - 17 - 17 - - - - - - org.apache.maven.plugins - maven-source-plugin - 3.3.0 - - - attach-sources - verify - - jar - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - 3.5.0 - - - attach-javadocs - verify - - jar - - - - - - - - org.apache.maven.plugins - maven-gpg-plugin - 3.1.0 - - - sign-artifacts - verify - - sign - - - pypi@esamtrade.com - - - - - - org.sonatype.central - central-publishing-maven-plugin - 0.7.0 - true - - central - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.0.0 - - - ${MINIO_ACCESS_KEY} - ${MINIO_SECRET_KEY} - - - - - - - - - - esamtrade - ESAMTrade - contact@esamtrade.com - - - esamtrade-pypi - ESAMTrade pypi - pypi@esamtrade.com - - - - - - Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0 - repo - - - - - scm:git:git://github.com/esamtrade/bucketbase.git - scm:git:ssh://github.com:esamtrade/bucketbase.git - https://github.com/esamtrade/bucketbase - - - \ No newline at end of file diff --git a/java/settings.gradle b/java/settings.gradle new file mode 100644 index 0000000..ee43905 --- /dev/null +++ b/java/settings.gradle @@ -0,0 +1,8 @@ +pluginManagement { + repositories { + gradlePluginPortal() + mavenCentral() + } +} + +rootProject.name = 'bucketbase' diff --git a/java/src/main/java/README.md b/java/src/main/java/README.md deleted file mode 100644 index 0ab1477..0000000 --- a/java/src/main/java/README.md +++ /dev/null @@ -1,66 +0,0 @@ -# Bucketbase - -## Instructions to build and publish a release - -### Prerequisites -- An account on central.sonatype.com has to be created using instructions at https://central.sonatype.org/register/central-portal/#create-an-account -_Important Note_: enable SNAPSHOT publishing after defining the namespace. -- Create authentication tokens on central.sonatype.com and save them in ~/.mvn/settings.xml: -```xml - - - - - central - PICK FROM ZOHO - PICK FROM ZOHO - - - -``` -_Important note_ The authentication tokens were regenerated so remember to update the above. -- GPG Signatures: This one-time-only step was already performed - - create a keypair - ``gpg --gen-key`` - - publish the public key: ``gpg --keyserver keys.openpgp.org --send-keys AAAB0592065F7ED7BC852DABAC39F2FDF53EE367``. Remember to confirm the publishing via email link. -- Picking the key pair from ZOHO, saved under pypi@esamtrade.com Password Custom Fields -- Define credentials in .mvn/settings.xml to allow tests to run with minio credentials: -```xml - - - - default - - minio-dev-tests - PICK FROM ZOHO KEY minio-dev-tests@minio - - - - - - default - - - -``` -### Adjustments in pom.xml -In case of multiple keys being locally defined, you need to indicate we want to sign with pypi@esamtrade.com -```xml - - org.apache.maven.plugins - maven-gpg-plugin - 3.1.0 - - - sign-artifacts - verify - - sign - - - pypi@esamtrade.com - - - - -``` diff --git a/java/src/main/java/com/esamtrade/bucketbase/AbstractAppendOnlySynchronizedBucket.java b/java/src/main/java/com/esamtrade/bucketbase/AbstractAppendOnlySynchronizedBucket.java index c666897..309f232 100644 --- a/java/src/main/java/com/esamtrade/bucketbase/AbstractAppendOnlySynchronizedBucket.java +++ b/java/src/main/java/com/esamtrade/bucketbase/AbstractAppendOnlySynchronizedBucket.java @@ -3,12 +3,25 @@ import java.io.IOException; import java.io.InputStream; import java.util.List; +import java.util.Objects; -public abstract class AbstractAppendOnlySynchronizedBucket extends BaseBucket { - private final BaseBucket baseBucket; +/** + * A write-once wrapper around another bucket: an object may be created but never overwritten or + * deleted, and concurrent writes to the same key are serialized by a lock provided by the + * concrete subclass. Reads are unsynchronized because a completed write is atomic. + * + *

Mirrors the Python library's {@code AbstractAppendOnlySynchronizedBucket}.

+ */ +public abstract class AbstractAppendOnlySynchronizedBucket implements IBucket { + private final IBucket baseBucket; - public AbstractAppendOnlySynchronizedBucket(BaseBucket baseBucket) { - this.baseBucket = baseBucket; + protected AbstractAppendOnlySynchronizedBucket(IBucket baseBucket) { + this.baseBucket = Objects.requireNonNull(baseBucket, "baseBucket"); + } + + /** The wrapped bucket that all operations delegate to. */ + protected IBucket baseBucket() { + return baseBucket; } @Override @@ -33,37 +46,26 @@ public void putObjectStream(PurePosixPath name, InputStream stream) throws IOExc @Override public byte[] getObject(PurePosixPath name) throws IOException { - if (exists(name)) { - return baseBucket.getObject(name); - } - lockObject(name); - try { - return baseBucket.getObject(name); - } finally { - unlockObject(name); - } + return baseBucket.getObject(name); } @Override - public ObjectStream getObjectStream(PurePosixPath name) throws IOException { - if (exists(name)) { - return baseBucket.getObjectStream(name); - } - lockObject(name); - try { - return baseBucket.getObjectStream(name); - } finally { - unlockObject(name); - } + public SeekableInputStream getObjectStream(PurePosixPath name) throws IOException { + return baseBucket.getObjectStream(name); + } + + @Override + public long getSize(PurePosixPath name) throws IOException { + return baseBucket.getSize(name); } @Override - public List listObjects(PurePosixPath prefix) throws IOException { + public List listObjects(PurePosixPrefix prefix) throws IOException { return baseBucket.listObjects(prefix); } @Override - public ShallowListing shallowListObjects(PurePosixPath prefix) throws IOException { + public ShallowListing shallowListObjects(PurePosixPrefix prefix) throws IOException { return baseBucket.shallowListObjects(prefix); } @@ -73,8 +75,29 @@ public boolean exists(PurePosixPath name) throws IOException { } @Override - public List removeObjects(List names) throws IOException { - throw new UnsupportedOperationException("remove_objects is not supported for AbstractAppendOnlySynchronizedBucket"); + public List removeObjects(List names) { + throw new UnsupportedOperationException("removeObjects is not supported for an append-only bucket"); + } + + @Override + public void removePrefix(PurePosixPrefix prefix) { + throw new UnsupportedOperationException("removePrefix is not supported for an append-only bucket"); + } + + @Override + public void movePrefix( + IBucket dstBucket, + PurePosixPrefix srcPrefix, + PurePosixPrefix dstPrefix, + int threads) { + // Fail fast: the copy would otherwise succeed and only the delete step would fail, leaving + // the data duplicated. + throw new UnsupportedOperationException("movePrefix is not supported for an append-only bucket"); + } + + @Override + public void close() throws IOException { + baseBucket.close(); } protected abstract void lockObject(PurePosixPath name); diff --git a/java/src/main/java/com/esamtrade/bucketbase/BaseBucket.java b/java/src/main/java/com/esamtrade/bucketbase/BaseBucket.java deleted file mode 100644 index 5dbd20e..0000000 --- a/java/src/main/java/com/esamtrade/bucketbase/BaseBucket.java +++ /dev/null @@ -1,170 +0,0 @@ -package com.esamtrade.bucketbase; - - -import java.io.IOException; -import java.io.InputStream; -import java.io.UnsupportedEncodingException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.util.*; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - - - -class S3Utils { - public static final String S3_NAME_CHARS_NO_SEP = "\\w!\\-\\.\\)\\("; - public static final Pattern S3_NAME_SAFE_RE = Pattern.compile("^[{S3_NAME_CHARS_NO_SEP}][{S3_NAME_CHARS_NO_SEP}/]+$"); -} - -public abstract class BaseBucket implements IBucket { - protected static final String SEP = "/"; - protected static final Pattern SPLIT_PREFIX_RE = Pattern.compile("^((?:[" + S3Utils.S3_NAME_CHARS_NO_SEP + "]+/)*)([" + S3Utils.S3_NAME_CHARS_NO_SEP + "]*)$"); - protected static final Pattern OBJ_NAME_RE = Pattern.compile("^(?:[" + S3Utils.S3_NAME_CHARS_NO_SEP + "]+/)*[" + S3Utils.S3_NAME_CHARS_NO_SEP + "]+$"); - protected static final String DEFAULT_ENCODING = "utf-8"; - protected static final int MINIO_PATH_TEMP_SUFFIX_LEN = 43; - protected static final int WINDOWS_MAX_PATH = 260; - - public static class Tuple { - public final T first; - public final U second; - - public Tuple(T first, U second) { - this.first = first; - this.second = second; - } - } - - protected static Tuple splitPrefix(PurePosixPath prefix) { - String prefixStr = prefix.toString(); - if (prefixStr.isEmpty()) { - return new Tuple<>("", ""); - } - Matcher matcher = SPLIT_PREFIX_RE.matcher(prefixStr); - if (matcher.matches()) { - String dirPrefix = Optional.ofNullable(matcher.group(1)).orElse(""); - String namePrefix = Optional.ofNullable(matcher.group(2)).orElse(""); - return new Tuple<>(dirPrefix, namePrefix); - } - throw new IllegalArgumentException("Invalid S3 prefix: " + prefixStr); - } - - protected static byte[] encodeContent(String content) throws UnsupportedEncodingException { - return content.getBytes(DEFAULT_ENCODING); - } - - protected static byte[] encodeContent(byte[] content) { - return content; - } - - protected static String validateName(String name) { - if (!OBJ_NAME_RE.matcher(name).matches()) { - throw new IllegalArgumentException("Invalid S3 object name: " + name); - } - return name; - } - protected static String validateName(PurePosixPath name) { - String nameStr = name.toString(); - return validateName(nameStr); - } - - public abstract void putObject(PurePosixPath name, byte[] content) throws IOException; - - public abstract void putObjectStream(PurePosixPath name, InputStream stream) throws IOException; - - public abstract byte[] getObject(PurePosixPath name) throws IOException; - - public abstract ObjectStream getObjectStream(PurePosixPath name) throws IOException; - - public void fputObject(PurePosixPath name, Path filePath) throws IOException { - byte[] content = Files.readAllBytes(filePath); - putObject(name, content); - } - - public void fgetObject(PurePosixPath name, Path filePath) throws IOException { - String randomSuffix = UUID.randomUUID().toString().substring(0, 8); - Path tmpFilePath = filePath.getParent().resolve("_" + filePath.getFileName() + "." + randomSuffix + ".part"); - - try { - byte[] response = getObject(name); - Files.write(tmpFilePath, response); - Files.move(tmpFilePath, filePath, StandardCopyOption.REPLACE_EXISTING); - } catch (IOException exc) { - if (System.getProperty("os.name").toLowerCase().contains("win")) { - if (tmpFilePath.toString().length() >= WINDOWS_MAX_PATH - MINIO_PATH_TEMP_SUFFIX_LEN) { - throw new IllegalArgumentException( - "Reduce the Minio cache path length, Windows has limitation on the path length. " + - "More details here: https://docs.python.org/3/using/windows.html#removing-the-max-path-limitation", - exc - ); - } - } - throw exc; - } finally { - Files.deleteIfExists(tmpFilePath); - } - } - - public void removePrefix(PurePosixPath prefix) throws IOException { - List objects = listObjects(prefix); - removeObjects(objects); - } - - public abstract List listObjects(PurePosixPath prefix) throws IOException; - - public abstract ShallowListing shallowListObjects(PurePosixPath prefix) throws IOException; - - public abstract boolean exists(PurePosixPath name) throws IOException; - - public abstract List removeObjects(List names) throws IOException; - - public void copyPrefix(BaseBucket dstBucket, PurePosixPath srcPrefix, PurePosixPath dstPrefix, int threads) throws IOException { - if (threads <= 0) { - throw new IllegalArgumentException("threads must be greater than 0"); - } - - List srcObjects = listObjects(srcPrefix); - String srcPrefixStr = srcPrefix.toString(); - String dstPrefixStr = dstPrefix.toString(); - int srcPrefixLen = srcPrefixStr.length(); - - ExecutorService executorService = Executors.newFixedThreadPool(Math.min(threads, srcObjects.size())); - List> futures = new ArrayList<>(); - - for (PurePosixPath srcObj : srcObjects) { - futures.add(executorService.submit(() -> { - String objStr = srcObj.toString(); - if (!objStr.startsWith(srcPrefixStr)) { - return null; - } - - String name = dstPrefixStr + objStr.substring(srcPrefixLen); - if (name.startsWith("/")) { - name = name.substring(1); - } - dstBucket.putObject(PurePosixPath.from(name), getObject(srcObj)); - return null; - })); - } - - for (Future future : futures) { - try { - future.get(); - } catch (ExecutionException | InterruptedException e) { - throw new IOException(e); - } - } - - executorService.shutdown(); - } - - public void movePrefix(BaseBucket dstBucket, PurePosixPath srcPrefix, PurePosixPath dstPrefix, int threads) throws IOException { - copyPrefix(dstBucket, srcPrefix, dstPrefix, threads); - removePrefix(srcPrefix); - } -} diff --git a/java/src/main/java/com/esamtrade/bucketbase/DeleteError.java b/java/src/main/java/com/esamtrade/bucketbase/DeleteError.java index 8f64918..c041b74 100644 --- a/java/src/main/java/com/esamtrade/bucketbase/DeleteError.java +++ b/java/src/main/java/com/esamtrade/bucketbase/DeleteError.java @@ -1,19 +1,15 @@ package com.esamtrade.bucketbase; -public class DeleteError extends Exception { - public DeleteError(String message) { - super(message); - } - - public DeleteError(String message, Throwable cause) { - super(message, cause); - } - - public DeleteError(Throwable cause) { - super(cause); - } - - public DeleteError() { - super(); - } +/** + * A single failed deletion returned by {@link IBucket#removeObjects(java.util.List)}. + * + *

This is a value type (not an exception): {@code removeObjects} is a bulk operation that + * reports per-key failures without aborting, so callers inspect the returned list. Mirrors the + * Python library's {@code DeleteError}.

+ * + * @param code the storage-provider error code (e.g. {@code AccessDenied}) + * @param message the human-readable error message + * @param name the object key that failed to delete + */ +public record DeleteError(String code, String message, String name) { } diff --git a/java/src/main/java/com/esamtrade/bucketbase/IBucket.java b/java/src/main/java/com/esamtrade/bucketbase/IBucket.java index ecd6ec4..5d7d261 100644 --- a/java/src/main/java/com/esamtrade/bucketbase/IBucket.java +++ b/java/src/main/java/com/esamtrade/bucketbase/IBucket.java @@ -1,5 +1,6 @@ package com.esamtrade.bucketbase; +import java.io.Closeable; import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; @@ -7,30 +8,82 @@ import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.List; -import java.util.Optional; import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -public interface IBucket { - String SEP = "/"; +public interface IBucket extends Closeable { int MINIO_PATH_TEMP_SUFFIX_LEN = 43; int WINDOWS_MAX_PATH = 260; String DEFAULT_ENCODING = "utf-8"; + /** Characters allowed in an object name besides the {@code /} separator; matches the Python library. */ + String S3_NAME_CHARS_NO_SEP = "\\w!\\-\\.')("; + + /** + * Releases any resources this bucket owns (for example S3 clients it created itself). + * The default is a no-op; backends that hold no resources need not override it. + */ + @Override + default void close() throws IOException { + } + void putObject(PurePosixPath name, byte[] content) throws IOException; void putObjectStream(PurePosixPath name, InputStream stream) throws IOException; byte[] getObject(PurePosixPath name) throws IOException; - ObjectStream getObjectStream(PurePosixPath name) throws IOException; + /** + * Returns a readable, closeable stream positioned at the start of the object that supports + * querying its position and seeking to any non-negative offset. + */ + SeekableInputStream getObjectStream(PurePosixPath name) throws IOException; + + /** + * Returns the size of the object in bytes. + * + * @throws java.io.FileNotFoundException if the object does not exist + */ + long getSize(PurePosixPath name) throws IOException; + + /** + * Opens a streaming sink for {@code name}. Bytes are uploaded as they are written, and the + * object becomes visible only once {@link ObjectWriter#commit()} succeeds - abandoning the + * writer leaves no object behind. + * + *

This is the counterpart of the Python library's {@code IBucket.open_write()} and is the + * intended way to write large or incrementally produced payloads such as Parquet files.

+ * + * @see ObjectWriter + */ + default ObjectWriter openWrite(PurePosixPath name) throws IOException { + return openWrite(name, ObjectWriter.DEFAULT_TIMEOUT_MILLIS); + } - List listObjects(PurePosixPath prefix) throws IOException; + /** + * As {@link #openWrite(PurePosixPath)}, but bounds how long commit and abort wait for the + * background uploader to settle. + */ + default ObjectWriter openWrite(PurePosixPath name, long timeoutMillis) throws IOException { + return new ObjectWriter(this, name, timeoutMillis); + } + + List listObjects(PurePosixPrefix prefix) throws IOException; - ShallowListing shallowListObjects(PurePosixPath prefix) throws IOException; + /** Lists every object in the bucket using the empty prefix. */ + default List listObjects() throws IOException { + return listObjects(new PurePosixPrefix()); + } + + ShallowListing shallowListObjects(PurePosixPrefix prefix) throws IOException; + + /** Lists root-level objects and common prefixes using the empty prefix. */ + default ShallowListing shallowListObjects() throws IOException { + return shallowListObjects(new PurePosixPrefix()); + } boolean exists(PurePosixPath name) throws IOException; @@ -43,7 +96,8 @@ default void fputObject(PurePosixPath name, Path filePath) throws IOException { default void fgetObject(PurePosixPath name, Path filePath) throws IOException { String randomSuffix = UUID.randomUUID().toString().substring(0, 8); - Path tmpFilePath = filePath.getParent().resolve("_" + filePath.getFileName() + "." + randomSuffix + ".part"); + Path parent = filePath.toAbsolutePath().getParent(); + Path tmpFilePath = parent.resolve("_" + filePath.getFileName() + "." + randomSuffix + ".part"); try { byte[] response = getObject(name); @@ -65,53 +119,73 @@ default void fgetObject(PurePosixPath name, Path filePath) throws IOException { } } - default void removePrefix(PurePosixPath prefix) throws IOException { + default void removePrefix(PurePosixPrefix prefix) throws IOException { List objects = listObjects(prefix); removeObjects(objects); } - default void copyPrefix(IBucket dstBucket, PurePosixPath srcPrefix, PurePosixPath dstPrefix, int threads) throws IOException { + /** + * Copies an object from another bucket into this one, streaming it through without holding + * the whole object in memory. + */ + default void copyObjectFrom(IBucket srcBucket, PurePosixPath srcName, PurePosixPath dstName) throws IOException { + try (SeekableInputStream stream = srcBucket.getObjectStream(srcName)) { + putObjectStream(dstName, stream); + } + } + + /** + * Copies every object under {@code srcPrefix} from this bucket to {@code dstPrefix} in + * {@code dstBucket}, using up to {@code threads} worker threads. + */ + default void copyPrefix( + IBucket dstBucket, + PurePosixPrefix srcPrefix, + PurePosixPrefix dstPrefix, + int threads) throws IOException { if (threads <= 0) { throw new IllegalArgumentException("threads must be greater than 0"); } - List srcObjects = listObjects(srcPrefix); - String srcPrefixStr = srcPrefix.toString(); + if (srcObjects.isEmpty()) { + return; + } + int srcPrefixLen = srcPrefix.toString().length(); String dstPrefixStr = dstPrefix.toString(); - int srcPrefixLen = srcPrefixStr.length(); - - ExecutorService executorService = Executors.newFixedThreadPool(Math.min(threads, srcObjects.size())); - List> futures = new ArrayList<>(); - for (PurePosixPath srcObj : srcObjects) { - futures.add(executorService.submit(() -> { - String objStr = srcObj.toString(); - if (!objStr.startsWith(srcPrefixStr)) { + ExecutorService executor = Executors.newFixedThreadPool(Math.min(threads, srcObjects.size())); + try { + List> futures = new ArrayList<>(srcObjects.size()); + for (PurePosixPath srcObj : srcObjects) { + futures.add(executor.submit(() -> { + String name = dstPrefixStr + srcObj.toString().substring(srcPrefixLen); + if (name.startsWith("/")) { + name = name.substring(1); + } + dstBucket.putObject(PurePosixPath.from(name), getObject(srcObj)); return null; - } - - String name = dstPrefixStr + objStr.substring(srcPrefixLen); - if (name.startsWith("/")) { - name = name.substring(1); - } - dstBucket.putObject(PurePosixPath.from(name), getObject(srcObj)); - return null; - })); - } - - for (Future future : futures) { - try { + })); + } + for (Future future : futures) { future.get(); - } catch (ExecutionException | InterruptedException e) { - throw new IOException(e); } + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + throw cause instanceof IOException io ? io : new IOException(cause); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted while copying prefix " + srcPrefix, e); + } finally { + executor.shutdownNow(); } - - executorService.shutdown(); } - default void movePrefix(IBucket dstBucket, PurePosixPath srcPrefix, PurePosixPath dstPrefix, int threads) throws IOException { + default void movePrefix( + IBucket dstBucket, + PurePosixPrefix srcPrefix, + PurePosixPrefix dstPrefix, + int threads) throws IOException { copyPrefix(dstBucket, srcPrefix, dstPrefix, threads); removePrefix(srcPrefix); } -} \ No newline at end of file +} diff --git a/java/src/main/java/com/esamtrade/bucketbase/MemoryBucket.java b/java/src/main/java/com/esamtrade/bucketbase/MemoryBucket.java index 25eaab1..991fe43 100644 --- a/java/src/main/java/com/esamtrade/bucketbase/MemoryBucket.java +++ b/java/src/main/java/com/esamtrade/bucketbase/MemoryBucket.java @@ -1,130 +1,106 @@ package com.esamtrade.bucketbase; -import java.io.ByteArrayInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; -import java.util.*; -import java.util.concurrent.locks.ReentrantLock; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; -public class MemoryBucket extends BaseBucket { - /** - * Implements BaseBucket interface, but stores all objects in memory. - * This class is intended to be used for testing purposes only. - */ +/** + * An in-memory {@link IBucket}, intended for tests and local development. + * + *

Stored bytes are copied on the way in and out, so a caller mutating an array it passed to + * {@link #putObject} — or one returned by {@link #getObject} — cannot corrupt stored objects. + * This matches the value semantics of the real backends. Backed by a {@link ConcurrentHashMap}, + * so concurrent access is safe.

+ */ +public class MemoryBucket implements IBucket { - private final Map objects; - private final ReentrantLock lock; - - public MemoryBucket() { - this.objects = new HashMap<>(); - this.lock = new ReentrantLock(); - } + private final ConcurrentHashMap objects = new ConcurrentHashMap<>(); @Override public void putObject(PurePosixPath name, byte[] content) { - String _name = validateName(name); - byte[] _content = encodeContent(content); - lock.lock(); - try { - objects.put(_name, _content); - } finally { - lock.unlock(); - } + String _name = S3Names.validateName(name); + objects.put(_name, content.clone()); } @Override public void putObjectStream(PurePosixPath name, InputStream stream) throws IOException { - byte[] _content = stream.readAllBytes(); - putObject(name, _content); + String _name = S3Names.validateName(name); + objects.put(_name, stream.readAllBytes()); } @Override public byte[] getObject(PurePosixPath name) throws FileNotFoundException { - String _name = validateName(name); - lock.lock(); - try { - if (!objects.containsKey(_name)) { - throw new FileNotFoundException("Object " + _name + " not found in MemoryBucket"); - } - return objects.get(_name); - } finally { - lock.unlock(); - } + return readObject(name).clone(); } @Override - public ObjectStream getObjectStream(PurePosixPath name) throws FileNotFoundException { - byte[] content = getObject(name); - return new ObjectStream(new ByteArrayInputStream(content), name.toString()); + public SeekableInputStream getObjectStream(PurePosixPath name) throws FileNotFoundException { + byte[] content = readObject(name); + return new RangeSeekableInputStream(content.length, 0, + (offset, dest, destOffset, length) -> System.arraycopy(content, (int) offset, dest, destOffset, length)); } @Override - public List listObjects(PurePosixPath prefix) { - splitPrefix(prefix); // validate prefix - String strPrefix = prefix.toString(); - lock.lock(); - try { - return objects.keySet().stream() - .filter(obj -> obj.startsWith(strPrefix)) - .map(PurePosixPath::from) - .collect(Collectors.toList()); - } finally { - lock.unlock(); + public long getSize(PurePosixPath name) throws FileNotFoundException { + return readObject(name).length; + } + + private byte[] readObject(PurePosixPath name) throws FileNotFoundException { + String _name = S3Names.validateName(name); + byte[] content = objects.get(_name); + if (content == null) { + throw new FileNotFoundException("Object " + _name + " not found in MemoryBucket"); } + return content; } @Override - public ShallowListing shallowListObjects(PurePosixPath prefix) throws IOException { - splitPrefix(prefix); // validate prefix - String strPrefix = prefix.toString(); + public List listObjects(PurePosixPrefix prefix) { + String strPrefix = S3Names.validatePrefix(prefix); + return objects.keySet().stream() + .filter(key -> key.startsWith(strPrefix)) + .map(PurePosixPath::from) + .collect(Collectors.toList()); + } + + @Override + public ShallowListing shallowListObjects(PurePosixPrefix prefix) { + String strPrefix = S3Names.validatePrefix(prefix); int prefLen = strPrefix.length(); List objectsList = new ArrayList<>(); - Set prefixesSet = new HashSet<>(); - lock.lock(); - try { - for (PurePosixPath obj : listObjects(prefix)) { - String sobj = obj.toString(); - if (!sobj.substring(prefLen).contains("/")) { - objectsList.add(obj); - } else { - String suffix = sobj.substring(prefLen); - String commonSuffix = suffix.split("/", 2)[0]; - String commonPrefix = strPrefix + commonSuffix + "/"; - prefixesSet.add(PurePosixPath.from(commonPrefix)); - } + Set prefixesSet = new HashSet<>(); + for (String key : objects.keySet()) { + if (!key.startsWith(strPrefix)) { + continue; + } + String suffix = key.substring(prefLen); + int slash = suffix.indexOf('/'); + if (slash < 0) { + objectsList.add(PurePosixPath.from(key)); + } else { + prefixesSet.add(PurePosixPrefix.from( + strPrefix + suffix.substring(0, slash) + "/")); } - } finally { - lock.unlock(); } return new ShallowListing(objectsList, new ArrayList<>(prefixesSet)); } @Override public boolean exists(PurePosixPath name) { - String _name = validateName(name); - lock.lock(); - try { - return objects.containsKey(_name); - } finally { - lock.unlock(); - } + return objects.containsKey(S3Names.validateName(name)); } @Override public List removeObjects(List names) { - List _listOfObjects = names.stream().collect(Collectors.toList()); - List deleteErrors = new ArrayList<>(); - lock.lock(); - try { - for (PurePosixPath obj : _listOfObjects) { - String validatedObj = validateName(obj); - objects.remove(validatedObj); - } - } finally { - lock.unlock(); - } - return deleteErrors; + // Validate every name up front so an invalid entry cannot leave a half-applied deletion. + List validated = names.stream().map(S3Names::validateName).toList(); + validated.forEach(objects::remove); + return List.of(); } } diff --git a/java/src/main/java/com/esamtrade/bucketbase/ObjectStream.java b/java/src/main/java/com/esamtrade/bucketbase/ObjectStream.java deleted file mode 100644 index 0733004..0000000 --- a/java/src/main/java/com/esamtrade/bucketbase/ObjectStream.java +++ /dev/null @@ -1,28 +0,0 @@ -package com.esamtrade.bucketbase; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Path; - -public class ObjectStream implements AutoCloseable { - private final InputStream stream; - private final String name; - - public ObjectStream(InputStream stream, String name) { - this.stream = stream; - this.name = name; - } - - public InputStream getStream() { - return stream; - } - - public String getName() { - return name; - } - - @Override - public void close() throws IOException { - stream.close(); - } -} diff --git a/java/src/main/java/com/esamtrade/bucketbase/ObjectWriter.java b/java/src/main/java/com/esamtrade/bucketbase/ObjectWriter.java new file mode 100644 index 0000000..b6dcc54 --- /dev/null +++ b/java/src/main/java/com/esamtrade/bucketbase/ObjectWriter.java @@ -0,0 +1,199 @@ +package com.esamtrade.bucketbase; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicReference; + +/** + * A streaming sink for a single object. Bytes written to {@link #stream()} are uploaded by a + * background thread as they arrive, so an arbitrarily large object can be produced without + * buffering it in memory. For S3-backed buckets this maps onto a multipart upload. + * + *

Nothing is stored unless {@link #commit()} is called. Closing without + * committing aborts the upload and leaves no object behind. This is the guarantee that makes + * the writer safe to use with formats that finalise on close, such as Parquet:

+ * + * 
{@code
+ * try (ObjectWriter writer = bucket.openWrite(PurePosixPath.from("data.parquet"))) {
+ *     try (ParquetWriter parquet = new ParquetWriter(writer.stream(), ...)) {
+ *         parquet.write(page);
+ *     }            // writes the Parquet footer and closes the stream - but does NOT commit
+ *     writer.commit();
+ * }                // on an exception the upload is aborted; no partial object is visible
+ * }
+ * + *

{@link #stream()} deliberately ignores {@link OutputStream#close()} - a nested writer + * closing the sink must not be able to publish the object. Only {@code commit()} does that. + * This mirrors the Python library's {@code NonClosingStream} wrapper inside + * {@code AsyncObjectWriter}.

+ * + *

Instances are not thread-safe: write from one thread at a time.

+ */ +public final class ObjectWriter implements AutoCloseable { + + /** How long {@link #commit()} and {@link #close()} wait for the uploader to settle. */ + public static final long DEFAULT_TIMEOUT_MILLIS = 5 * 60 * 1000L; + + /** How long a cancelled uploader is given to unwind after being interrupted. */ + private static final long CANCELLATION_GRACE_MILLIS = 2_000L; + + private final PurePosixPath name; + private final StreamPipe pipe = new StreamPipe(); + private final AtomicReference uploadFailure = new AtomicReference<>(); + private final Thread uploader; + private final long timeoutMillis; + private final OutputStream sink; + + private boolean committed; + private boolean closed; + + ObjectWriter(IBucket bucket, PurePosixPath name, long timeoutMillis) { + this.name = Objects.requireNonNull(name, "name"); + Objects.requireNonNull(bucket, "bucket"); + if (timeoutMillis <= 0) { + throw new IllegalArgumentException("timeoutMillis must be positive"); + } + this.timeoutMillis = timeoutMillis; + this.sink = new NonClosingOutputStream(pipe.outputStream()); + this.uploader = new Thread(() -> { + // Closing the pipe input on exit wakes a writer blocked on back-pressure if the + // upload ends early (e.g. the store rejected it), so the caller cannot hang. + try (java.io.InputStream in = pipe.inputStream()) { + bucket.putObjectStream(name, in); + } catch (Throwable t) { + uploadFailure.set(t); + } + }, "bucketbase-writer-" + name); + this.uploader.setDaemon(true); + this.uploader.start(); + } + + /** + * The sink to write the object body to. Closing it has no effect; see the class javadoc. + */ + public OutputStream stream() { + return sink; + } + + public PurePosixPath name() { + return name; + } + + /** + * Finishes the upload and makes the object visible. After this returns successfully the + * object is stored. + * + * @throws IOException if the upload failed, timed out, or the writer was already closed + */ + public void commit() throws IOException { + if (closed) { + throw new IOException("Cannot commit " + name + ": the writer is already closed"); + } + if (committed) { + return; + } + pipe.finish(); + awaitUploader(); + Throwable failure = uploadFailure.get(); + if (failure != null) { + throw asIOException("Failed to write object " + name, failure); + } + committed = true; + } + + /** + * Aborts the upload unless {@link #commit()} succeeded first. Safe to call repeatedly, and + * safe to call after {@code commit()} - in that case it is a no-op. + */ + @Override + public void close() throws IOException { + if (closed) { + return; + } + closed = true; + if (committed) { + return; + } + pipe.abort(new IOException("Object write for " + name + " was abandoned without commit")); + awaitUploader(); + // A failure here is the expected consequence of the abort, so it is not propagated: + // the contract is simply that no object was created. + uploadFailure.set(null); + } + + /** + * Waits for the uploader to settle, cancelling it if it outlives the timeout. + * + *

The timeout has to be an upper bound on the upload, not merely on the caller's + * wait: an uploader left running could finish later and publish an object the caller was + * already told had failed. Cancellation is best-effort - it depends on the underlying store + * honouring interruption - but leaving the thread running guarantees the contract is broken.

+ */ + private void awaitUploader() throws IOException { + boolean timedOut = false; + try { + uploader.join(timeoutMillis); + if (uploader.isAlive()) { + timedOut = true; + uploader.interrupt(); + // Give the interrupt a brief chance to unwind before reporting the timeout. + uploader.join(CANCELLATION_GRACE_MILLIS); + } + } catch (InterruptedException e) { + uploader.interrupt(); + Thread.currentThread().interrupt(); + throw new IOException("Interrupted while finishing the write of " + name, e); + } + if (uploader.isAlive()) { + throw new IOException("Timed out after " + timeoutMillis + " ms waiting for the upload of " + name + + "; the upload was cancelled but did not stop"); + } + if (timedOut) { + throw new IOException("Timed out after " + timeoutMillis + " ms waiting for the upload of " + name); + } + } + + private static IOException asIOException(String message, Throwable cause) { + if (cause instanceof IOException io) { + return new IOException(message + ": " + io.getMessage(), io); + } + return new IOException(message, cause); + } + + /** + * Wraps the pipe so that a nested writer's {@code close()} cannot commit the object. + */ + private static final class NonClosingOutputStream extends OutputStream { + private final OutputStream delegate; + + NonClosingOutputStream(OutputStream delegate) { + this.delegate = delegate; + } + + @Override + public void write(int b) throws IOException { + delegate.write(b); + } + + @Override + public void write(byte[] source) throws IOException { + delegate.write(source); + } + + @Override + public void write(byte[] source, int offset, int length) throws IOException { + delegate.write(source, offset, length); + } + + @Override + public void flush() throws IOException { + delegate.flush(); + } + + @Override + public void close() { + // Intentionally not closing: only ObjectWriter.commit() may end the upload. + } + } +} diff --git a/java/src/main/java/com/esamtrade/bucketbase/PurePosixPath.java b/java/src/main/java/com/esamtrade/bucketbase/PurePosixPath.java index ac123f8..22dce64 100644 --- a/java/src/main/java/com/esamtrade/bucketbase/PurePosixPath.java +++ b/java/src/main/java/com/esamtrade/bucketbase/PurePosixPath.java @@ -3,125 +3,184 @@ import org.apache.commons.io.FilenameUtils; import java.nio.file.Path; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; -public class PurePosixPath implements Comparable { +/** + * Immutable, purely lexical POSIX path matching Python's standard-library + * {@code pathlib.PurePosixPath} semantics (verified against the installed Python 3.14 stdlib). + * + *

In particular, empty paths render as {@code "."}, redundant separators and {@code "."} + * components are removed, {@code ".."} components are preserved, a later absolute constructor + * or join operand replaces everything before it, trailing separators are removed, and exactly + * two leading separators are preserved. No method accesses the filesystem. Use + * {@link PurePosixPrefix} when a trailing separator must remain significant for an object-store + * prefix.

+ * + *

The Java-only {@link #resolve(String)} methods are aliases for the equivalent lexical + * {@link #join(String, String...)} operation.

+ * + * @see + * Python pathlib.PurePosixPath + */ +public final class PurePosixPath implements Comparable { - protected final String[] parts; - public final String SEP = "/"; + public static final String SEP = "/"; + + private final String root; + private final String[] tail; + private final String[] parts; public static PurePosixPath from(String path, String... more) { return new PurePosixPath(path, more); } public static PurePosixPath from(Path path) { - String name = path.toString(); - String posixPath = FilenameUtils.separatorsToUnix(name); - return new PurePosixPath(posixPath); + Objects.requireNonNull(path, "path"); + return new PurePosixPath(FilenameUtils.separatorsToUnix(path.toString())); + } + + public PurePosixPath() { + this(""); } public PurePosixPath(String first, String... more) { - List allParts = new ArrayList<>(); - if (!first.isEmpty()) { - allParts.addAll(Arrays.asList(first.split("/", -1))); - } - for (String part : more) { - if (!part.isEmpty()) { - allParts.addAll(Arrays.asList(part.split("/", -1))); - } - } - this.parts = normalizeParts(allParts); + Objects.requireNonNull(first, "first"); + Objects.requireNonNull(more, "more"); + + String[] paths = new String[more.length + 1]; + paths[0] = first; + System.arraycopy(more, 0, paths, 1, more.length); + ParsedPath parsed = parse(joinRawPaths(paths)); + this.root = parsed.root(); + this.tail = parsed.tail(); + this.parts = makeParts(root, tail); } + /** + * Constructs a path as though every array element were supplied as a separate Python + * {@code PurePosixPath(*paths)} argument. + */ public PurePosixPath(String[] paths) { - this.parts = normalizeParts(Arrays.asList(paths)); - } - - private String[] normalizeParts(List parts) { - List result = new ArrayList<>(); - int i = 0; - int n = parts.size(); - // add the first part if it is empty - if (!parts.isEmpty() && parts.get(0).isEmpty()) { - result.add(""); - i = 1; - } - while (i < n - 1) { - String part = parts.get(i++); - if (part.equals("..")) { - if (!result.isEmpty() && !result.get(result.size() - 1).equals("..")) { - result.remove(result.size() - 1); - } else { - result.add(part); - } - } else if (!part.equals(".") && !part.isEmpty()) { - result.add(part); + Objects.requireNonNull(paths, "paths"); + ParsedPath parsed = parse(joinRawPaths(paths)); + this.root = parsed.root(); + this.tail = parsed.tail(); + this.parts = makeParts(root, tail); + } + + private PurePosixPath(ParsedPath parsed) { + this.root = parsed.root(); + this.tail = parsed.tail(); + this.parts = makeParts(root, this.tail); + } + + private static String joinRawPaths(String[] paths) { + String result = ""; + for (String path : paths) { + Objects.requireNonNull(path, "path segment"); + if (path.startsWith(SEP)) { + result = path; + } else if (result.isEmpty() || result.endsWith(SEP)) { + result += path; + } else { + result += SEP + path; } } - if (i < n) { - String part = parts.get(i); - if (part.equals("..")) { - if (!result.isEmpty() && !result.get(result.size() - 1).equals("..")) { - result.set(result.size() - 1, ""); // set the last part to empty, as the .. means reference to directory - } else { - result.add(part); - } - } else if (part.equals(".")) { - result.add(""); - } else { - result.add(part); + return result; + } + + private static ParsedPath parse(String path) { + if (path.isEmpty()) { + return new ParsedPath("", new String[0]); + } + + int leadingSeparators = 0; + while (leadingSeparators < path.length() && path.charAt(leadingSeparators) == '/') { + leadingSeparators++; + } + + String root; + if (leadingSeparators == 2) { + root = "//"; + } else if (leadingSeparators > 0) { + root = SEP; + } else { + root = ""; + } + + List tail = new ArrayList<>(); + for (String part : path.substring(leadingSeparators).split(SEP, -1)) { + if (!part.isEmpty() && !part.equals(".")) { + tail.add(part); } } - return result.toArray(new String[0]); + return new ParsedPath(root, tail.toArray(String[]::new)); } + private static String[] makeParts(String root, String[] tail) { + if (root.isEmpty()) { + return tail.clone(); + } + String[] parts = new String[tail.length + 1]; + parts[0] = root; + System.arraycopy(tail, 0, parts, 1, tail.length); + return parts; + } public PurePosixPath join(String other, String... more) { - String[] combined = new String[1 + more.length]; - combined[0] = other; - System.arraycopy(more, 0, combined, 1, more.length); - return new PurePosixPath(this.toString(), combined); + Objects.requireNonNull(other, "other"); + Objects.requireNonNull(more, "more"); + String[] paths = new String[more.length + 2]; + paths[0] = toString(); + paths[1] = other; + System.arraycopy(more, 0, paths, 2, more.length); + return new PurePosixPath(paths); } public PurePosixPath join(PurePosixPath other) { - return new PurePosixPath(this.toString(), other.toString()); + Objects.requireNonNull(other, "other"); + return new PurePosixPath(toString(), other.toString()); + } + + /** + * Joins a prefix to this path. The result is a prefix because the final operand determines + * whether a trailing separator is significant. + */ + public PurePosixPrefix join(PurePosixPrefix other) { + Objects.requireNonNull(other, "other"); + return new PurePosixPrefix(toString(), other.toString()); + } + + /** Converts this path to the equivalent object-store prefix. */ + public PurePosixPrefix toPrefix() { + return PurePosixPrefix.from(this); } public PurePosixPath parent() { - int i = parts.length - 1; - if (i < 0) { - throw new IllegalArgumentException("Path " + this + " has no parent"); + if (tail.length == 0) { + return this; } - if (parts[i].isEmpty()) - --i; - if (parts[i].isEmpty()) - throw new IllegalArgumentException("Path " + this + " has no parent"); - String[] parentParts = Arrays.copyOf(parts, i); - return new PurePosixPath(parentParts); + return new PurePosixPath( + new ParsedPath(root, Arrays.copyOf(tail, tail.length - 1))); } public String name() { - if (parts.length == 0) { - return ""; - } - return parts[parts.length - 1]; + return tail.length == 0 ? "" : tail[tail.length - 1]; } - public PurePosixPath resolve(String other) { - if (other.startsWith("/")) { - return new PurePosixPath(other); - } - return new PurePosixPath(this.toString(), other); + return join(other); } public PurePosixPath resolve(PurePosixPath other) { - return this.resolve(other.toString()); + return join(other); } - public List parts() { - return Collections.unmodifiableList(Arrays.asList(parts)); + return List.of(parts); } public String get(int index) { @@ -129,80 +188,123 @@ public String get(int index) { } public String suffix() { - String name = name(); - int index = name.lastIndexOf("."); - if (index == -1) { - return ""; - } - return name.substring(index); + String name = stripLeadingDots(name()); + int index = name.lastIndexOf('.'); + return index == -1 ? "" : name.substring(index); } public List suffixes() { - String name = name(); - List suffixes = new ArrayList<>(); - int index = name.lastIndexOf("."); - while (index != -1) { - suffixes.add(name.substring(index)); - name = name.substring(0, index); - index = name.lastIndexOf("."); + String name = stripLeadingDots(name()); + String[] split = name.split("\\.", -1); + if (split.length <= 1) { + return List.of(); } - Collections.reverse(suffixes); - return suffixes; + List suffixes = new ArrayList<>(split.length - 1); + for (int i = 1; i < split.length; i++) { + suffixes.add("." + split[i]); + } + return List.copyOf(suffixes); } public String stem() { String name = name(); - int index = name.lastIndexOf("."); - if (index == -1) { - return name; + int index = name.lastIndexOf('.'); + if (index != -1) { + String stem = name.substring(0, index); + if (!stripLeadingDots(stem).isEmpty()) { + return stem; + } + } + return name; + } + + private static String stripLeadingDots(String value) { + int index = 0; + while (index < value.length() && value.charAt(index) == '.') { + index++; } - return name.substring(0, index); + return value.substring(index); } public boolean isAbsolute() { - return parts.length > 0 && parts[0].isEmpty(); + return !root.isEmpty(); } public boolean isRelativeTo(PurePosixPath other) { - if (other.parts.length > parts.length) { + Objects.requireNonNull(other, "other"); + if (!root.equals(other.root) || other.tail.length > tail.length) { return false; } - for (int i = 0; i < other.parts.length; i++) { - if (!other.parts[i].equals(parts[i])) { + for (int i = 0; i < other.tail.length; i++) { + if (!tail[i].equals(other.tail[i])) { return false; } } return true; } - @Override public String toString() { - if (parts.length == 0) { - return ""; + if (tail.length == 0) { + return root.isEmpty() ? "." : root; } - return String.join("/", parts); + return root + String.join(SEP, tail); } @Override - public boolean equals(Object o) { - if (this == o) + public boolean equals(Object other) { + if (this == other) { return true; - if (o == null || getClass() != o.getClass()) + } + if (!(other instanceof PurePosixPath that)) { return false; - PurePosixPath that = (PurePosixPath) o; - return Arrays.equals(parts, that.parts); + } + return root.equals(that.root) && Arrays.equals(tail, that.tail); } @Override public int hashCode() { - return Objects.hash((Object[]) parts); + return toString().hashCode(); } @Override public int compareTo(PurePosixPath other) { - String thisStr = this.toString(); - String otherStr = other.toString(); - return thisStr.compareTo(otherStr); + Objects.requireNonNull(other, "other"); + String[] left = toString().split(SEP, -1); + String[] right = other.toString().split(SEP, -1); + int shared = Math.min(left.length, right.length); + for (int i = 0; i < shared; i++) { + int comparison = compareByCodePoint(left[i], right[i]); + if (comparison != 0) { + return comparison; + } + } + return Integer.compare(left.length, right.length); + } + + private static int compareByCodePoint(String left, String right) { + int leftOffset = 0; + int rightOffset = 0; + while (leftOffset < left.length() && rightOffset < right.length()) { + int leftCodePoint = left.codePointAt(leftOffset); + int rightCodePoint = right.codePointAt(rightOffset); + if (leftCodePoint != rightCodePoint) { + return Integer.compare(leftCodePoint, rightCodePoint); + } + leftOffset += Character.charCount(leftCodePoint); + rightOffset += Character.charCount(rightCodePoint); + } + return Integer.compare(left.length() - leftOffset, right.length() - rightOffset); + } + + private record ParsedPath(String root, String[] tail) { + private ParsedPath { + tail = tail.clone(); + } + + @Override + public String[] tail() { + return tail.clone(); + } } } diff --git a/java/src/main/java/com/esamtrade/bucketbase/PurePosixPrefix.java b/java/src/main/java/com/esamtrade/bucketbase/PurePosixPrefix.java new file mode 100644 index 0000000..4048c58 --- /dev/null +++ b/java/src/main/java/com/esamtrade/bucketbase/PurePosixPrefix.java @@ -0,0 +1,189 @@ +package com.esamtrade.bucketbase; + +import org.apache.commons.io.FilenameUtils; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * Immutable, purely lexical POSIX prefix for object-store operations. + * + *

A prefix differs from {@link PurePosixPath} in two intentional ways: the empty prefix + * renders as {@code ""}, and a trailing {@code "/"} is preserved. Consequently, + * {@code new PurePosixPrefix("dir/")} and {@code new PurePosixPrefix("dir")} select different + * sets of object keys. Redundant separators and {@code "."} components are otherwise normalized + * with the same lexical rules as {@link PurePosixPath}, while {@code ".."} remains lexical.

+ * + *

Conversions are explicit: {@link #toPath()} discards prefix-only trailing-separator + * information, while {@link PurePosixPath#toPrefix()} creates a prefix without inventing a + * trailing separator.

+ */ +public final class PurePosixPrefix { + + public static final String SEP = "/"; + + private final PurePosixPath path; + private final boolean trailingSeparator; + + public static PurePosixPrefix from(String prefix, String... more) { + return new PurePosixPrefix(prefix, more); + } + + public static PurePosixPrefix from(Path prefix) { + Objects.requireNonNull(prefix, "prefix"); + return new PurePosixPrefix(FilenameUtils.separatorsToUnix(prefix.toString())); + } + + public static PurePosixPrefix from(PurePosixPath path) { + return new PurePosixPrefix(path); + } + + public PurePosixPrefix() { + this(""); + } + + public PurePosixPrefix(String first, String... more) { + Objects.requireNonNull(first, "first"); + Objects.requireNonNull(more, "more"); + + String[] prefixes = new String[more.length + 1]; + prefixes[0] = first; + System.arraycopy(more, 0, prefixes, 1, more.length); + String raw = joinRaw(prefixes); + PurePosixPath normalized = new PurePosixPath(raw); + this.path = normalized; + this.trailingSeparator = !isEmptyPath(normalized) + && raw.endsWith(SEP) + && !normalized.toString().endsWith(SEP); + } + + public PurePosixPrefix(PurePosixPath path) { + this(Objects.requireNonNull(path, "path"), false); + } + + private PurePosixPrefix(PurePosixPath path, boolean trailingSeparator) { + this.path = path; + this.trailingSeparator = trailingSeparator + && !isEmptyPath(path) + && !path.toString().endsWith(SEP); + } + + private static String joinRaw(String[] values) { + String result = ""; + for (String value : values) { + Objects.requireNonNull(value, "prefix segment"); + if (value.startsWith(SEP)) { + result = value; + } else if (result.isEmpty() || result.endsWith(SEP)) { + result += value; + } else { + result += SEP + value; + } + } + return result; + } + + private static boolean isEmptyPath(PurePosixPath path) { + return path.parts().isEmpty(); + } + + /** Joins textual path components to this prefix and returns an object path. */ + public PurePosixPath join(String other, String... more) { + Objects.requireNonNull(other, "other"); + Objects.requireNonNull(more, "more"); + String[] values = new String[more.length + 2]; + values[0] = toString(); + values[1] = other; + System.arraycopy(more, 0, values, 2, more.length); + return new PurePosixPath(values); + } + + /** + * Joins textual prefix components and returns a prefix, preserving a trailing separator in + * the last component. + */ + public PurePosixPrefix joinPrefix(String other, String... more) { + Objects.requireNonNull(other, "other"); + Objects.requireNonNull(more, "more"); + String[] values = new String[more.length + 2]; + values[0] = toString(); + values[1] = other; + System.arraycopy(more, 0, values, 2, more.length); + return new PurePosixPrefix(joinRaw(values)); + } + + /** Joins an object path to this prefix and returns an object path. */ + public PurePosixPath join(PurePosixPath other) { + Objects.requireNonNull(other, "other"); + return new PurePosixPath(toString(), other.toString()); + } + + /** Joins another prefix and returns a prefix. */ + public PurePosixPrefix join(PurePosixPrefix other) { + Objects.requireNonNull(other, "other"); + return new PurePosixPrefix(toString(), other.toString()); + } + + /** + * Converts this prefix to a path. An empty prefix becomes the empty path ({@code "."}), and + * any trailing separator is intentionally discarded. + */ + public PurePosixPath toPath() { + return path; + } + + public boolean isEmpty() { + return isEmptyPath(path); + } + + public boolean hasTrailingSeparator() { + return trailingSeparator; + } + + public boolean isAbsolute() { + return path.isAbsolute(); + } + + public String name() { + return trailingSeparator || isEmpty() ? "" : path.name(); + } + + public List parts() { + if (!trailingSeparator) { + return path.parts(); + } + List parts = new ArrayList<>(path.parts()); + parts.add(""); + return List.copyOf(parts); + } + + public String get(int index) { + return parts().get(index); + } + + @Override + public String toString() { + if (isEmpty()) { + return ""; + } + return path + (trailingSeparator ? SEP : ""); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof PurePosixPrefix that)) { + return false; + } + return trailingSeparator == that.trailingSeparator && path.equals(that.path); + } + + @Override + public int hashCode() { + return Objects.hash(path, trailingSeparator); + } +} diff --git a/java/src/main/java/com/esamtrade/bucketbase/RangeSeekableInputStream.java b/java/src/main/java/com/esamtrade/bucketbase/RangeSeekableInputStream.java new file mode 100644 index 0000000..f2ef30e --- /dev/null +++ b/java/src/main/java/com/esamtrade/bucketbase/RangeSeekableInputStream.java @@ -0,0 +1,187 @@ +package com.esamtrade.bucketbase; + +import java.io.IOException; +import java.util.Objects; + +/** + * A {@link SeekableInputStream} backed by a {@link RangeReader} that fetches byte ranges on + * demand. Small reads are served from a single prefetched buffer; reads at least as large as + * the buffer bypass it and are filled straight into the caller's array, so a large read never + * allocates a second copy. + * + *

Not thread-safe on its own; all mutating methods are {@code synchronized} so a single + * instance behaves correctly when shared, but callers should not interleave reads and seeks + * from multiple threads expecting a coherent position.

+ */ +final class RangeSeekableInputStream extends SeekableInputStream { + + /** + * Fetches an exact byte range from the backing store into a caller-provided buffer. This is + * the single extension point for a backend: it must fill exactly {@code length} bytes or + * throw. Ranges are always within {@code [0, objectSize)}. + */ + @FunctionalInterface + interface RangeReader { + void readInto(long offset, byte[] destination, int destinationOffset, int length) throws IOException; + } + + private static final byte[] EMPTY_BUFFER = new byte[0]; + + private final long objectSize; + private final int readBufferSize; + private final RangeReader rangeReader; + + private byte[] buffer = EMPTY_BUFFER; + private long bufferOffset; + private long position; + private long markedPosition = -1; + private boolean closed; + + RangeSeekableInputStream(long objectSize, int readBufferSize, RangeReader rangeReader) { + if (objectSize < 0) { + throw new IllegalArgumentException("objectSize must be non-negative"); + } + if (readBufferSize < 0) { + throw new IllegalArgumentException("readBufferSize must be non-negative"); + } + this.objectSize = objectSize; + this.readBufferSize = readBufferSize; + this.rangeReader = Objects.requireNonNull(rangeReader, "rangeReader"); + } + + @Override + public synchronized int read() throws IOException { + byte[] one = new byte[1]; + return read(one, 0, 1) == -1 ? -1 : Byte.toUnsignedInt(one[0]); + } + + @Override + public synchronized int read(byte[] destination, int offset, int length) throws IOException { + Objects.checkFromIndexSize(offset, length, destination.length); + ensureOpen(); + if (length == 0) { + return 0; + } + if (position >= objectSize) { + return -1; + } + + int toRead = (int) Math.min(length, objectSize - position); + // Large read: fill the caller's array directly, no intermediate buffer. + if (readBufferSize == 0 || toRead >= readBufferSize) { + rangeReader.readInto(position, destination, offset, toRead); + position += toRead; + return toRead; + } + // Small read: serve from (and refill) the prefetch buffer. + fillBufferAround(position); + int bufferIndex = (int) (position - bufferOffset); + int fromBuffer = Math.min(toRead, buffer.length - bufferIndex); + System.arraycopy(buffer, bufferIndex, destination, offset, fromBuffer); + position += fromBuffer; + return fromBuffer; + } + + @Override + public synchronized byte[] readAllBytes() throws IOException { + ensureOpen(); + long remaining = Math.max(0, objectSize - position); + if (remaining == 0) { + return EMPTY_BUFFER; + } + if (remaining > Integer.MAX_VALUE) { + throw new IOException("Object is too large to return as one byte array: " + remaining + " bytes"); + } + byte[] data = new byte[(int) remaining]; + rangeReader.readInto(position, data, 0, data.length); + position += remaining; + return data; + } + + @Override + public synchronized long skip(long count) throws IOException { + ensureOpen(); + if (count <= 0) { + return 0; + } + long skipped = Math.min(count, objectSize - Math.min(position, objectSize)); + position += skipped; + return skipped; + } + + /** + * Bytes readable without a fetch: what remains in the prefetch buffer ahead of the current + * position. Every other byte requires a blocking range read, so it is not reported here. + */ + @Override + public synchronized int available() throws IOException { + ensureOpen(); + if (position >= bufferOffset && position < bufferOffset + buffer.length) { + return (int) (bufferOffset + buffer.length - position); + } + return 0; + } + + @Override + public synchronized long position() throws IOException { + ensureOpen(); + return position; + } + + @Override + public long size() { + return objectSize; + } + + @Override + public synchronized void seek(long newPosition) throws IOException { + ensureOpen(); + if (newPosition < 0) { + throw new IOException("Cannot seek to a negative position: " + newPosition); + } + position = newPosition; + } + + // mark/reset are cheap on a seekable stream: they just remember and restore the position. + @Override + public boolean markSupported() { + return true; + } + + @Override + public synchronized void mark(int readLimit) { + markedPosition = position; + } + + @Override + public synchronized void reset() throws IOException { + ensureOpen(); + if (markedPosition < 0) { + throw new IOException("reset() called before mark()"); + } + position = markedPosition; + } + + @Override + public synchronized void close() { + closed = true; + buffer = EMPTY_BUFFER; + } + + private void fillBufferAround(long from) throws IOException { + if (from >= bufferOffset && from < bufferOffset + buffer.length) { + return; + } + int length = (int) Math.min(readBufferSize, objectSize - from); + byte[] refilled = new byte[length]; + rangeReader.readInto(from, refilled, 0, length); + buffer = refilled; + bufferOffset = from; + } + + private void ensureOpen() throws IOException { + if (closed) { + throw new IOException("Stream is closed"); + } + } +} diff --git a/java/src/main/java/com/esamtrade/bucketbase/S3Bucket.java b/java/src/main/java/com/esamtrade/bucketbase/S3Bucket.java index f50cfea..417268e 100644 --- a/java/src/main/java/com/esamtrade/bucketbase/S3Bucket.java +++ b/java/src/main/java/com/esamtrade/bucketbase/S3Bucket.java @@ -1,11 +1,10 @@ package com.esamtrade.bucketbase; -import org.apache.commons.codec.digest.DigestUtils; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.core.sync.RequestBody; -import software.amazon.awssdk.core.sync.ResponseTransformer; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3Client; @@ -16,28 +15,45 @@ import java.io.InputStream; import java.net.URI; import java.util.ArrayList; -import java.util.Base64; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.concurrent.CompletableFuture; -import java.util.stream.Collectors; -public class S3Bucket extends BaseBucket { +public class S3Bucket implements IBucket { + public static final int DEFAULT_READ_BUFFER_SIZE = 128 * 1024; + private static final int DELETE_BATCH_SIZE = 1000; protected S3Client s3Client; protected S3AsyncClient s3AsyncClient; protected String bucketName; - + protected int readBufferSize; + /** Whether this instance created the clients (and must therefore close them). */ + private final boolean ownsClients; public S3Bucket(S3Client s3Client, S3AsyncClient s3AsyncClient, String bucketName) { + this(s3Client, s3AsyncClient, bucketName, DEFAULT_READ_BUFFER_SIZE); + } + + public S3Bucket(S3Client s3Client, S3AsyncClient s3AsyncClient, String bucketName, int readBufferSize) { + this(s3Client, s3AsyncClient, bucketName, readBufferSize, false); + } + + private S3Bucket(S3Client s3Client, S3AsyncClient s3AsyncClient, String bucketName, int readBufferSize, boolean ownsClients) { + if (readBufferSize < 0) { + throw new IllegalArgumentException("readBufferSize must be non-negative"); + } this.s3Client = s3Client; this.s3AsyncClient = s3AsyncClient; this.bucketName = bucketName; + this.readBufferSize = readBufferSize; + this.ownsClients = ownsClients; } public S3Bucket(String endpoint, String accessKey, String secretKey, String bucketName) { - this(createS3Client(endpoint, accessKey, secretKey), createS3AsyncClient(endpoint, accessKey, secretKey), bucketName); + this(endpoint, accessKey, secretKey, bucketName, DEFAULT_READ_BUFFER_SIZE); + } + + public S3Bucket(String endpoint, String accessKey, String secretKey, String bucketName, int readBufferSize) { + this(createS3Client(endpoint, accessKey, secretKey), createS3AsyncClient(endpoint, accessKey, secretKey), bucketName, readBufferSize, true); } private static S3Client createS3Client(String endpoint, String accessKey, String secretKey) { @@ -62,31 +78,24 @@ private static S3AsyncClient createS3AsyncClient(String endpoint, String accessK @Override public void putObject(PurePosixPath name, byte[] content) { + String _name = S3Names.validateName(name); PutObjectRequest request = PutObjectRequest.builder() .bucket(bucketName) - .key(name.toString()) + .key(_name) .build(); s3Client.putObject(request, RequestBody.fromBytes(content)); } @Override - public void putObjectStream(PurePosixPath name, InputStream stream) { - String _name = validateName(name); - try { - uploadLargeStream(_name, stream); - } catch (Exception e) { - throw new RuntimeException("Failed to upload object: " + _name, e); - } finally { - s3AsyncClient.close(); - } + public void putObjectStream(PurePosixPath name, InputStream stream) throws IOException { + String _name = S3Names.validateName(name); + uploadLargeStream(_name, stream); } - - private void uploadLargeStream(String key, InputStream inputStream) { - int partSize = 5 * 1024 * 1024; // 5 MB + private void uploadLargeStream(String key, InputStream inputStream) throws IOException { + // S3 requires every part except the last to be at least 5 MiB. + final int partSize = 5 * 1024 * 1024; List completedParts = new ArrayList<>(); - byte[] buffer = new byte[partSize]; - int bytesRead; int partNumber = 1; // 1. Initiate the multipart upload @@ -98,23 +107,34 @@ private void uploadLargeStream(String key, InputStream inputStream) { String uploadId = response.uploadId(); try { - // 2. Read the input stream and upload each part - while ((bytesRead = inputStream.read(buffer)) != -1) { - byte[] bytesToUpload = (bytesRead < partSize) ? java.util.Arrays.copyOf(buffer, bytesRead) : buffer; - UploadPartRequest uploadPartRequest = UploadPartRequest.builder() - .bucket(bucketName) - .key(key) - .uploadId(uploadId) - .partNumber(partNumber) - .contentLength((long) bytesRead) - .build(); - AsyncRequestBody requestBody = AsyncRequestBody.fromBytes(bytesToUpload); - CompletableFuture uploadPartResponse = s3AsyncClient.uploadPart(uploadPartRequest, requestBody); - completedParts.add(CompletedPart.builder() - .partNumber(partNumber) - .eTag(uploadPartResponse.join().eTag()) - .build()); - partNumber++; + // 2. Read the input stream and upload each part. + // readNBytes fills the buffer across however many short reads the source needs - + // a plain read(buffer) may legally return far less than partSize without being at + // EOF, which would produce undersized parts and an EntityTooSmall failure. + while (true) { + byte[] part = inputStream.readNBytes(partSize); + boolean lastPart = part.length < partSize; + // Skip a trailing empty read, but keep a single empty part for an empty object: + // completeMultipartUpload rejects an upload with no parts at all. + if (part.length > 0 || completedParts.isEmpty()) { + UploadPartRequest uploadPartRequest = UploadPartRequest.builder() + .bucket(bucketName) + .key(key) + .uploadId(uploadId) + .partNumber(partNumber) + .contentLength((long) part.length) + .build(); + AsyncRequestBody requestBody = AsyncRequestBody.fromBytes(part); + CompletableFuture uploadPartResponse = s3AsyncClient.uploadPart(uploadPartRequest, requestBody); + completedParts.add(CompletedPart.builder() + .partNumber(partNumber) + .eTag(uploadPartResponse.join().eTag()) + .build()); + partNumber++; + } + if (lastPart) { + break; + } } // 3. Complete the multipart upload @@ -126,42 +146,91 @@ private void uploadLargeStream(String key, InputStream inputStream) { .build(); s3AsyncClient.completeMultipartUpload(completeMultipartUploadRequest).join(); } catch (Exception e) { - // Abort the multipart upload in case of failure - AbortMultipartUploadRequest abortMultipartUploadRequest = AbortMultipartUploadRequest.builder() + abortQuietly(key, uploadId, e); + throw new IOException("Failed to upload object: " + key, e); + } + } + + /** + * Aborts a multipart upload without letting the abort failure hide the original cause. + */ + private void abortQuietly(String key, String uploadId, Exception primary) { + try { + s3AsyncClient.abortMultipartUpload(AbortMultipartUploadRequest.builder() .bucket(bucketName) .key(key) .uploadId(uploadId) - .build(); - s3AsyncClient.abortMultipartUpload(abortMultipartUploadRequest).join(); - throw new RuntimeException("Failed to upload object: " + key, e); + .build()).join(); + } catch (Exception abortFailure) { + primary.addSuppressed(abortFailure); } } @Override public byte[] getObject(PurePosixPath name) throws IOException { - try { - GetObjectRequest request = GetObjectRequest.builder() - .bucket(bucketName) - .key(name.toString()) - .build(); - return s3Client.getObjectAsBytes(request).asByteArray(); - } catch (NoSuchKeyException e) { - throw new FileNotFoundException("Object " + name + " not found in S3 bucket " + bucketName); + try (SeekableInputStream stream = getObjectStream(name)) { + return stream.readAllBytes(); } } @Override - public ObjectStream getObjectStream(PurePosixPath name) throws IOException { + public SeekableInputStream getObjectStream(PurePosixPath name) throws IOException { + String objectName = S3Names.validateName(name); + HeadObjectResponse metadata = headObject(objectName); + return new RangeSeekableInputStream( + contentLengthOf(metadata, objectName), + readBufferSize, + (offset, dest, destOffset, length) -> readRange(objectName, metadata.eTag(), offset, dest, destOffset, length) + ); + } + + @Override + public long getSize(PurePosixPath name) throws IOException { + String objectName = S3Names.validateName(name); + return contentLengthOf(headObject(objectName), objectName); + } + + private HeadObjectResponse headObject(String objectName) throws IOException { try { - GetObjectRequest request = GetObjectRequest.builder() + return s3Client.headObject(HeadObjectRequest.builder() .bucket(bucketName) - .key(name.toString()) - .build(); - InputStream inputStream = s3Client.getObject(request, ResponseTransformer.toInputStream()); - return new ObjectStream(inputStream, name.toString()); - } catch (NoSuchKeyException e) { - throw new FileNotFoundException("Object " + name + " not found in S3 bucket " + bucketName); + .key(objectName) + .build()); + } catch (S3Exception e) { + // HEAD has no response body, so many S3-compatible servers (MinIO included) return a + // bare 404 that does not deserialize into NoSuchKeyException. Match on the status. + if (e.statusCode() == 404) { + throw new FileNotFoundException("Object " + objectName + " not found in S3 bucket " + bucketName); + } + throw e; + } + } + + private static long contentLengthOf(HeadObjectResponse metadata, String objectName) throws IOException { + Long contentLength = metadata.contentLength(); + if (contentLength == null) { + throw new IOException("S3 did not report a content length for object " + objectName); + } + return contentLength; + } + + private void readRange(String objectName, String eTag, long offset, byte[] dest, int destOffset, int length) throws IOException { + GetObjectRequest.Builder request = GetObjectRequest.builder() + .bucket(bucketName) + .key(objectName) + .range("bytes=" + offset + "-" + (offset + length - 1)); + if (eTag != null) { + request.ifMatch(eTag); + } + try (ResponseInputStream response = s3Client.getObject(request.build())) { + // Read straight into the caller's buffer - no intermediate copy for large reads. + int read = response.readNBytes(dest, destOffset, length); + if (read != length) { + throw new IOException("Expected " + length + " bytes at offset " + offset + " of " + objectName + ", but received " + read); + } + } catch (S3Exception e) { + throw new IOException("Failed to read object " + objectName + " range at offset " + offset + " with length " + length, e); } } @@ -175,37 +244,37 @@ public ObjectStream getObjectStream(PurePosixPath name) throws IOException { * @param prefix The prefix to filter objects by. * @return A list of paths to the objects in the bucket. */ - public List listObjects(PurePosixPath prefix) { - splitPrefix(prefix); // validate prefix - List result = new ArrayList<>(); + public List listObjects(PurePosixPrefix prefix) { + String validated = S3Names.validatePrefix(prefix); ListObjectsV2Request request = ListObjectsV2Request.builder() .bucket(bucketName) - .prefix(prefix.toString()) + .prefix(validated) .build(); - - List results = s3Client.listObjectsV2Paginator(request).contents().stream().map(S3Object::key).map(PurePosixPath::from).toList(); - - return results; + // The paginator transparently follows continuation tokens across all pages. + return s3Client.listObjectsV2Paginator(request).contents().stream() + .map(S3Object::key) + .map(PurePosixPath::from) + .collect(java.util.stream.Collectors.toList()); } @Override - public ShallowListing shallowListObjects(PurePosixPath prefix) { - splitPrefix(prefix); // validate prefix + public ShallowListing shallowListObjects(PurePosixPrefix prefix) { + String validated = S3Names.validatePrefix(prefix); List objects = new ArrayList<>(); - List prefixes = new ArrayList<>(); + List prefixes = new ArrayList<>(); ListObjectsV2Request request = ListObjectsV2Request.builder() .bucket(bucketName) - .prefix(prefix.toString()) - .delimiter(SEP) + .prefix(validated) + .delimiter(PurePosixPrefix.SEP) .build(); - s3Client.listObjectsV2Paginator(request).stream().forEach(response -> { + s3Client.listObjectsV2Paginator(request).forEach(response -> { for (S3Object object : response.contents()) { - objects.add(new PurePosixPath(object.key())); + objects.add(PurePosixPath.from(object.key())); } for (CommonPrefix commonPrefix : response.commonPrefixes()) { - prefixes.add(new PurePosixPath(commonPrefix.prefix())); + prefixes.add(PurePosixPrefix.from(commonPrefix.prefix())); } }); @@ -214,111 +283,63 @@ public ShallowListing shallowListObjects(PurePosixPath prefix) { @Override public boolean exists(PurePosixPath name) { - String _name = validateName(name); + String _name = S3Names.validateName(name); try { - HeadObjectRequest request = HeadObjectRequest.builder() + s3Client.headObject(HeadObjectRequest.builder() .bucket(bucketName) .key(_name) - .build(); - s3Client.headObject(request); + .build()); return true; - } catch (NoSuchKeyException e) { - return false; + } catch (S3Exception e) { + // A HEAD 404 does not always deserialize into NoSuchKeyException on S3-compatible + // servers, so treat any 404 as "does not exist". + if (e.statusCode() == 404) { + return false; + } + throw e; } } /** - * Removes multiple objects from the S3 bucket. - * - *

**Note:** Amazon S3 requires the `Content-MD5` header for all Multi-Object Delete requests to ensure - * data integrity. When interacting with S3-compatible storage solutions like MinIO, omitting this header - * can result in a `400 Bad Request` error indicating a missing `Content-Md5` header. - * - *

This ensures compatibility with MinIO and similar services that enforce the presence of the `Content-MD5` header. - * More info - * - * @param names List of object paths to be removed. - * @return List of errors encountered during the deletion process. + * Deletes multiple objects in a single request per 1000 keys. Deleting a key that does not + * exist is not an error (consistent with S3). Any keys the server reports as failed - for + * example due to permissions - are returned as {@link DeleteError}s rather than thrown. */ @Override public List removeObjects(List names) { - List validatedNames = names.stream() - .map(BaseBucket::validateName) - .toList(); + List validatedNames = names.stream().map(S3Names::validateName).toList(); List allErrors = new ArrayList<>(); - // Process in batches of 1000 (S3's maximum limit for a single delete operation) - for (int i = 0; i < validatedNames.size(); i += 1000) { - int endIndex = Math.min(i + 1000, validatedNames.size()); - List batch = validatedNames.subList(i, endIndex); - - List batchErrors = removeBatch(batch); - allErrors.addAll(batchErrors); + for (int i = 0; i < validatedNames.size(); i += DELETE_BATCH_SIZE) { + List batch = validatedNames.subList(i, Math.min(i + DELETE_BATCH_SIZE, validatedNames.size())); + allErrors.addAll(removeBatch(batch)); } - return allErrors; } private List removeBatch(List names) { + if (names.isEmpty()) { + return List.of(); + } List keys = names.stream() - .map(name -> ObjectIdentifier.builder() - .key(name) - .build()) - .collect(Collectors.toList()); - Set namesSet = new HashSet<>(names); - - List errors = new ArrayList<>(); - DeleteObjectsResponse response; - if (!keys.isEmpty()) { - try { - Delete delete = Delete.builder().objects(keys).build(); - - DeleteObjectsRequest request = DeleteObjectsRequest.builder() - .bucket(bucketName) - .delete(delete) - .build(); - - response = s3Client.deleteObjects(request); - } catch (S3Exception e) { - if (e.statusCode() == 400 && e.getMessage().contains("Content-Md5")) { - // Explicitly adding the Content-MD5 header for compatibility with MinIO, so manually constructing the XML payload - StringBuilder xmlBuilder = new StringBuilder(); - xmlBuilder.append(""); - for (ObjectIdentifier key : keys) { - xmlBuilder.append("") - .append(key.key()) - .append(""); - } - xmlBuilder.append(""); - String xmlPayload = xmlBuilder.toString(); - - // Compute MD5 checksum - byte[] md5Bytes = DigestUtils.md5(xmlPayload); - String contentMd5 = Base64.getEncoder().encodeToString(md5Bytes); - - // Create Delete object - Delete delete = Delete.builder().objects(keys).build(); - - // Create DeleteObjectsRequest with Content-MD5 header - DeleteObjectsRequest request = DeleteObjectsRequest.builder() - .bucket(bucketName) - .delete(delete) - .overrideConfiguration(o -> o.putHeader("Content-MD5", contentMd5)) - .build(); - - // Perform the delete operation - response = s3Client.deleteObjects(request); - } else { - throw e; - } - } + .map(name -> ObjectIdentifier.builder().key(name).build()) + .toList(); + DeleteObjectsResponse response = s3Client.deleteObjects(DeleteObjectsRequest.builder() + .bucket(bucketName) + .delete(Delete.builder().objects(keys).build()) + .build()); + // response.errors() carries the per-key failures; response.deleted() are the successes. + return response.errors().stream() + .map(error -> new DeleteError(error.code(), error.message(), error.key())) + .collect(java.util.stream.Collectors.toList()); + } - // Process the response - for (DeletedObject deleted : response.deleted()) { - if (!namesSet.contains(deleted.key())) { - errors.add(new DeleteError("Object not found: " + deleted.key())); - } + @Override + public void close() { + if (ownsClients) { + s3Client.close(); + if (s3AsyncClient != null) { + s3AsyncClient.close(); } } - return errors; } -} \ No newline at end of file +} diff --git a/java/src/main/java/com/esamtrade/bucketbase/S3BucketSDKv1.java b/java/src/main/java/com/esamtrade/bucketbase/S3BucketSDKv1.java index f35507e..e1f3076 100644 --- a/java/src/main/java/com/esamtrade/bucketbase/S3BucketSDKv1.java +++ b/java/src/main/java/com/esamtrade/bucketbase/S3BucketSDKv1.java @@ -5,17 +5,23 @@ import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.AbortMultipartUploadRequest; import com.amazonaws.services.s3.model.AmazonS3Exception; +import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; import com.amazonaws.services.s3.model.DeleteObjectsRequest; +import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; +import com.amazonaws.services.s3.model.PartETag; +import com.amazonaws.services.s3.model.UploadPartRequest; import com.amazonaws.services.s3.model.DeleteObjectsResult; +import com.amazonaws.services.s3.model.GetObjectRequest; import com.amazonaws.services.s3.model.ListObjectsV2Request; import com.amazonaws.services.s3.model.ListObjectsV2Result; +import com.amazonaws.services.s3.model.MultiObjectDeleteException; import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectInputStream; import com.amazonaws.services.s3.model.S3ObjectSummary; -import com.amazonaws.util.IOUtils; import java.io.ByteArrayInputStream; import java.io.FileNotFoundException; @@ -23,63 +29,187 @@ import java.io.InputStream; import java.util.ArrayList; import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; /** * S3BucketSDKv1 is a class that provides methods to interact with an S3 bucket. - * It extends the BaseBucket class and uses the old AWS SDK v1 for Java + * It uses the old AWS SDK v1 for Java, kept for compatibility with SDK-v1 consumers. */ -public class S3BucketSDKv1 extends BaseBucket { +public class S3BucketSDKv1 implements IBucket { + public static final int DEFAULT_READ_BUFFER_SIZE = 128 * 1024; + + private static final int DELETE_BATCH_SIZE = 1000; + /** S3 requires every part except the last to be at least 5 MiB. */ + private static final int PART_SIZE = 5 * 1024 * 1024; protected AmazonS3 s3Client; protected String bucketName; + protected int readBufferSize; + /** Whether this instance created the client (and must therefore shut it down). */ + private boolean ownsClient; public S3BucketSDKv1(String endpoint, String accessKey, String secretKey, String bucketName) { + this(endpoint, accessKey, secretKey, bucketName, DEFAULT_READ_BUFFER_SIZE); + } + + /** Default signing region, matching {@link S3Bucket}. S3-compatible services generally ignore it. */ + public static final String DEFAULT_REGION = "us-east-1"; + + public S3BucketSDKv1(String endpoint, String accessKey, String secretKey, String bucketName, int readBufferSize) { + this(endpoint, accessKey, secretKey, bucketName, readBufferSize, DEFAULT_REGION); + } + + public S3BucketSDKv1(String endpoint, String accessKey, String secretKey, String bucketName, int readBufferSize, String region) { BasicAWSCredentials awsCreds = new BasicAWSCredentials(accessKey, secretKey); - this.s3Client = AmazonS3ClientBuilder.standard() - .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, "")) + // The signing region must be a real region name: an empty string produces a signature + // that S3-compatible servers reject with AuthorizationHeaderMalformed. + AmazonS3 client = AmazonS3ClientBuilder.standard() + .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, region)) .withCredentials(new AWSStaticCredentialsProvider(awsCreds)) .withPathStyleAccessEnabled(true) .build(); + initialize(client, bucketName, readBufferSize, true); + } + + public S3BucketSDKv1(AmazonS3 s3Client, String bucketName) { + this(s3Client, bucketName, DEFAULT_READ_BUFFER_SIZE); + } + + public S3BucketSDKv1(AmazonS3 s3Client, String bucketName, int readBufferSize) { + initialize(s3Client, bucketName, readBufferSize, false); + } + + private void initialize(AmazonS3 s3Client, String bucketName, int readBufferSize, boolean ownsClient) { + if (readBufferSize < 0) { + throw new IllegalArgumentException("readBufferSize must be non-negative"); + } + this.s3Client = s3Client; this.bucketName = bucketName; + this.readBufferSize = readBufferSize; + this.ownsClient = ownsClient; } @Override public void putObject(PurePosixPath name, byte[] content) { + String _name = S3Names.validateName(name); ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(content.length); - s3Client.putObject(bucketName, name.toString(), new ByteArrayInputStream(content), metadata); + s3Client.putObject(bucketName, _name, new ByteArrayInputStream(content), metadata); } @Override - public void putObjectStream(PurePosixPath name, InputStream stream) { - ObjectMetadata metadata = new ObjectMetadata(); - s3Client.putObject(bucketName, name.toString(), stream, metadata); + public void putObjectStream(PurePosixPath name, InputStream stream) throws IOException { + String _name = S3Names.validateName(name); + uploadLargeStream(_name, stream); + } + + /** + * Uploads a stream of unknown length as a multipart upload. + * + *

Passing the stream to {@code putObject} without a content length would make the SDK read + * the whole thing into memory just to measure it, which defeats streaming entirely.

+ */ + private void uploadLargeStream(String key, InputStream inputStream) throws IOException { + List partETags = new ArrayList<>(); + String uploadId = s3Client.initiateMultipartUpload( + new InitiateMultipartUploadRequest(bucketName, key)).getUploadId(); + try { + int partNumber = 1; + while (true) { + // readNBytes coalesces short reads into a full part - a queue-backed sink hands + // over whatever the caller wrote, and S3 rejects non-final parts under 5 MiB. + byte[] part = inputStream.readNBytes(PART_SIZE); + boolean lastPart = part.length < PART_SIZE; + // Skip a trailing empty read, but keep a single empty part for an empty object: + // completeMultipartUpload rejects an upload with no parts at all. + if (part.length > 0 || partETags.isEmpty()) { + UploadPartRequest request = new UploadPartRequest() + .withBucketName(bucketName) + .withKey(key) + .withUploadId(uploadId) + .withPartNumber(partNumber++) + .withInputStream(new ByteArrayInputStream(part)) + .withPartSize(part.length) + .withLastPart(lastPart); + partETags.add(s3Client.uploadPart(request).getPartETag()); + } + if (lastPart) { + break; + } + } + s3Client.completeMultipartUpload( + new CompleteMultipartUploadRequest(bucketName, key, uploadId, partETags)); + } catch (Exception e) { + abortQuietly(key, uploadId, e); + throw new IOException("Failed to upload object: " + key, e); + } + } + + /** Aborts a multipart upload without letting the abort failure hide the original cause. */ + private void abortQuietly(String key, String uploadId, Exception primary) { + try { + s3Client.abortMultipartUpload(new AbortMultipartUploadRequest(bucketName, key, uploadId)); + } catch (Exception abortFailure) { + primary.addSuppressed(abortFailure); + } } @Override public byte[] getObject(PurePosixPath name) throws IOException { - S3Object s3Object; + try (SeekableInputStream stream = getObjectStream(name)) { + return stream.readAllBytes(); + } + } + + @Override + public SeekableInputStream getObjectStream(PurePosixPath name) throws IOException { + String objectName = S3Names.validateName(name); try { - s3Object = s3Client.getObject(bucketName, name.toString()); + ObjectMetadata metadata = s3Client.getObjectMetadata(bucketName, objectName); + return new RangeSeekableInputStream( + metadata.getContentLength(), + readBufferSize, + (offset, dest, destOffset, length) -> readRange(objectName, metadata.getETag(), offset, dest, destOffset, length) + ); } catch (AmazonS3Exception e) { + if (e.getStatusCode() != 404) { + throw e; + } throw new FileNotFoundException("Object " + name + " not found in S3 bucket " + bucketName); } - S3ObjectInputStream inputStream = s3Object.getObjectContent(); - return IOUtils.toByteArray(inputStream); } @Override - public ObjectStream getObjectStream(PurePosixPath name) throws IOException { - S3Object s3Object; + public long getSize(PurePosixPath name) throws IOException { + String objectName = S3Names.validateName(name); try { - s3Object = s3Client.getObject(bucketName, name.toString()); + return s3Client.getObjectMetadata(bucketName, objectName).getContentLength(); } catch (AmazonS3Exception e) { + if (e.getStatusCode() != 404) { + throw e; + } throw new FileNotFoundException("Object " + name + " not found in S3 bucket " + bucketName); } - S3ObjectInputStream inputStream = s3Object.getObjectContent(); - return new ObjectStream(inputStream, name.toString()); + } + + private void readRange(String objectName, String eTag, long offset, byte[] dest, int destOffset, int length) throws IOException { + GetObjectRequest request = new GetObjectRequest(bucketName, objectName).withRange(offset, offset + length - 1); + if (eTag != null) { + request.withMatchingETagConstraint(eTag); + } + // SDK v1 returns null (rather than throwing) when a precondition such as + // withMatchingETagConstraint is not met, so this must be checked before dereferencing. + S3Object object = s3Client.getObject(request); + if (object == null) { + throw new IOException("Object " + objectName + " changed while the stream was open"); + } + try (S3Object closeable = object; S3ObjectInputStream inputStream = closeable.getObjectContent()) { + int read = inputStream.readNBytes(dest, destOffset, length); + if (read != length) { + throw new IOException("Expected " + length + " bytes at offset " + offset + " of " + objectName + ", but received " + read); + } + } catch (AmazonS3Exception e) { + throw new IOException("Failed to read object " + objectName + " range at offset " + offset + " with length " + length, e); + } } @@ -92,13 +222,13 @@ public ObjectStream getObjectStream(PurePosixPath name) throws IOException { * @return a list of PurePosixPath objects representing the matching objects */ @Override - public List listObjects(PurePosixPath prefix) { - splitPrefix(prefix); // validate prefix + public List listObjects(PurePosixPrefix prefix) { + String validated = S3Names.validatePrefix(prefix); List result = new ArrayList<>(); - ObjectListing objectListing = s3Client.listObjects(bucketName, prefix.toString()); + ObjectListing objectListing = s3Client.listObjects(bucketName, validated); while (true) { for (S3ObjectSummary summary : objectListing.getObjectSummaries()) { - result.add(new PurePosixPath(summary.getKey())); + result.add(PurePosixPath.from(summary.getKey())); } if (!objectListing.isTruncated()) { break; @@ -109,18 +239,18 @@ public List listObjects(PurePosixPath prefix) { } @Override - public ShallowListing shallowListObjects(PurePosixPath prefix) { - splitPrefix(prefix); // validate prefix + public ShallowListing shallowListObjects(PurePosixPrefix prefix) { + String validated = S3Names.validatePrefix(prefix); List objects = new ArrayList<>(); - List prefixes = new ArrayList<>(); - ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(bucketName).withPrefix(prefix.toString()).withDelimiter(SEP); + List prefixes = new ArrayList<>(); + ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(bucketName).withPrefix(validated).withDelimiter(PurePosixPrefix.SEP); ListObjectsV2Result result; do { result = s3Client.listObjectsV2(request); for (S3ObjectSummary summary : result.getObjectSummaries()) { - objects.add(new PurePosixPath(summary.getKey())); + objects.add(PurePosixPath.from(summary.getKey())); } - prefixes.addAll(result.getCommonPrefixes().stream().map(PurePosixPath::new).toList()); + prefixes.addAll(result.getCommonPrefixes().stream().map(PurePosixPrefix::from).toList()); request.setContinuationToken(result.getNextContinuationToken()); } while (result.isTruncated()); @@ -129,28 +259,40 @@ public ShallowListing shallowListObjects(PurePosixPath prefix) { @Override public boolean exists(PurePosixPath name) { - String _name = validateName(name); + String _name = S3Names.validateName(name); return s3Client.doesObjectExist(bucketName, _name); } + /** + * Deletes objects in batches of 1000 (S3's per-request limit). Deleting a missing key is not + * an error; keys the server reports as failed are returned as {@link DeleteError}s. A partial + * failure raises {@link MultiObjectDeleteException} in SDK v1, whose per-key errors are + * collected rather than propagated. + */ @Override public List removeObjects(List names) { - Set namesSet = names.stream().map(BaseBucket::validateName).collect(Collectors.toSet()); - List keys = new ArrayList<>(); - for (PurePosixPath name : names) { - keys.add(new DeleteObjectsRequest.KeyVersion(name.toString())); - } + List validated = names.stream().map(S3Names::validateName).toList(); List errors = new ArrayList<>(); - if (!keys.isEmpty()) { - DeleteObjectsRequest request = new DeleteObjectsRequest(bucketName).withKeys(keys); - DeleteObjectsResult result = s3Client.deleteObjects(request); - - for (DeleteObjectsResult.DeletedObject deleted : result.getDeletedObjects()) { - if (!namesSet.contains(deleted.getKey())) { - errors.add(new DeleteError("Object not found: " + deleted.getKey())); + for (int i = 0; i < validated.size(); i += DELETE_BATCH_SIZE) { + List batch = validated.subList(i, Math.min(i + DELETE_BATCH_SIZE, validated.size())); + List keys = batch.stream() + .map(DeleteObjectsRequest.KeyVersion::new) + .toList(); + try { + s3Client.deleteObjects(new DeleteObjectsRequest(bucketName).withKeys(keys).withQuiet(true)); + } catch (MultiObjectDeleteException e) { + for (MultiObjectDeleteException.DeleteError error : e.getErrors()) { + errors.add(new DeleteError(error.getCode(), error.getMessage(), error.getKey())); } } } return errors; } + + @Override + public void close() { + if (ownsClient) { + s3Client.shutdown(); + } + } } diff --git a/java/src/main/java/com/esamtrade/bucketbase/S3Names.java b/java/src/main/java/com/esamtrade/bucketbase/S3Names.java new file mode 100644 index 0000000..6a2b6a5 --- /dev/null +++ b/java/src/main/java/com/esamtrade/bucketbase/S3Names.java @@ -0,0 +1,43 @@ +package com.esamtrade.bucketbase; + +import java.util.regex.Pattern; + +/** + * Object-name and prefix validation shared by every backend. + * + *

The character set mirrors the Python library's {@code S3_NAME_CHARS_NO_SEP} + * ({@link IBucket#S3_NAME_CHARS_NO_SEP}) so keys written by one language are accepted by the + * other. {@code \w} is matched with {@link Pattern#UNICODE_CHARACTER_CLASS} so non-ASCII keys + * (e.g. {@code café.txt}) validate the same way they do under Python 3's Unicode-aware {@code re}.

+ */ +final class S3Names { + + private static final Pattern OBJ_NAME_RE = Pattern.compile( + "^(?:[" + IBucket.S3_NAME_CHARS_NO_SEP + "]+/)*[" + IBucket.S3_NAME_CHARS_NO_SEP + "]+$", + Pattern.UNICODE_CHARACTER_CLASS); + private static final Pattern PREFIX_RE = Pattern.compile( + "^(?:[" + IBucket.S3_NAME_CHARS_NO_SEP + "]+/)*[" + IBucket.S3_NAME_CHARS_NO_SEP + "]*$", + Pattern.UNICODE_CHARACTER_CLASS); + + private S3Names() { + } + + static String validateName(PurePosixPath name) { + return validateName(name.toString()); + } + + static String validateName(String name) { + if (!OBJ_NAME_RE.matcher(name).matches()) { + throw new IllegalArgumentException("Invalid S3 object name: " + name); + } + return name; + } + + static String validatePrefix(PurePosixPrefix prefix) { + String value = prefix.toString(); + if (!PREFIX_RE.matcher(value).matches()) { + throw new IllegalArgumentException("Invalid S3 prefix: " + value); + } + return value; + } +} diff --git a/java/src/main/java/com/esamtrade/bucketbase/SeekableInputStream.java b/java/src/main/java/com/esamtrade/bucketbase/SeekableInputStream.java new file mode 100644 index 0000000..dd0e986 --- /dev/null +++ b/java/src/main/java/com/esamtrade/bucketbase/SeekableInputStream.java @@ -0,0 +1,15 @@ +package com.esamtrade.bucketbase; + +import java.io.IOException; +import java.io.InputStream; + +/** + * A readable stream whose position can be changed without reopening the object. + */ +public abstract class SeekableInputStream extends InputStream { + public abstract long position() throws IOException; + + public abstract long size(); + + public abstract void seek(long position) throws IOException; +} diff --git a/java/src/main/java/com/esamtrade/bucketbase/ShallowListing.java b/java/src/main/java/com/esamtrade/bucketbase/ShallowListing.java index c3ba8d8..1d6cf37 100644 --- a/java/src/main/java/com/esamtrade/bucketbase/ShallowListing.java +++ b/java/src/main/java/com/esamtrade/bucketbase/ShallowListing.java @@ -1,23 +1,18 @@ package com.esamtrade.bucketbase; -import java.nio.file.Path; -import java.util.Collections; import java.util.List; -public class ShallowListing { - private final List objects; - private final List prefixes; +/** + * The result of a non-recursive listing: the objects directly under a prefix, plus the common + * sub-prefixes (the equivalent of sub-directories). + * + * @param objects object keys directly under the prefix + * @param prefixes common child prefixes (each ending with {@code /}) + */ +public record ShallowListing(List objects, List prefixes) { - public ShallowListing(List objects, List prefixes) { - this.objects = Collections.unmodifiableList(objects); - this.prefixes = Collections.unmodifiableList(prefixes); - } - - public List getObjects() { - return objects; - } - - public List getPrefixes() { - return prefixes; + public ShallowListing { + objects = List.copyOf(objects); + prefixes = List.copyOf(prefixes); } } diff --git a/java/src/main/java/com/esamtrade/bucketbase/StreamPipe.java b/java/src/main/java/com/esamtrade/bucketbase/StreamPipe.java new file mode 100644 index 0000000..bce39dd --- /dev/null +++ b/java/src/main/java/com/esamtrade/bucketbase/StreamPipe.java @@ -0,0 +1,163 @@ +package com.esamtrade.bucketbase; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Deque; +import java.util.Objects; + +/** + * A bounded single-producer/single-consumer byte pipe bridging an {@link OutputStream} written by + * the caller to an {@link InputStream} read by a background uploader. + * + *

Unlike {@link java.io.PipedInputStream} it can make the reader fail on demand + * ({@link #abort}), which is what lets an aborted write make the in-flight upload throw instead of + * committing a truncated object. A slow reader back-pressures the writer rather than letting data + * accumulate: at most {@value #MAX_QUEUED_CHUNKS} chunks of {@value #MAX_CHUNK_BYTES} bytes are + * buffered, and a larger write blocks until the reader drains, however big a single + * {@code write(byte[])} call is.

+ * + *

Coordination is a single monitor with blocking {@code wait}/{@code notifyAll} — no timed + * polling.

+ */ +final class StreamPipe { + + /** Largest chunk enqueued at once; caps the copy made per write and bounds queued bytes. */ + private static final int MAX_CHUNK_BYTES = 1 << 20; + /** Chunks buffered before the writer blocks: capacity is MAX_QUEUED_CHUNKS * MAX_CHUNK_BYTES. */ + private static final int MAX_QUEUED_CHUNKS = 2; + private static final byte[] EMPTY = new byte[0]; + + private final Object lock = new Object(); + private final Deque queue = new ArrayDeque<>(); + private boolean writerFinished; // finish() was called: EOF once the queue drains + private boolean readerClosed; // the reader stopped consuming + private Throwable failure; // abort() cause + + private final InputStream input = new PipeInputStream(); + private final OutputStream output = new PipeOutputStream(); + + InputStream inputStream() { + return input; + } + + OutputStream outputStream() { + return output; + } + + /** Clean end of stream: the reader sees EOF and the upload commits normally. */ + void finish() { + synchronized (lock) { + writerFinished = true; + lock.notifyAll(); + } + } + + /** Makes the reader fail with {@code cause}; any queued-but-unsent data is discarded. */ + void abort(Throwable cause) { + synchronized (lock) { + if (failure == null) { + failure = cause; + } + queue.clear(); + lock.notifyAll(); + } + } + + private void awaitOn() throws IOException { + try { + lock.wait(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted while streaming the object body", e); + } + } + + /** Consumer side, handed to {@code putObjectStream} on the uploader thread. */ + private final class PipeInputStream extends InputStream { + private byte[] chunk = EMPTY; + private int pos; + + @Override + public int read() throws IOException { + byte[] one = new byte[1]; + return read(one, 0, 1) == -1 ? -1 : Byte.toUnsignedInt(one[0]); + } + + @Override + public int read(byte[] dest, int off, int len) throws IOException { + Objects.checkFromIndexSize(off, len, dest.length); + if (len == 0) { + return 0; + } + if (pos >= chunk.length && !nextChunk()) { + return -1; + } + int n = Math.min(len, chunk.length - pos); + System.arraycopy(chunk, pos, dest, off, n); + pos += n; + return n; + } + + /** Blocks until a chunk is available; returns false at clean EOF; throws on abort. */ + private boolean nextChunk() throws IOException { + synchronized (lock) { + while (queue.isEmpty() && !writerFinished && failure == null) { + awaitOn(); + } + if (failure != null) { + throw new IOException("Object write was aborted before completion", failure); + } + if (queue.isEmpty()) { // writerFinished with nothing left + return false; + } + chunk = queue.removeFirst(); + pos = 0; + lock.notifyAll(); // a slot freed up for the writer + return true; + } + } + + @Override + public void close() { + synchronized (lock) { + readerClosed = true; + lock.notifyAll(); // wake a writer blocked on back-pressure + } + } + } + + /** Producer side, handed to the caller. */ + private final class PipeOutputStream extends OutputStream { + @Override + public void write(int b) throws IOException { + write(new byte[]{(byte) b}, 0, 1); + } + + @Override + public void write(byte[] src, int off, int len) throws IOException { + Objects.checkFromIndexSize(off, len, src.length); + // Split into bounded chunks so capacity is bounded by bytes buffered, not by the + // number of write() calls: one huge write must be back-pressured like many small ones. + int written = 0; + while (written < len) { + int size = Math.min(MAX_CHUNK_BYTES, len - written); + // Copy: the caller may reuse its buffer as soon as write() returns. + byte[] copy = Arrays.copyOfRange(src, off + written, off + written + size); + synchronized (lock) { + while (queue.size() >= MAX_QUEUED_CHUNKS && !readerClosed && failure == null) { + awaitOn(); + } + if (failure != null || readerClosed) { + throw new IOException("Cannot write: the object stream is closed"); + } + queue.addLast(copy); + lock.notifyAll(); + } + written += size; + } + } + } +} diff --git a/java/src/test/java/com/esamtrade/bucketbase/BucketHardeningTest.java b/java/src/test/java/com/esamtrade/bucketbase/BucketHardeningTest.java new file mode 100644 index 0000000..b9d6cd9 --- /dev/null +++ b/java/src/test/java/com/esamtrade/bucketbase/BucketHardeningTest.java @@ -0,0 +1,176 @@ +package com.esamtrade.bucketbase; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Backend-agnostic correctness fixes, exercised against {@link MemoryBucket} because they hold + * for every backend and need no network. + */ +class BucketHardeningTest { + + private final MemoryBucket bucket = new MemoryBucket(); + + @Test + void getObjectReturnsAnIndependentCopy() throws IOException { + PurePosixPath name = PurePosixPath.from("dir/file.bin"); + byte[] original = {1, 2, 3}; + bucket.putObject(name, original); + + // Mutating the array we passed in must not change stored bytes. + original[0] = 99; + assertArrayEquals(new byte[]{1, 2, 3}, bucket.getObject(name)); + + // Mutating the array we got back must not change stored bytes either. + byte[] fetched = bucket.getObject(name); + fetched[0] = 42; + assertArrayEquals(new byte[]{1, 2, 3}, bucket.getObject(name)); + } + + @Test + void removeObjectsIsAtomicOnInvalidName() throws IOException { + PurePosixPath valid = PurePosixPath.from("dir/keep.bin"); + bucket.putObject(valid, new byte[]{1}); + + // A batch containing an invalid name must be rejected wholesale, not applied halfway. + assertThrows(IllegalArgumentException.class, + () -> bucket.removeObjects(List.of(valid, new PurePosixPath("bad*")))); + assertTrue(bucket.exists(valid), "a rejected batch must not have deleted the valid key"); + } + + @Test + void unicodeAndApostropheNamesAreAccepted() throws IOException { + // Matches the Python library's allowed character set, so keys interoperate across languages. + for (String key : List.of("dir/café.txt", "dir/файл.bin", "dir/o'brien.txt", "dir/日本語.dat")) { + PurePosixPath name = PurePosixPath.from(key); + bucket.putObject(name, key.getBytes()); + assertArrayEquals(key.getBytes(), bucket.getObject(name)); + } + } + + @Test + void invalidNamesAreStillRejected() { + for (String key : List.of("star*1", "at@gmail", "sharp#1", "comma,", "back\\slash")) { + assertThrows(IllegalArgumentException.class, + () -> bucket.putObject(PurePosixPath.from(key), new byte[0]), + "expected rejection of: " + key); + } + } + + @Test + void removeObjectsReportsNoErrorsForMissingKeys() throws IOException { + // Deleting a key that does not exist is not an error, consistent with S3. + List errors = bucket.removeObjects(List.of(PurePosixPath.from("dir/ghost.bin"))); + assertEquals(List.of(), errors); + } + + @Test + void copyPrefixWithEmptySourceIsANoOp() throws IOException { + MemoryBucket dst = new MemoryBucket(); + // Must not throw (the old code created a zero-size thread pool). + bucket.copyPrefix(dst, PurePosixPrefix.from("nothing-here/"), PurePosixPrefix.from("out/"), 4); + assertTrue(dst.listObjects(PurePosixPrefix.from("out/")).isEmpty()); + } + + @Test + void copyPrefixCopiesAllObjects() throws IOException { + bucket.putObject(PurePosixPath.from("src/a.txt"), "a".getBytes()); + bucket.putObject(PurePosixPath.from("src/nested/b.txt"), "b".getBytes()); + MemoryBucket dst = new MemoryBucket(); + + // threads > 1 exercises the executor path. + bucket.copyPrefix(dst, PurePosixPrefix.from("src/"), PurePosixPrefix.from("dst/"), 3); + + assertArrayEquals("a".getBytes(), dst.getObject(PurePosixPath.from("dst/a.txt"))); + assertArrayEquals("b".getBytes(), dst.getObject(PurePosixPath.from("dst/nested/b.txt"))); + assertEquals(2, dst.listObjects(PurePosixPrefix.from("dst/")).size()); + } + + @Test + void movePrefixRemovesTheSource() throws IOException { + bucket.putObject(PurePosixPath.from("src/a.txt"), "a".getBytes()); + MemoryBucket dst = new MemoryBucket(); + + bucket.movePrefix(dst, PurePosixPrefix.from("src/"), PurePosixPrefix.from("dst/"), 1); + + assertArrayEquals("a".getBytes(), dst.getObject(PurePosixPath.from("dst/a.txt"))); + assertTrue(bucket.listObjects(PurePosixPrefix.from("src/")).isEmpty()); + } + + @Test + void copyObjectFromStreamsBetweenBuckets() throws IOException { + bucket.putObject(PurePosixPath.from("src.bin"), "payload".getBytes()); + MemoryBucket dst = new MemoryBucket(); + + dst.copyObjectFrom(bucket, PurePosixPath.from("src.bin"), PurePosixPath.from("dst.bin")); + + assertArrayEquals("payload".getBytes(), dst.getObject(PurePosixPath.from("dst.bin"))); + } + + @Test + void memoryBucketCloseIsANoOp() throws IOException { + bucket.putObject(PurePosixPath.from("a.bin"), new byte[]{1}); + bucket.close(); + // Still usable: close() releases resources but MemoryBucket holds none. + assertTrue(bucket.exists(PurePosixPath.from("a.bin"))); + } + + @Test + void noArgumentListingsUseTheEmptyPrefix() throws IOException { + bucket.putObject(PurePosixPath.from("root.bin"), new byte[]{1}); + bucket.putObject(PurePosixPath.from("dir/nested.bin"), new byte[]{2}); + + assertEquals(2, bucket.listObjects().size()); + assertEquals(List.of(PurePosixPath.from("root.bin")), bucket.shallowListObjects().objects()); + assertEquals(List.of(PurePosixPrefix.from("dir/")), bucket.shallowListObjects().prefixes()); + } + + @Test + void appendOnlyRejectsMutations() throws IOException { + LockRecordingAppendOnly appendOnly = new LockRecordingAppendOnly(bucket); + PurePosixPath name = PurePosixPath.from("dir/once.bin"); + + appendOnly.putObject(name, "first".getBytes()); + assertArrayEquals("first".getBytes(), appendOnly.getObject(name)); + assertTrue(appendOnly.wasLocked()); + + assertThrows(UnsupportedOperationException.class, () -> appendOnly.removeObjects(List.of(name))); + assertThrows(UnsupportedOperationException.class, () -> appendOnly.removePrefix(PurePosixPrefix.from("dir/"))); + // movePrefix must fail fast rather than copy-then-fail. + MemoryBucket dst = new MemoryBucket(); + assertThrows(UnsupportedOperationException.class, + () -> appendOnly.movePrefix( + dst, PurePosixPrefix.from("dir/"), PurePosixPrefix.from("out/"), 1)); + assertFalse(dst.exists(PurePosixPath.from("out/once.bin")), "movePrefix must not have copied anything"); + } + + /** Minimal concrete append-only bucket that records whether a lock was taken. */ + private static final class LockRecordingAppendOnly extends AbstractAppendOnlySynchronizedBucket { + private volatile boolean locked; + + LockRecordingAppendOnly(IBucket base) { + super(base); + } + + boolean wasLocked() { + return locked; + } + + @Override + protected void lockObject(PurePosixPath name) { + locked = true; + } + + @Override + protected void unlockObject(PurePosixPath name) { + } + } +} diff --git a/java/src/test/java/com/esamtrade/bucketbase/BucketParquetDataSource.java b/java/src/test/java/com/esamtrade/bucketbase/BucketParquetDataSource.java new file mode 100644 index 0000000..41e8112 --- /dev/null +++ b/java/src/test/java/com/esamtrade/bucketbase/BucketParquetDataSource.java @@ -0,0 +1,63 @@ +package com.esamtrade.bucketbase; + +import io.trino.parquet.AbstractParquetDataSource; +import io.trino.parquet.ParquetDataSourceId; +import io.trino.parquet.ParquetReaderOptions; + +import java.io.IOException; + +/** + * Adapts a bucketbase object to Trino's {@link io.trino.parquet.ParquetDataSource} so that a + * real Parquet reader can drive the seekable, range-backed stream returned by + * {@link IBucket#getObjectStream(PurePosixPath)}. + * + *

This lives in test scope on purpose: it proves the stream is sufficient for partial reads, + * column projection and predicate pushdown without making bucketbase itself depend on Trino or + * on any Parquet implementation. A shipped adapter belongs in a separate optional module.

+ * + *

Every byte the Parquet reader pulls funnels through {@link #readInternal}, so the counters + * here are an exact account of what the reader actually fetched.

+ */ +public final class BucketParquetDataSource extends AbstractParquetDataSource { + + private final SeekableInputStream stream; + + private long bytesFetched; + private int rangeRequests; + + private BucketParquetDataSource(PurePosixPath name, SeekableInputStream stream, ParquetReaderOptions options) { + // The stream already knows the object size, so this costs no extra request. + super(new ParquetDataSourceId(name.toString()), stream.size(), options); + this.stream = stream; + } + + public static BucketParquetDataSource open(IBucket bucket, PurePosixPath name, ParquetReaderOptions options) throws IOException { + return new BucketParquetDataSource(name, bucket.getObjectStream(name), options); + } + + /** Total bytes the Parquet reader pulled out of the bucket. */ + public long bytesFetched() { + return bytesFetched; + } + + /** Number of distinct ranged reads issued. */ + public int rangeRequests() { + return rangeRequests; + } + + @Override + protected void readInternal(long position, byte[] buffer, int bufferOffset, int bufferLength) throws IOException { + bytesFetched += bufferLength; + rangeRequests++; + stream.seek(position); + int read = stream.readNBytes(buffer, bufferOffset, bufferLength); + if (read != bufferLength) { + throw new IOException("Expected " + bufferLength + " bytes at offset " + position + " but got " + read); + } + } + + @Override + public void close() throws IOException { + stream.close(); + } +} diff --git a/java/src/test/java/com/esamtrade/bucketbase/IBucketTester.java b/java/src/test/java/com/esamtrade/bucketbase/IBucketTester.java index 401513e..adb7cc5 100644 --- a/java/src/test/java/com/esamtrade/bucketbase/IBucketTester.java +++ b/java/src/test/java/com/esamtrade/bucketbase/IBucketTester.java @@ -22,18 +22,18 @@ public class IBucketTester { private static final List INVALID_PREFIXES = List.of("/", "/dir", "star*1", "dir1/a\\file.txt", "at@gmail", "sharp#1", "dollar$1", "comma,"); - private final BaseBucket storage; + private final IBucket storage; private final String uniqueSuffix; private final String PATH_WITH_2025_KEYS = "test-dir-with-2025-keys/"; - public IBucketTester(BaseBucket storage) { + public IBucketTester(IBucket storage) { this.storage = storage; // Generate a unique suffix to be used in the names of dirs and files this.uniqueSuffix = String.format("%08d", System.currentTimeMillis() % 100_000_000); } public void cleanup() throws IOException { - storage.removePrefix(PurePosixPath.from("dir" + uniqueSuffix)); + storage.removePrefix(PurePosixPrefix.from("dir" + uniqueSuffix)); } public void testPutAndGetObject() throws IOException { @@ -85,8 +85,8 @@ public void testPutAndGetObjectStream() throws IOException { ByteArrayInputStream gzippedStream = new ByteArrayInputStream(byteStream.toByteArray()); storage.putObjectStream(path, gzippedStream); - try (ObjectStream file = storage.getObjectStream(path)) { - try (BufferedReader reader = new BufferedReader(new InputStreamReader(new GZIPInputStream(file.getStream())))) { + try (SeekableInputStream file = storage.getObjectStream(path)) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(new GZIPInputStream(file)))) { String[] result = new String[3]; for (int i = 0; i < 3; i++) { result[i] = reader.readLine(); @@ -97,8 +97,8 @@ public void testPutAndGetObjectStream() throws IOException { // String path path = PurePosixPath.from(uniqueDir, "file1.bin"); - try (ObjectStream file = storage.getObjectStream(path)) { - try (BufferedReader reader = new BufferedReader(new InputStreamReader(new GZIPInputStream(file.getStream())))) { + try (SeekableInputStream file = storage.getObjectStream(path)) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(new GZIPInputStream(file)))) { char[] cbuf = new char[100]; int readCount = reader.read(cbuf, 0, 100); String result = new String(cbuf, 0, readCount); @@ -111,13 +111,34 @@ public void testPutAndGetObjectStream() throws IOException { assertThrows(FileNotFoundException.class, () -> storage.getObjectStream(nonExistentPath)); } + public void testGetObjectStreamIsSeekable() throws IOException { + String uniqueDir = "dir" + uniqueSuffix; + PurePosixPath path = PurePosixPath.from(uniqueDir, "seekable.bin"); + byte[] content = "0123456789abcdefghijklmnopqrstuvwxyz".getBytes(); + storage.putObject(path, content); + + try (SeekableInputStream stream = storage.getObjectStream(path)) { + assertEquals(content.length, stream.size()); + assertArrayEquals("0123".getBytes(), stream.readNBytes(4)); + stream.seek(10); + assertArrayEquals("abcd".getBytes(), stream.readNBytes(4)); + stream.seek(stream.position() - 2); + assertArrayEquals("cd".getBytes(), stream.readNBytes(2)); + stream.seek(stream.size() - 4); + assertArrayEquals("wxyz".getBytes(), stream.readNBytes(4)); + stream.seek(stream.size() + 10); + assertEquals(-1, stream.read()); + assertThrows(IOException.class, () -> stream.seek(-1)); + } + } + public void testListObjects() throws IOException { String uniqueDir = "dir" + uniqueSuffix; storage.putObject(PurePosixPath.from(uniqueDir, "file1.txt"), "Content 1".getBytes()); storage.putObject(PurePosixPath.from(uniqueDir, "dir2/file2.txt"), "Content 2".getBytes()); storage.putObject(PurePosixPath.from(uniqueDir + "file1.txt"), "Content 3".getBytes()); - List objects = storage.listObjects(PurePosixPath.from(uniqueDir)).stream().sorted().toList(); + List objects = storage.listObjects(PurePosixPrefix.from(uniqueDir)).stream().sorted().toList(); List expectedObjects = List.of( PurePosixPath.from(uniqueDir, "dir2/file2.txt"), PurePosixPath.from(uniqueDir, "file1.txt"), @@ -125,7 +146,7 @@ public void testListObjects() throws IOException { ); assertEquals(expectedObjects, objects); - objects = storage.listObjects(PurePosixPath.from(uniqueDir + "/")).stream().sorted().toList(); + objects = storage.listObjects(PurePosixPrefix.from(uniqueDir + "/")).stream().sorted().toList(); expectedObjects = List.of( PurePosixPath.from(uniqueDir, "dir2/file2.txt"), PurePosixPath.from(uniqueDir, "file1.txt") @@ -134,7 +155,7 @@ public void testListObjects() throws IOException { // Invalid Prefix cases for (String prefix : INVALID_PREFIXES) { - assertThrows(IllegalArgumentException.class, () -> storage.listObjects(PurePosixPath.from(prefix)), "Invalid prefix: " + prefix); + assertThrows(IllegalArgumentException.class, () -> storage.listObjects(PurePosixPrefix.from(prefix)), "Invalid prefix: " + prefix); } } @@ -146,8 +167,8 @@ public void testListObjectsWithOver1000keys() throws IOException { assertEquals(2025, objects.size()); } - private PurePosixPath ensureDirWith2025Keys() throws IOException { - var pathWith2025Keys = new PurePosixPath(PATH_WITH_2025_KEYS); + private PurePosixPrefix ensureDirWith2025Keys() throws IOException { + var pathWith2025Keys = new PurePosixPrefix(PATH_WITH_2025_KEYS); List existingKeys = storage.listObjects(pathWith2025Keys); if (existingKeys.isEmpty()) { // Create the directory and add 2025 files @@ -156,7 +177,8 @@ private PurePosixPath ensureDirWith2025Keys() throws IOException { customThreadPool.submit(() -> IntStream.range(0, 2025).parallel().forEach(i -> { try { - var path = pathWith2025Keys.join("file" + i + ".txt"); + var path = pathWith2025Keys.join( + new PurePosixPath("file" + i + ".txt")); storage.putObject(path, ("Content " + i).getBytes()); } catch (IOException e) { throw new RuntimeException(e); @@ -178,8 +200,8 @@ public void testShallowListObjectsWithOver1000keys() throws IOException { var pathWith2025Keys = ensureDirWith2025Keys(); ShallowListing objects = storage.shallowListObjects(pathWith2025Keys); - assertEquals(2025, objects.getObjects().size()); - assertEquals(0, objects.getPrefixes().size()); + assertEquals(2025, objects.objects().size()); + assertEquals(0, objects.prefixes().size()); } public void testShallowListObjects() throws IOException { @@ -188,26 +210,24 @@ public void testShallowListObjects() throws IOException { storage.putObject(new PurePosixPath(uniqueDir + "/dir2/file2.txt"), "Content 2".getBytes()); storage.putObject(new PurePosixPath(uniqueDir + "file1.txt"), "Content 3".getBytes()); - assertThrows(IllegalArgumentException.class, () -> storage.shallowListObjects(new PurePosixPath("/"))); - assertThrows(IllegalArgumentException.class, () -> storage.shallowListObjects(new PurePosixPath("/d"))); + assertThrows(IllegalArgumentException.class, () -> storage.shallowListObjects(new PurePosixPrefix("/"))); + assertThrows(IllegalArgumentException.class, () -> storage.shallowListObjects(new PurePosixPrefix("/d"))); - ShallowListing objects = storage.shallowListObjects(new PurePosixPath(uniqueDir + "/")); + ShallowListing objects = storage.shallowListObjects(new PurePosixPrefix(uniqueDir + "/")); List expectedObjects = List.of(PurePosixPath.from(uniqueDir + "/file1.txt")); - List expectedPrefixes = List.of(PurePosixPath.from(uniqueDir + "/dir2/")); - assertIterableEquals(expectedObjects, objects.getObjects()); - assertIterableEquals(expectedPrefixes, objects.getPrefixes()); + List expectedPrefixes = List.of(PurePosixPrefix.from(uniqueDir + "/dir2/")); + assertIterableEquals(expectedObjects, objects.objects()); + assertIterableEquals(expectedPrefixes, objects.prefixes()); - ShallowListing shallowListing = storage.shallowListObjects(new PurePosixPath(uniqueDir)); + ShallowListing shallowListing = storage.shallowListObjects(new PurePosixPrefix(uniqueDir)); expectedObjects = List.of(new PurePosixPath(uniqueDir + "file1.txt")); - expectedPrefixes = List.of(PurePosixPath.from(uniqueDir + "/")); - assertTrue(shallowListing.getObjects() instanceof List); - assertTrue(shallowListing.getPrefixes() instanceof List); - assertIterableEquals(expectedObjects, shallowListing.getObjects()); - assertIterableEquals(expectedPrefixes, shallowListing.getPrefixes()); + expectedPrefixes = List.of(PurePosixPrefix.from(uniqueDir + "/")); + assertIterableEquals(expectedObjects, shallowListing.objects()); + assertIterableEquals(expectedPrefixes, shallowListing.prefixes()); // Invalid Prefix cases for (String prefix : INVALID_PREFIXES) { - assertThrows(IllegalArgumentException.class, () -> storage.shallowListObjects(new PurePosixPath(prefix))); + assertThrows(IllegalArgumentException.class, () -> storage.shallowListObjects(new PurePosixPrefix(prefix))); } } @@ -218,7 +238,7 @@ public void testExists() throws IOException { assertTrue(storage.exists(path)); assertFalse(storage.exists(new PurePosixPath(uniqueDir))); - assertThrows(IllegalArgumentException.class, () -> storage.exists(new PurePosixPath(uniqueDir + "/"))); + assertThrows(IllegalArgumentException.class, () -> storage.exists(new PurePosixPath(uniqueDir + "*"))); } public void testRemoveObjects() throws IOException { @@ -238,11 +258,11 @@ public void testRemoveObjects() throws IOException { assertFalse(storage.exists(path1)); assertFalse(storage.exists(path2)); assertThrows(FileNotFoundException.class, () -> storage.getObject(new PurePosixPath(uniqueDir + "/file1.txt"))); - assertThrows(IllegalArgumentException.class, () -> storage.removeObjects(List.of(new PurePosixPath(uniqueDir + "/")))); + assertThrows(IllegalArgumentException.class, () -> storage.removeObjects(List.of(new PurePosixPath(uniqueDir + "*")))); // Check that the leftover empty directories are also removed, but the bucket may contain leftovers from the other test runs - ShallowListing shallowListing = storage.shallowListObjects(new PurePosixPath("")); - List prefixes = shallowListing.getPrefixes(); - assertFalse(prefixes.contains(uniqueDir + "/")); + ShallowListing shallowListing = storage.shallowListObjects(new PurePosixPrefix("")); + List prefixes = shallowListing.prefixes(); + assertFalse(prefixes.contains(PurePosixPrefix.from(uniqueDir + "/"))); } } diff --git a/java/src/test/java/com/esamtrade/bucketbase/MemoryBucketTest.java b/java/src/test/java/com/esamtrade/bucketbase/MemoryBucketTest.java index 7f056cc..ff0ed92 100644 --- a/java/src/test/java/com/esamtrade/bucketbase/MemoryBucketTest.java +++ b/java/src/test/java/com/esamtrade/bucketbase/MemoryBucketTest.java @@ -31,6 +31,11 @@ void putObjectAndGetObjectStream() throws IOException { tester.testPutAndGetObjectStream(); } + @Test + void getObjectStreamIsSeekable() throws IOException { + tester.testGetObjectStreamIsSeekable(); + } + @Test void getListObjects() throws IOException { tester.testListObjects(); @@ -51,4 +56,4 @@ void testRemoveObjects() throws IOException { tester.testRemoveObjects(); } -} \ No newline at end of file +} diff --git a/java/src/test/java/com/esamtrade/bucketbase/MinioParquetFunctionalTest.java b/java/src/test/java/com/esamtrade/bucketbase/MinioParquetFunctionalTest.java new file mode 100644 index 0000000..e365536 --- /dev/null +++ b/java/src/test/java/com/esamtrade/bucketbase/MinioParquetFunctionalTest.java @@ -0,0 +1,190 @@ +package com.esamtrade.bucketbase; + +import io.trino.spi.predicate.Domain; +import io.trino.spi.predicate.Range; +import io.trino.spi.predicate.ValueSet; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIf; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import static io.trino.spi.type.BigintType.BIGINT; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * End-to-end Parquet tests against a real S3-compatible service. + * + *

These are the Java counterpart of the Python MinIO tests: they verify that ranged HTTP GETs + * really do let a Parquet reader read a footer, project columns and prune row groups without + * downloading the whole object, and that a streamed multipart write commits or aborts correctly. + * The in-memory tests cannot catch range-header, ETag or multipart-sizing problems.

+ * + *

The bucket is configured through {@link TestConfig}; each run uses a unique prefix and + * cleans up after itself.

+ */ +@EnabledIf("com.esamtrade.bucketbase.TestConfig#functionalTestsEnabled") +class MinioParquetFunctionalTest { + + private static final int ROWS = 30_000; + + private S3Bucket bucket; + private PurePosixPrefix prefix; + private PurePosixPath object; + + @BeforeEach + void setUp() throws IOException { + // readBufferSize 0 disables client-side read-ahead so the byte counts below are an exact + // account of what was pulled over the wire. + bucket = new S3Bucket( + TestConfig.MINIO_ENDPOINT, + TestConfig.MINIO_ACCESS_KEY, + TestConfig.MINIO_SECRET_KEY, + TestConfig.MINIO_BUCKET, + 0); + TestConfig.ensureBucketExists(bucket); + prefix = PurePosixPrefix.from( + "bucketbase-java-parquet-" + UUID.randomUUID().toString().substring(0, 8) + "/"); + object = prefix.join(new PurePosixPath("data.parquet")); + } + + @AfterEach + void tearDown() throws IOException { + if (bucket != null && prefix != null) { + bucket.removePrefix(prefix); + } + } + + @Test + @DisplayName("a Parquet file streamed through openWrite round-trips via real multipart upload") + void streamedWriteRoundTrips() throws IOException { + try (ObjectWriter writer = bucket.openWrite(object)) { + ParquetTestSupport.writeParquet(writer.stream(), ROWS); + writer.commit(); + } + + assertTrue(bucket.exists(object)); + assertTrue(bucket.getSize(object) > 0); + assertEquals(ROWS, ParquetTestSupport.countAllRows(bucket, object)); + } + + @Test + @DisplayName("footer, projection and pushdown each read only part of the remote object") + void partialReadsOverHttpRange() throws IOException { + bucket.putObject(object, ParquetTestSupport.parquetBytes(ROWS)); + long objectSize = bucket.getSize(object); + + ParquetTestSupport.ReadResult footer = ParquetTestSupport.readFooterOnly(bucket, object); + ParquetTestSupport.ReadResult projected = ParquetTestSupport.readIds(bucket, object, Optional.empty()); + ParquetTestSupport.ReadResult pushedDown = ParquetTestSupport.readIds(bucket, object, + Optional.of(Domain.create(ValueSet.ofRanges(Range.greaterThan(BIGINT, 25_000L)), false))); + + // Footer-only: the reader seeks to the tail rather than downloading the object. + assertTrue(footer.bytesFetched() < objectSize / 10, + "footer read pulled " + footer.bytesFetched() + " of " + objectSize + " bytes"); + + // Column projection: the wide `name` column is never fetched. + assertEquals(ROWS, projected.ids().size()); + assertTrue(projected.bytesFetched() < objectSize, + "projected read pulled " + projected.bytesFetched() + " of " + objectSize + " bytes"); + + // Predicate pushdown: two of three row groups are pruned from statistics alone. + assertEquals(1, pushedDown.rowGroupsRead()); + assertEquals(3, pushedDown.rowGroupsTotal()); + assertTrue(pushedDown.bytesFetched() < projected.bytesFetched(), + "pushdown pulled " + pushedDown.bytesFetched() + " bytes vs " + projected.bytesFetched() + + " for the unfiltered projection"); + assertTrue(pushedDown.ids().stream().allMatch(id -> id >= 20_000)); + + // Several distinct ranged GETs, not one big download. + assertTrue(projected.rangeRequests() > 1, + "expected multiple ranged reads, got " + projected.rangeRequests()); + } + + @Test + @DisplayName("an unsatisfiable predicate reads nothing but the footer from the remote object") + void unsatisfiablePredicateReadsOnlyFooter() throws IOException { + bucket.putObject(object, ParquetTestSupport.parquetBytes(ROWS)); + + ParquetTestSupport.ReadResult footer = ParquetTestSupport.readFooterOnly(bucket, object); + ParquetTestSupport.ReadResult filtered = ParquetTestSupport.readIds(bucket, object, + Optional.of(Domain.create(ValueSet.ofRanges(Range.greaterThan(BIGINT, 10_000_000L)), false))); + + assertEquals(0, filtered.rowGroupsRead()); + assertTrue(filtered.ids().isEmpty()); + assertEquals(footer.bytesFetched(), filtered.bytesFetched()); + } + + @Test + @DisplayName("an abandoned streamed write leaves no object in the bucket") + void abortedWriteLeavesNoObject() throws IOException { + assertThrows(RuntimeException.class, () -> { + try (ObjectWriter writer = bucket.openWrite(object)) { + ParquetTestSupport.writeParquet(writer.stream(), ROWS); + throw new RuntimeException("simulated failure before commit"); + } + }); + + assertFalse(bucket.exists(object), "the multipart upload should have been aborted"); + List leftovers = bucket.listObjects(prefix); + assertTrue(leftovers.isEmpty(), "no partial object should be listed, found: " + leftovers); + } + + @Test + @DisplayName("a body larger than the multipart part size uploads correctly from a chunked source") + void multiPartUploadFromChunkedSource() throws IOException { + // Larger than the 5 MiB part size, written in small chunks. The sink delivers whatever + // the caller wrote, so the uploader must coalesce short reads into full-size parts - + // otherwise S3 rejects the undersized non-final parts with EntityTooSmall. + final int chunk = 8 * 1024; + final int chunks = 1_400; // ~11 MiB => 3 parts + byte[] payload = new byte[chunk]; + for (int i = 0; i < chunk; i++) { + payload[i] = (byte) (i % 251); + } + + try (ObjectWriter writer = bucket.openWrite(object)) { + OutputStream sink = writer.stream(); + for (int i = 0; i < chunks; i++) { + sink.write(payload); + } + writer.commit(); + } + + long expectedSize = (long) chunk * chunks; + assertEquals(expectedSize, bucket.getSize(object)); + + // Spot-check the bytes at a part boundary rather than downloading 11 MiB. + try (SeekableInputStream stream = bucket.getObjectStream(object)) { + long boundary = 5L * 1024 * 1024; + stream.seek(boundary); + byte[] sample = stream.readNBytes(16); + for (int i = 0; i < sample.length; i++) { + int offsetInChunk = (int) ((boundary + i) % chunk); + assertEquals((byte) (offsetInChunk % 251), sample[i], "byte mismatch at offset " + (boundary + i)); + } + } + } + + @Test + @DisplayName("putObjectStream can be called more than once on the same bucket instance") + void putObjectStreamIsReusable() throws IOException { + PurePosixPath first = prefix.join("first.bin"); + PurePosixPath second = prefix.join("second.bin"); + + bucket.putObjectStream(first, new java.io.ByteArrayInputStream("first payload".getBytes())); + bucket.putObjectStream(second, new java.io.ByteArrayInputStream("second payload".getBytes())); + + assertEquals("first payload", new String(bucket.getObject(first))); + assertEquals("second payload", new String(bucket.getObject(second))); + } +} diff --git a/java/src/test/java/com/esamtrade/bucketbase/ObjectWriterCancellationTest.java b/java/src/test/java/com/esamtrade/bucketbase/ObjectWriterCancellationTest.java new file mode 100644 index 0000000..a6b725d --- /dev/null +++ b/java/src/test/java/com/esamtrade/bucketbase/ObjectWriterCancellationTest.java @@ -0,0 +1,82 @@ +package com.esamtrade.bucketbase; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.InputStream; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * {@link ObjectWriter} promises that an object is stored only after {@link ObjectWriter#commit()} + * returns successfully, and that an abandoned write leaves nothing behind. + * + *

Bounding only the caller's wait is not enough to honour that: if the uploader thread is left + * running after a timeout or an abort, it can finish the upload later and publish an object the + * caller was already told had failed. Cancellation is best-effort - it depends on the underlying + * store honouring interruption - but the writer must at least request it.

+ */ +class ObjectWriterCancellationTest { + + private static final PurePosixPath OBJECT = PurePosixPath.from("dir/slow.bin"); + + @Test + void aTimedOutCommitCancelsTheUploader() throws Exception { + SlowBucket bucket = new SlowBucket(1_500); + + IOException failure = assertThrows(IOException.class, () -> { + try (ObjectWriter writer = bucket.openWrite(OBJECT, 200)) { + writer.stream().write("payload".getBytes()); + writer.commit(); + } + }); + assertTrue(failure.getMessage().contains("Timed out"), failure.getMessage()); + + assertFalse(bucket.stored.await(3, TimeUnit.SECONDS), + "the uploader kept running after the timeout and stored the object anyway"); + assertFalse(bucket.exists(OBJECT), "no object may exist after a failed commit"); + } + + @Test + void anAbandonedWriteCancelsTheUploader() throws Exception { + SlowBucket bucket = new SlowBucket(1_500); + + try (ObjectWriter writer = bucket.openWrite(OBJECT, 200)) { + writer.stream().write("payload".getBytes()); + // No commit: closing must abort, and must not leave the uploader running. + } catch (IOException expectedOnAbort) { + // close() may surface a timeout while cancelling; the object must still be absent. + } + + assertFalse(bucket.stored.await(3, TimeUnit.SECONDS), + "the uploader kept running after the write was abandoned and stored the object anyway"); + assertFalse(bucket.exists(OBJECT)); + } + + /** Drains the body, then stalls before storing, so a cancellation window exists. */ + private static final class SlowBucket extends MemoryBucket { + private final CountDownLatch stored = new CountDownLatch(1); + private final long stallMillis; + + SlowBucket(long stallMillis) { + this.stallMillis = stallMillis; + } + + @Override + public void putObjectStream(PurePosixPath name, InputStream stream) throws IOException { + byte[] body = stream.readAllBytes(); + try { + Thread.sleep(stallMillis); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("upload cancelled", e); + } + putObject(name, body); + stored.countDown(); + } + } +} diff --git a/java/src/test/java/com/esamtrade/bucketbase/ParquetReadTest.java b/java/src/test/java/com/esamtrade/bucketbase/ParquetReadTest.java new file mode 100644 index 0000000..e867db2 --- /dev/null +++ b/java/src/test/java/com/esamtrade/bucketbase/ParquetReadTest.java @@ -0,0 +1,143 @@ +package com.esamtrade.bucketbase; + +import io.trino.spi.predicate.Domain; +import io.trino.spi.predicate.Range; +import io.trino.spi.predicate.ValueSet; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.List; +import java.util.Optional; + +import static io.trino.spi.type.BigintType.BIGINT; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Proves that {@link IBucket#getObjectStream(PurePosixPath)} is a good enough random-access + * stream to drive a real Parquet reader: footer-only reads, column projection and predicate + * pushdown all translate into partial reads of the object rather than a full download. + * + *

This is the Java counterpart of the Python assertions in + * {@code tests/test_minio_range_reader.py::test_parquet_tail_reads_selected_columns_without_full_object_get} + * and {@code tests/bucket_tester.py::test_get_object_stream_with_parquet_tail}.

+ */ +class ParquetReadTest { + + private static final int ROWS = 30_000; + private static final PurePosixPath OBJECT = PurePosixPath.from("parquet/read-test.parquet"); + + private MemoryBucket bucket; + private long objectSize; + + @BeforeEach + void setUp() throws IOException { + bucket = new MemoryBucket(); + byte[] parquet = ParquetTestSupport.parquetBytes(ROWS); + bucket.putObject(OBJECT, parquet); + objectSize = parquet.length; + } + + @Test + @DisplayName("getSize reports the stored object size") + void getSizeMatchesStoredObject() throws IOException { + assertEquals(objectSize, bucket.getSize(OBJECT)); + } + + @Test + @DisplayName("all rows and row groups round-trip through the bucket") + void roundTripsEveryRow() throws IOException { + assertEquals(ROWS, ParquetTestSupport.countAllRows(bucket, OBJECT)); + } + + @Test + @DisplayName("reading the footer touches only the tail of the object") + void footerReadIsPartial() throws IOException { + ParquetTestSupport.ReadResult footer = ParquetTestSupport.readFooterOnly(bucket, OBJECT); + + assertEquals(ROWS / ParquetTestSupport.ROWS_PER_ROW_GROUP, footer.rowGroupsTotal(), + "fixture should have one row group per " + ParquetTestSupport.ROWS_PER_ROW_GROUP + " rows"); + assertTrue(footer.bytesFetched() < objectSize / 10, + "footer read fetched " + footer.bytesFetched() + " of " + objectSize + " bytes; expected under 10%"); + } + + @Test + @DisplayName("column projection skips the unread columns' data") + void projectionReadsLessThanFullScan() throws IOException { + ParquetTestSupport.ReadResult projected = ParquetTestSupport.readIds(bucket, OBJECT, Optional.empty()); + + assertEquals(ROWS, projected.ids().size(), "projection must still return every row"); + assertEquals(0L, projected.ids().get(0)); + assertEquals(ROWS - 1L, projected.ids().get(projected.ids().size() - 1)); + assertEquals(projected.rowGroupsTotal(), projected.rowGroupsRead(), + "no predicate means no row group is pruned"); + assertTrue(projected.bytesFetched() < objectSize / 2, + "projected read fetched " + projected.bytesFetched() + " of " + objectSize + + " bytes; the wide `name` column should not have been read"); + } + + @Test + @DisplayName("predicate pushdown prunes row groups using column statistics") + void predicatePushdownPrunesRowGroups() throws IOException { + // ids run 0..29999 in three row groups of 10000; only the last can satisfy id > 25000. + Domain domain = Domain.create(ValueSet.ofRanges(Range.greaterThan(BIGINT, 25_000L)), false); + ParquetTestSupport.ReadResult filtered = ParquetTestSupport.readIds(bucket, OBJECT, Optional.of(domain)); + ParquetTestSupport.ReadResult unfiltered = ParquetTestSupport.readIds(bucket, OBJECT, Optional.empty()); + + assertEquals(1, filtered.rowGroupsRead(), "only the last row group can match id > 25000"); + assertEquals(3, filtered.rowGroupsTotal()); + + List ids = filtered.ids(); + assertFalse(ids.isEmpty(), "the matching row group must still be read"); + // Row-group pruning is coarse: the reader returns whole row groups, so every returned id + // comes from the surviving group rather than being exactly the predicate's result set. + assertTrue(ids.stream().allMatch(id -> id >= 20_000), + "pruned row groups must not contribute rows; got min id " + ids.stream().mapToLong(Long::longValue).min().orElseThrow()); + assertTrue(ids.contains(29_999L), "the surviving row group runs to the end of the file"); + + assertTrue(filtered.bytesFetched() < unfiltered.bytesFetched(), + "pushdown fetched " + filtered.bytesFetched() + " bytes, which should be less than the " + + unfiltered.bytesFetched() + " bytes of an unfiltered projection"); + } + + @Test + @DisplayName("a predicate no row group can satisfy reads nothing but the footer") + void unsatisfiablePredicateReadsOnlyFooter() throws IOException { + Domain domain = Domain.create(ValueSet.ofRanges(Range.greaterThan(BIGINT, 10_000_000L)), false); + ParquetTestSupport.ReadResult filtered = ParquetTestSupport.readIds(bucket, OBJECT, Optional.of(domain)); + ParquetTestSupport.ReadResult footerOnly = ParquetTestSupport.readFooterOnly(bucket, OBJECT); + + assertEquals(0, filtered.rowGroupsRead(), "every row group should be pruned"); + assertTrue(filtered.ids().isEmpty()); + assertEquals(footerOnly.bytesFetched(), filtered.bytesFetched(), + "with all row groups pruned, only the footer should have been fetched"); + } + + @Test + @DisplayName("the object stream seeks backwards and forwards without reopening") + void streamSupportsRandomAccess() throws IOException { + byte[] whole = bucket.getObject(OBJECT); + try (SeekableInputStream stream = bucket.getObjectStream(OBJECT)) { + assertEquals(objectSize, stream.size()); + + stream.seek(objectSize - 4); + byte[] magic = stream.readNBytes(4); + assertEquals("PAR1", new String(magic, java.nio.charset.StandardCharsets.US_ASCII)); + assertEquals(objectSize, stream.position()); + + // Seek backwards to the header - a fresh range read, no reopen. + stream.seek(0); + assertEquals("PAR1", new String(stream.readNBytes(4), java.nio.charset.StandardCharsets.US_ASCII)); + + // A mid-object seek must agree with the fully downloaded bytes. + long middle = objectSize / 2; + stream.seek(middle); + byte[] sample = stream.readNBytes(64); + byte[] expected = java.util.Arrays.copyOfRange(whole, (int) middle, (int) middle + 64); + org.junit.jupiter.api.Assertions.assertArrayEquals(expected, sample); + } + } +} diff --git a/java/src/test/java/com/esamtrade/bucketbase/ParquetTestSupport.java b/java/src/test/java/com/esamtrade/bucketbase/ParquetTestSupport.java new file mode 100644 index 0000000..9375286 --- /dev/null +++ b/java/src/test/java/com/esamtrade/bucketbase/ParquetTestSupport.java @@ -0,0 +1,271 @@ +package com.esamtrade.bucketbase; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.airlift.slice.Slices; +import io.airlift.units.DataSize; +import io.trino.parquet.Column; +import io.trino.parquet.ParquetReaderOptions; +import io.trino.parquet.ParquetTypeUtils; +import io.trino.parquet.metadata.FileMetadata; +import io.trino.parquet.metadata.ParquetMetadata; +import io.trino.parquet.predicate.PredicateUtils; +import io.trino.parquet.predicate.TupleDomainParquetPredicate; +import io.trino.parquet.reader.MetadataReader; +import io.trino.parquet.reader.ParquetReader; +import io.trino.parquet.reader.RowGroupInfo; +import io.trino.parquet.writer.ParquetSchemaConverter; +import io.trino.parquet.writer.ParquetWriterOptions; +import io.trino.spi.Page; +import io.trino.spi.block.Block; +import io.trino.spi.block.BlockBuilder; +import io.trino.spi.connector.SourcePage; +import io.trino.spi.predicate.Domain; +import io.trino.spi.predicate.TupleDomain; +import io.trino.spi.type.Type; +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.format.CompressionCodec; +import org.apache.parquet.io.MessageColumnIO; +import org.apache.parquet.schema.MessageType; +import org.joda.time.DateTimeZone; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.google.common.base.Throwables.throwIfUnchecked; +import static io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.BooleanType.BOOLEAN; +import static io.trino.spi.type.VarcharType.VARCHAR; + +/** + * Shared Parquet fixtures and read helpers built on Trino's Hadoop-free Parquet library. + * + *

The fixture schema is {@code (id BIGINT, name VARCHAR, active BOOLEAN)} with sequential + * ids, deliberately mirroring the Python test fixture in {@code python/tests/bucket_tester.py} + * so both languages assert the same shape of data.

+ */ +public final class ParquetTestSupport { + + public static final List COLUMN_NAMES = ImmutableList.of("id", "name", "active"); + public static final List COLUMN_TYPES = ImmutableList.of(BIGINT, VARCHAR, BOOLEAN); + + /** Rows per row group in the fixtures, small enough to make pruning observable. */ + public static final int ROWS_PER_ROW_GROUP = 10_000; + + private ParquetTestSupport() { + } + + /** + * Reader options tuned for byte accounting: a small footer read and no range merging, so a + * measured byte count reflects the data actually needed rather than Trino's read-ahead + * heuristics. Production defaults are 48 KB footer reads and 1 MB merge distance. + */ + public static ParquetReaderOptions measurableOptions() { + return ParquetReaderOptions.builder() + .withFooterReadSize(DataSize.ofBytes(8192)) + .withMaxMergeDistance(DataSize.ofBytes(0)) + .build(); + } + + /** + * Writes a Parquet file to an arbitrary {@link OutputStream}. The footer is emitted when the + * underlying {@link io.trino.parquet.writer.ParquetWriter} is closed, which is precisely the + * behaviour that makes {@link ObjectWriter}'s close/commit split necessary. + */ + public static void writeParquet(OutputStream out, int rows) throws IOException { + ParquetSchemaConverter schema = new ParquetSchemaConverter(COLUMN_TYPES, COLUMN_NAMES, false, false); + ParquetWriterOptions options = ParquetWriterOptions.builder() + .setMaxRowGroupRowCount(ROWS_PER_ROW_GROUP) + .setMaxPageSize(DataSize.ofBytes(16 * 1024)) + .build(); + + try (io.trino.parquet.writer.ParquetWriter writer = new io.trino.parquet.writer.ParquetWriter( + out, + schema.getMessageType(), + schema.getPrimitiveTypes(), + options, + CompressionCodec.SNAPPY, + "bucketbase-test", + Optional.of(DateTimeZone.UTC), + Optional.empty())) { + // One page per row group keeps row-group boundaries deterministic. + for (int start = 0; start < rows; start += ROWS_PER_ROW_GROUP) { + int count = Math.min(ROWS_PER_ROW_GROUP, rows - start); + writer.write(buildPage(start, count)); + } + } + } + + /** Builds the fixture bytes in memory. */ + public static byte[] parquetBytes(int rows) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + writeParquet(out, rows); + return out.toByteArray(); + } + + private static Page buildPage(int firstId, int count) { + BlockBuilder ids = BIGINT.createFixedSizeBlockBuilder(count); + BlockBuilder names = VARCHAR.createBlockBuilder(null, count); + BlockBuilder active = BOOLEAN.createFixedSizeBlockBuilder(count); + for (int i = 0; i < count; i++) { + long id = firstId + i; + BIGINT.writeLong(ids, id); + // Padded so the VARCHAR column is by far the largest, making projection measurable. + VARCHAR.writeSlice(names, Slices.utf8Slice("object-name-with-padding-to-make-it-wide-" + id)); + BOOLEAN.writeBoolean(active, id % 2 == 0); + } + return new Page(ids.build(), names.build(), active.build()); + } + + /** Result of a projected/filtered read, with the byte accounting that proves partial reads. */ + public record ReadResult(List ids, int rowGroupsRead, int rowGroupsTotal, long bytesFetched, int rangeRequests) { + } + + /** Reads only the footer, returning how many bytes that cost. */ + public static ReadResult readFooterOnly(IBucket bucket, PurePosixPath name) throws IOException { + ParquetReaderOptions options = measurableOptions(); + try (BucketParquetDataSource dataSource = BucketParquetDataSource.open(bucket, name, options)) { + ParquetMetadata metadata = MetadataReader.readFooter(dataSource, options, Optional.empty(), Optional.empty()); + return new ReadResult( + List.of(), + 0, + metadata.getBlocks().size(), + dataSource.bytesFetched(), + dataSource.rangeRequests()); + } + } + + /** + * Reads the {@code id} column only, optionally pushing a predicate down so row groups whose + * statistics cannot match are never fetched. + * + * @param idPredicate a domain over the {@code id} column, or empty for no pushdown + */ + public static ReadResult readIds(IBucket bucket, PurePosixPath name, Optional idPredicate) throws IOException { + ParquetReaderOptions options = measurableOptions(); + try (BucketParquetDataSource dataSource = BucketParquetDataSource.open(bucket, name, options)) { + ParquetMetadata metadata = MetadataReader.readFooter(dataSource, options, Optional.empty(), Optional.empty()); + FileMetadata fileMetadata = metadata.getFileMetaData(); + MessageType fileSchema = fileMetadata.getSchema(); + MessageColumnIO columnIO = ParquetTypeUtils.getColumnIO(fileSchema, fileSchema); + + // Column projection: only the columns we build a Column for are planned and fetched. + List projected = ImmutableList.of(new Column( + "id", + ParquetTypeUtils.constructField(BIGINT, ParquetTypeUtils.lookupColumnByName(columnIO, "id")).orElseThrow())); + + Map, ColumnDescriptor> descriptorsByPath = ParquetTypeUtils.getDescriptors(fileSchema, fileSchema); + ColumnDescriptor idDescriptor = descriptorsByPath.get(ImmutableList.of("id")); + + TupleDomain tupleDomain = idPredicate + .map(domain -> TupleDomain.withColumnDomains(ImmutableMap.of(idDescriptor, domain))) + .orElseGet(TupleDomain::all); + TupleDomainParquetPredicate predicate = + PredicateUtils.buildPredicate(fileSchema, tupleDomain, descriptorsByPath, DateTimeZone.UTC); + + // Row-group pruning is the caller's job in Trino's API; ParquetReader only does + // page-level filtering on the row groups it is handed. + List rowGroups = PredicateUtils.getFilteredRowGroups( + 0, + dataSource.getEstimatedSize(), + dataSource, + metadata, + ImmutableList.of(tupleDomain), + ImmutableList.of(predicate), + descriptorsByPath, + DateTimeZone.UTC, + 1000, + options); + + List ids = new ArrayList<>(); + try (ParquetReader reader = new ParquetReader( + Optional.ofNullable(fileMetadata.getCreatedBy()), + projected, + false, + rowGroups, + dataSource, + DateTimeZone.UTC, + newSimpleAggregatedMemoryContext(), + options, + e -> { + throwIfUnchecked(e); + return new RuntimeException(e); + }, + Optional.of(predicate), + Optional.empty(), + Optional.empty())) { + SourcePage page; + while ((page = reader.nextPage()) != null) { + Block block = page.getBlock(0); + for (int i = 0; i < block.getPositionCount(); i++) { + ids.add(BIGINT.getLong(block, i)); + } + } + } + return new ReadResult( + ids, + rowGroups.size(), + metadata.getBlocks().size(), + dataSource.bytesFetched(), + dataSource.rangeRequests()); + } + } + + /** Reads every column of every row, for round-trip verification. */ + public static int countAllRows(IBucket bucket, PurePosixPath name) throws IOException { + ParquetReaderOptions options = measurableOptions(); + try (BucketParquetDataSource dataSource = BucketParquetDataSource.open(bucket, name, options)) { + ParquetMetadata metadata = MetadataReader.readFooter(dataSource, options, Optional.empty(), Optional.empty()); + FileMetadata fileMetadata = metadata.getFileMetaData(); + MessageType fileSchema = fileMetadata.getSchema(); + MessageColumnIO columnIO = ParquetTypeUtils.getColumnIO(fileSchema, fileSchema); + + ImmutableList.Builder columns = ImmutableList.builder(); + for (int i = 0; i < COLUMN_NAMES.size(); i++) { + String columnName = COLUMN_NAMES.get(i); + columns.add(new Column(columnName, ParquetTypeUtils + .constructField(COLUMN_TYPES.get(i), ParquetTypeUtils.lookupColumnByName(columnIO, columnName)) + .orElseThrow())); + } + + Map, ColumnDescriptor> descriptorsByPath = ParquetTypeUtils.getDescriptors(fileSchema, fileSchema); + TupleDomain all = TupleDomain.all(); + TupleDomainParquetPredicate predicate = + PredicateUtils.buildPredicate(fileSchema, all, descriptorsByPath, DateTimeZone.UTC); + List rowGroups = PredicateUtils.getFilteredRowGroups( + 0, dataSource.getEstimatedSize(), dataSource, metadata, + ImmutableList.of(all), ImmutableList.of(predicate), + descriptorsByPath, DateTimeZone.UTC, 1000, options); + + int rows = 0; + try (ParquetReader reader = new ParquetReader( + Optional.ofNullable(fileMetadata.getCreatedBy()), + columns.build(), + false, + rowGroups, + dataSource, + DateTimeZone.UTC, + newSimpleAggregatedMemoryContext(), + options, + e -> { + throwIfUnchecked(e); + return new RuntimeException(e); + }, + Optional.of(predicate), + Optional.empty(), + Optional.empty())) { + SourcePage page; + while ((page = reader.nextPage()) != null) { + rows += page.getPositionCount(); + } + } + return rows; + } + } +} diff --git a/java/src/test/java/com/esamtrade/bucketbase/ParquetWriteTest.java b/java/src/test/java/com/esamtrade/bucketbase/ParquetWriteTest.java new file mode 100644 index 0000000..eb983cc --- /dev/null +++ b/java/src/test/java/com/esamtrade/bucketbase/ParquetWriteTest.java @@ -0,0 +1,152 @@ +package com.esamtrade.bucketbase; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.OutputStream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Covers {@link IBucket#openWrite(PurePosixPath)} with a real Parquet writer. + * + *

Parquet finalises a file by writing its footer when the writer is closed, so a naive sink + * would publish the object at exactly the wrong moment. These tests pin down the commit/abort + * contract that makes that safe, mirroring the Python regression tests + * {@code test_regression_exception_thrown_in_parquet_writer_context_doesnt_save_object} and + * friends in {@code python/tests/bucket_tester.py}.

+ */ +class ParquetWriteTest { + + private static final int ROWS = 25_000; + private static final PurePosixPath OBJECT = PurePosixPath.from("parquet/write-test.parquet"); + + private MemoryBucket bucket; + + @BeforeEach + void setUp() { + bucket = new MemoryBucket(); + } + + @Test + @DisplayName("a streamed Parquet file commits and reads back intact") + void streamingWriteRoundTrips() throws IOException { + try (ObjectWriter writer = bucket.openWrite(OBJECT)) { + ParquetTestSupport.writeParquet(writer.stream(), ROWS); + writer.commit(); + } + + assertTrue(bucket.exists(OBJECT)); + assertEquals(ROWS, ParquetTestSupport.countAllRows(bucket, OBJECT)); + assertTrue(bucket.getSize(OBJECT) > 0); + } + + @Test + @DisplayName("the object is invisible until commit, even after the Parquet footer is written") + void nestedCloseDoesNotCommit() throws IOException { + try (ObjectWriter writer = bucket.openWrite(OBJECT)) { + // writeParquet closes the ParquetWriter internally, which emits the footer and + // closes the sink. That must not publish the object. + ParquetTestSupport.writeParquet(writer.stream(), 1_000); + assertFalse(bucket.exists(OBJECT), + "object must not exist after the Parquet writer closed the sink but before commit()"); + + writer.commit(); + assertTrue(bucket.exists(OBJECT), "commit() must publish the object"); + } + } + + @Test + @DisplayName("an exception inside the writer context leaves no object behind") + void abandonedWriteLeavesNoObject() { + RuntimeException boom = assertThrows(RuntimeException.class, () -> { + try (ObjectWriter writer = bucket.openWrite(OBJECT)) { + ParquetTestSupport.writeParquet(writer.stream(), 5_000); + throw new RuntimeException("simulated failure after writing the Parquet body"); + } + }); + + assertEquals("simulated failure after writing the Parquet body", boom.getMessage()); + assertFalse(bucket.exists(OBJECT), "an aborted write must not create an object"); + assertTrue(bucket.listObjects(PurePosixPrefix.from("parquet/")).isEmpty(), + "no partial object should be listed either"); + } + + @Test + @DisplayName("failing mid-stream, before any footer, also leaves no object") + void failureMidStreamLeavesNoObject() { + assertThrows(IllegalStateException.class, () -> { + try (ObjectWriter writer = bucket.openWrite(OBJECT)) { + OutputStream sink = writer.stream(); + sink.write(new byte[64 * 1024]); + throw new IllegalStateException("aborted halfway"); + } + }); + + assertFalse(bucket.exists(OBJECT)); + } + + @Test + @DisplayName("closing without committing is not an error, it just discards the write") + void closeWithoutCommitIsSilent() throws IOException { + try (ObjectWriter writer = bucket.openWrite(OBJECT)) { + writer.stream().write("some bytes that will never be stored".getBytes()); + } + assertFalse(bucket.exists(OBJECT)); + } + + @Test + @DisplayName("committing an empty body stores an empty object") + void emptyWriteCommits() throws IOException { + try (ObjectWriter writer = bucket.openWrite(OBJECT)) { + writer.commit(); + } + assertTrue(bucket.exists(OBJECT)); + assertEquals(0, bucket.getSize(OBJECT)); + } + + @Test + @DisplayName("commit after close is rejected") + void commitAfterCloseFails() throws IOException { + ObjectWriter writer = bucket.openWrite(OBJECT); + writer.stream().write("data".getBytes()); + writer.close(); + + IOException failure = assertThrows(IOException.class, writer::commit); + assertTrue(failure.getMessage().contains("already closed"), failure.getMessage()); + assertFalse(bucket.exists(OBJECT)); + } + + @Test + @DisplayName("commit is idempotent and close after commit is a no-op") + void commitIsIdempotent() throws IOException { + try (ObjectWriter writer = bucket.openWrite(OBJECT)) { + writer.stream().write("payload".getBytes()); + writer.commit(); + writer.commit(); + } + assertEquals("payload", new String(bucket.getObject(OBJECT))); + } + + @Test + @DisplayName("many small writes stream through without buffering the whole body") + void manySmallWritesStream() throws IOException { + try (ObjectWriter writer = bucket.openWrite(OBJECT)) { + OutputStream sink = writer.stream(); + for (int i = 0; i < 5_000; i++) { + sink.write(("line-" + i + "\n").getBytes()); + } + writer.commit(); + } + + String stored = new String(bucket.getObject(OBJECT)); + assertTrue(stored.startsWith("line-0\n")); + assertTrue(stored.endsWith("line-4999\n")); + assertEquals(5_000, stored.lines().count()); + } +} diff --git a/java/src/test/java/com/esamtrade/bucketbase/PurePosixPathTest.java b/java/src/test/java/com/esamtrade/bucketbase/PurePosixPathTest.java index 6aa226a..521e395 100644 --- a/java/src/test/java/com/esamtrade/bucketbase/PurePosixPathTest.java +++ b/java/src/test/java/com/esamtrade/bucketbase/PurePosixPathTest.java @@ -2,181 +2,187 @@ import org.junit.jupiter.api.Test; -import java.util.Arrays; +import java.nio.file.Path; import java.util.List; - -import static org.junit.jupiter.api.Assertions.*; - +import java.util.TreeSet; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Contract tests mirrored by {@code python/tests/test_pure_posix_path_stdlib_contract.py}. + * + *

The Python test runs these expectations against the installed stdlib + * {@code pathlib.PurePosixPath}; this class applies the same expectations to the Java port.

+ */ class PurePosixPathTest { @Test - void nominal_test() { - PurePosixPath basePath = new PurePosixPath("/home/user"); - assertEquals("/home/user", basePath.toString()); - assertEquals(Arrays.asList("", "home", "user"), Arrays.asList(basePath.parts)); - basePath = new PurePosixPath("home", "user"); - assertEquals("home/user", basePath.toString()); - assertEquals(Arrays.asList("home", "user"), Arrays.asList(basePath.parts)); - } - - @Test - void slash_ending_path() { - PurePosixPath basePath = new PurePosixPath("/home/user/"); - assertEquals("/home/user/", basePath.toString()); - assertEquals(Arrays.asList("", "home", "user", ""), Arrays.asList(basePath.parts)); + void constructionAndPartsMatchStdlib() { + PurePosixPath absolute = new PurePosixPath("/home/user"); + assertEquals("/home/user", absolute.toString()); + assertEquals(List.of("/", "home", "user"), absolute.parts()); + assertEquals("/", absolute.get(0)); - basePath = new PurePosixPath("/home/user", "/"); - assertEquals(Arrays.asList("", "home", "user", ""), Arrays.asList(basePath.parts)); - basePath = new PurePosixPath("home/user", "/"); - assertEquals(Arrays.asList("home", "user", ""), Arrays.asList(basePath.parts)); + PurePosixPath relative = new PurePosixPath("home", "user"); + assertEquals("home/user", relative.toString()); + assertEquals(List.of("home", "user"), relative.parts()); - basePath = new PurePosixPath("./home/user", "/."); - assertEquals(Arrays.asList("home", "user", ""), Arrays.asList(basePath.parts)); - - basePath = new PurePosixPath(".", "./home/./user", "/."); - assertEquals(Arrays.asList("home", "user", ""), Arrays.asList(basePath.parts)); + PurePosixPath empty = new PurePosixPath(""); + assertEquals(".", empty.toString()); + assertEquals(List.of(), empty.parts()); + assertEquals(new PurePosixPath(), empty); } @Test - void handling_updir_path() { - PurePosixPath basePath = new PurePosixPath("/home/user/.."); - assertEquals("/home/", basePath.toString()); - assertEquals(Arrays.asList("", "home", ""), Arrays.asList(basePath.parts)); - - basePath = new PurePosixPath("/home/../user", "/"); - assertEquals(Arrays.asList("", "user", ""), Arrays.asList(basePath.parts)); - - basePath = new PurePosixPath("home/../user", "/"); - assertEquals(Arrays.asList("user", ""), Arrays.asList(basePath.parts)); + void redundantSeparatorsTrailingSeparatorsAndDotComponentsCollapse() { + assertEquals("/home/user", new PurePosixPath("/home/user/").toString()); + assertEquals("home/user", new PurePosixPath("home//./user/.").toString()); + assertEquals(".", new PurePosixPath("./").toString()); + assertEquals("/", new PurePosixPath(".", "./home/./user", "/.").toString()); } @Test - void join_appendsSinglePathCorrectly() { - PurePosixPath basePath = new PurePosixPath("/home/user"); - PurePosixPath resultPath = basePath.join("docs"); - assertEquals("/home/user/docs", resultPath.toString()); - - basePath = new PurePosixPath("/home/user/"); - resultPath = basePath.join("./docs/"); - assertEquals("/home/user/docs/", resultPath.toString()); + void dotDotComponentsRemainLexical() { + assertEquals("/home/user/..", new PurePosixPath("/home/user/..").toString()); + assertEquals("/../x", new PurePosixPath("/../x").toString()); + assertEquals("/a/../../x", new PurePosixPath("/a/../../x").toString()); + assertEquals("a/..", new PurePosixPath("a/..").toString()); + assertEquals("../x", new PurePosixPath("../x").toString()); } @Test - void join_appendsMultiplePathsCorrectly() { - PurePosixPath basePath = new PurePosixPath("/home"); - PurePosixPath resultPath = basePath.join("user", "docs"); - assertEquals("/home/user/docs", resultPath.toString()); - - basePath = new PurePosixPath("/home/"); - resultPath = basePath.join("./user/", "./../docs/./", "./last-path/."); - assertEquals("/home/docs/last-path/", resultPath.toString()); - } + void laterAbsoluteSegmentReplacesEarlierSegments() { + assertEquals("/b", new PurePosixPath("/a", "/b").toString()); + assertEquals("/", new PurePosixPath("/home/user", "/").toString()); - @Test - void parent_returnsParentPathForNonRoot() { - PurePosixPath path = new PurePosixPath("/home/user/docs"); - assertEquals("/home/user", path.parent().toString()); + PurePosixPath base = new PurePosixPath("/a"); + assertEquals("/b", base.join("/b").toString()); + assertEquals("/b", base.join(new PurePosixPath("/b")).toString()); + assertEquals("/b", base.resolve("/b").toString()); } @Test - void parent_returnsRootForRootPath() { - PurePosixPath path = new PurePosixPath("/"); - // expect to raise exception when call path.parent() on the root - assertThrows(IllegalArgumentException.class, path::parent); - - path = new PurePosixPath(""); - assertThrows(IllegalArgumentException.class, path::parent); + void parentMatchesStdlib() { + assertEquals("/home", new PurePosixPath("/home/user").parent().toString()); + assertEquals("/", new PurePosixPath("/home").parent().toString()); + assertEquals("/", new PurePosixPath("/").parent().toString()); + assertEquals(".", new PurePosixPath("leaf").parent().toString()); + assertEquals(".", new PurePosixPath("").parent().toString()); + assertEquals("/home/user", new PurePosixPath("/home/user/..").parent().toString()); } @Test - void name_returnsFileNameForFilePath() { - PurePosixPath path = new PurePosixPath("/home/user/docs/file.txt"); - assertEquals("file.txt", path.name()); + void nameMatchesStdlib() { + assertEquals("file.txt", new PurePosixPath("/home/user/file.txt").name()); + assertEquals("..", new PurePosixPath("/home/user/..").name()); + assertEquals("", new PurePosixPath("/").name()); + assertEquals("", new PurePosixPath("").name()); } @Test - void resolve_resolvesRelativePathAgainstBasePath() { - PurePosixPath basePath = new PurePosixPath("/home/user"); - PurePosixPath resultPath = basePath.resolve("docs"); - assertEquals("/home/user/docs", resultPath.toString()); + void absoluteDetectionMatchesStdlib() { + assertTrue(new PurePosixPath("/home/user").isAbsolute()); + assertTrue(new PurePosixPath("//server/share").isAbsolute()); + assertFalse(new PurePosixPath("home/user").isAbsolute()); + assertFalse(new PurePosixPath(".").isAbsolute()); } @Test - void resolve_usesOtherPathIfAbsolute() { - PurePosixPath basePath = new PurePosixPath("/home/user"); - PurePosixPath resultPath = basePath.resolve("/etc"); - assertEquals("/etc", resultPath.toString()); + void isRelativeToMatchesStdlib() { + assertTrue(new PurePosixPath("/home/user").isRelativeTo(new PurePosixPath("/home"))); + assertFalse(new PurePosixPath("/home/user").isRelativeTo(new PurePosixPath("/etc"))); + assertFalse(new PurePosixPath("/a").isRelativeTo(new PurePosixPath(""))); + assertTrue(new PurePosixPath("a").isRelativeTo(new PurePosixPath("."))); + assertTrue(new PurePosixPath("../a").isRelativeTo(new PurePosixPath(".."))); } @Test - void isAbsolute_identifiesAbsolutePath() { - PurePosixPath path = new PurePosixPath("/home/user"); - assertTrue(path.isAbsolute()); - } + void suffixSuffixesAndStemMatchStdlib() { + PurePosixPath archive = new PurePosixPath("archive.tar.gz"); + assertEquals(".gz", archive.suffix()); + assertEquals(List.of(".tar", ".gz"), archive.suffixes()); + assertEquals("archive.tar", archive.stem()); - @Test - void isAbsolute_identifiesRelativePath() { - PurePosixPath path = new PurePosixPath("home/user"); - assertFalse(path.isAbsolute()); - } + PurePosixPath dotFile = new PurePosixPath(".bashrc"); + assertEquals("", dotFile.suffix()); + assertEquals(List.of(), dotFile.suffixes()); + assertEquals(".bashrc", dotFile.stem()); - @Test - void isRelativeTo_identifiesSubPath() { - PurePosixPath basePath = new PurePosixPath("/home"); - PurePosixPath otherPath = new PurePosixPath("/home/user"); - assertTrue(otherPath.isRelativeTo(basePath)); - } + // Python 3.14 treats a final dot as a valid suffix. + PurePosixPath finalDot = new PurePosixPath("name."); + assertEquals(".", finalDot.suffix()); + assertEquals(List.of("."), finalDot.suffixes()); + assertEquals("name", finalDot.stem()); - @Test - void isRelativeTo_identifiesNonSubPath() { - PurePosixPath basePath = new PurePosixPath("/home"); - PurePosixPath otherPath = new PurePosixPath("/etc"); - assertFalse(otherPath.isRelativeTo(basePath)); + PurePosixPath multipleLeadingDots = new PurePosixPath("..bashrc"); + assertEquals("", multipleLeadingDots.suffix()); + assertEquals(List.of(), multipleLeadingDots.suffixes()); + assertEquals("..bashrc", multipleLeadingDots.stem()); } @Test - void suffix_returnsFileExtension() { - PurePosixPath path = new PurePosixPath("/home/user/file.tar.gz"); - assertEquals(".gz", path.suffix()); + void exactlyTwoLeadingSeparatorsArePreserved() { + assertEquals("//server/share", new PurePosixPath("//server/share").toString()); + assertEquals(List.of("//", "server", "share"), + new PurePosixPath("//server/share").parts()); + assertEquals("/server/share", new PurePosixPath("///server/share").toString()); } @Test - void suffixes_returnsAllFileExtensions() { - PurePosixPath path = new PurePosixPath("/home/user/archive.tar.gz"); - assertEquals(Arrays.asList(".tar", ".gz"), path.suffixes()); - } + void equalityHashingAndOrderingMatchCanonicalStdlibPaths() { + PurePosixPath empty = new PurePosixPath(""); + PurePosixPath dot = new PurePosixPath("."); + assertEquals(empty, dot); + assertEquals(empty.hashCode(), dot.hashCode()); + assertEquals(new PurePosixPath("a"), new PurePosixPath("a/")); - @Test - void stem_returnsFileNameWithoutExtension() { - PurePosixPath path = new PurePosixPath("/home/user/file.txt"); - assertEquals("file", path.stem()); + assertTrue(new PurePosixPath("a/b").compareTo(new PurePosixPath("a-b")) < 0); + assertTrue(new PurePosixPath("a").compareTo(new PurePosixPath("a/b")) < 0); + assertTrue(new PurePosixPath("/a").compareTo(new PurePosixPath("a")) < 0); + assertTrue(new PurePosixPath("//a").compareTo(new PurePosixPath("/a")) < 0); + assertTrue(new PurePosixPath("\uE000").compareTo(new PurePosixPath("\uD83D\uDE00")) < 0, + "ordering must compare Unicode code points, as Python does"); + + TreeSet paths = new TreeSet<>(List.of( + new PurePosixPath("/home/./user"), + new PurePosixPath("/home/user"), + new PurePosixPath("/home/user2"))); + assertEquals(2, paths.size()); } @Test - void get() { - PurePosixPath path = new PurePosixPath("/home/user/file.txt"); - assertEquals("", path.get(0)); - assertEquals("home", path.get(1)); - assertEquals("user", path.get(2)); - assertEquals("file.txt", path.get(3)); + void arrayConstructorMatchesSeparateConstructorArguments() { + PurePosixPath fromArray = new PurePosixPath(new String[]{"a/b", "c"}); + PurePosixPath fromArguments = new PurePosixPath("a/b", "c"); + assertEquals(fromArguments, fromArray); + assertEquals(new PurePosixPath(fromArray.toString()), fromArray); + assertTrue(new PurePosixPath(new String[]{"/a"}).isAbsolute()); } @Test - void testEquals() { - PurePosixPath path1 = new PurePosixPath("/home/./user/file.txt"); - PurePosixPath path2 = new PurePosixPath("/home/user/./file.txt"); - assertEquals(path1, path2); + void javaSpecificFactoriesPreserveTheStdlibContract() { + assertEquals("a/b", PurePosixPath.from("a", "b").toString()); + assertEquals("a/b", PurePosixPath.from(Path.of("a", "b")).toString()); + assertEquals("/", PurePosixPath.SEP); } @Test - void testHashCode() { - PurePosixPath path1 = new PurePosixPath("/home/user/file.txt"); - PurePosixPath path2 = new PurePosixPath("/home/user/file.txt"); - assertEquals(path1.hashCode(), path2.hashCode()); + void valueIsImmutable() throws NoSuchFieldException { + int classModifiers = PurePosixPath.class.getModifiers(); + int partsModifiers = PurePosixPath.class.getDeclaredField("parts").getModifiers(); + assertTrue(java.lang.reflect.Modifier.isFinal(classModifiers)); + assertTrue(java.lang.reflect.Modifier.isPrivate(partsModifiers)); + assertThrows(UnsupportedOperationException.class, + () -> new PurePosixPath("a/b").parts().set(0, "changed")); } @Test - void invalidPath() { - assertThrows(NullPointerException.class, () -> new PurePosixPath(null)); + void nullInputsAreRejected() { + assertThrows(NullPointerException.class, () -> new PurePosixPath((String) null)); + assertThrows(NullPointerException.class, () -> new PurePosixPath((String[]) null)); + assertThrows(NullPointerException.class, () -> new PurePosixPath("a", (String[]) null)); } -} \ No newline at end of file +} diff --git a/java/src/test/java/com/esamtrade/bucketbase/PurePosixPrefixTest.java b/java/src/test/java/com/esamtrade/bucketbase/PurePosixPrefixTest.java new file mode 100644 index 0000000..9043389 --- /dev/null +++ b/java/src/test/java/com/esamtrade/bucketbase/PurePosixPrefixTest.java @@ -0,0 +1,71 @@ +package com.esamtrade.bucketbase; + +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class PurePosixPrefixTest { + + @Test + void emptyPrefixIsNotTheDotPath() { + PurePosixPrefix empty = new PurePosixPrefix(); + assertEquals("", empty.toString()); + assertEquals(List.of(), empty.parts()); + assertEquals("", empty.name()); + assertTrue(empty.isEmpty()); + assertEquals(".", empty.toPath().toString()); + } + + @Test + void trailingSeparatorIsPreservedAsAnEmptyFinalPart() { + PurePosixPrefix prefix = new PurePosixPrefix("dir/"); + assertEquals("dir/", prefix.toString()); + assertEquals(List.of("dir", ""), prefix.parts()); + assertEquals("", prefix.get(1)); + assertEquals("", prefix.name()); + assertTrue(prefix.hasTrailingSeparator()); + assertNotEquals(prefix, new PurePosixPrefix("dir")); + } + + @Test + void pathConversionIsExplicitlyLossyOnlyForTrailingSeparator() { + PurePosixPath path = new PurePosixPath("dir/file.txt"); + assertEquals(new PurePosixPrefix("dir/file.txt"), path.toPrefix()); + assertEquals(path.toPrefix(), new PurePosixPrefix(path)); + assertEquals(path, path.toPrefix().toPath()); + assertEquals(new PurePosixPath("dir"), new PurePosixPrefix("dir/").toPath()); + } + + @Test + void joinResultTypeFollowsTheFinalOperandType() { + PurePosixPrefix base = new PurePosixPrefix("tenant/"); + assertEquals(new PurePosixPath("tenant/data"), base.join("data/")); + assertEquals(new PurePosixPrefix("tenant/data/"), base.joinPrefix("data/")); + assertEquals(new PurePosixPrefix("tenant/data/"), + base.join(new PurePosixPrefix("data/"))); + assertEquals(new PurePosixPath("tenant/data/file.parquet"), + base.join(new PurePosixPath("data/file.parquet"))); + assertEquals(new PurePosixPrefix("tenant/data/"), + new PurePosixPath("tenant").join(new PurePosixPrefix("data/"))); + } + + @Test + void absoluteJoinOperandReplacesEarlierComponents() { + PurePosixPrefix base = new PurePosixPrefix("tenant/"); + assertEquals(new PurePosixPrefix("/other/"), base.joinPrefix("/other/")); + assertEquals(new PurePosixPath("/other/file"), base.join(new PurePosixPath("/other/file"))); + assertTrue(new PurePosixPrefix("/other/").isAbsolute()); + assertFalse(base.isAbsolute()); + } + + @Test + void redundantSeparatorsAndDotsCollapseButDotDotRemainsLexical() { + assertEquals(new PurePosixPrefix("a/b/"), new PurePosixPrefix("a//./b//")); + assertEquals(new PurePosixPrefix("a/../b/"), new PurePosixPrefix("a/../b/")); + } +} diff --git a/java/src/test/java/com/esamtrade/bucketbase/RangeSeekableInputStreamTest.java b/java/src/test/java/com/esamtrade/bucketbase/RangeSeekableInputStreamTest.java new file mode 100644 index 0000000..00adc89 --- /dev/null +++ b/java/src/test/java/com/esamtrade/bucketbase/RangeSeekableInputStreamTest.java @@ -0,0 +1,118 @@ +package com.esamtrade.bucketbase; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class RangeSeekableInputStreamTest { + private record RangeCall(long offset, int length) { + } + + @Test + void bufferedReadsReuseOneBoundedRange() throws IOException { + byte[] content = "0123456789abcdefghijklmnopqrstuvwxyz".getBytes(); + List calls = new ArrayList<>(); + RangeSeekableInputStream stream = createStream(content, 8, calls); + + stream.seek(3); + assertArrayEquals("34".getBytes(), stream.readNBytes(2)); + stream.seek(6); + assertArrayEquals("67".getBytes(), stream.readNBytes(2)); + + assertEquals(List.of(new RangeCall(3, 8)), calls); + } + + @Test + void zeroBufferUsesExactRangesAndSupportsSeeking() throws IOException { + byte[] content = "0123456789abcdefghijklmnopqrstuvwxyz".getBytes(); + List calls = new ArrayList<>(); + RangeSeekableInputStream stream = createStream(content, 0, calls); + + stream.seek(content.length - 4L); + assertArrayEquals("wxyz".getBytes(), stream.readNBytes(8)); + assertEquals(content.length, stream.position()); + assertEquals(-1, stream.read()); + stream.seek(content.length + 10L); + assertEquals(-1, stream.read()); + + assertEquals(List.of(new RangeCall(content.length - 4L, 4)), calls); + } + + @Test + void largeReadIsFilledDirectlyWithoutEnlargement() throws IOException { + byte[] content = "0123456789abcdefghijklmnopqrstuvwxyz".getBytes(); + List calls = new ArrayList<>(); + RangeSeekableInputStream stream = createStream(content, 8, calls); + + assertArrayEquals(Arrays.copyOf(content, 9), stream.readNBytes(9)); + + assertEquals(List.of(new RangeCall(0, 9)), calls); + } + + @Test + void markAndResetRestorePosition() throws IOException { + byte[] content = "0123456789".getBytes(); + RangeSeekableInputStream stream = createStream(content, 0, new ArrayList<>()); + + stream.seek(2); + stream.mark(0); + assertArrayEquals("234".getBytes(), stream.readNBytes(3)); + stream.reset(); + assertArrayEquals("234".getBytes(), stream.readNBytes(3)); + } + + @Test + void availableReportsOnlyBufferedBytes() throws IOException { + byte[] content = "0123456789abcdef".getBytes(); + RangeSeekableInputStream stream = createStream(content, 8, new ArrayList<>()); + + // Nothing prefetched yet: no bytes are readable without a fetch. + assertEquals(0, stream.available()); + stream.read(); + // One 8-byte range is now buffered; 7 remain ahead of the position. + assertEquals(7, stream.available()); + } + + @Test + void aFailingRangeReaderPropagates() { + RangeSeekableInputStream stream = new RangeSeekableInputStream(10, 0, + (offset, dest, destOffset, length) -> { + throw new IOException("boom"); + }); + assertThrows(IOException.class, () -> stream.readNBytes(10)); + } + + @Test + void closedStreamRejectsOperations() throws IOException { + RangeSeekableInputStream closedStream = createStream(new byte[10], 0, new ArrayList<>()); + closedStream.close(); + assertThrows(IOException.class, closedStream::read); + assertThrows(IOException.class, () -> closedStream.seek(0)); + assertThrows(IOException.class, closedStream::position); + } + + @Test + void invalidSizesAndNegativeSeekAreRejected() throws IOException { + assertThrows(IllegalArgumentException.class, + () -> new RangeSeekableInputStream(-1, 0, (offset, dest, destOffset, length) -> {})); + assertThrows(IllegalArgumentException.class, + () -> new RangeSeekableInputStream(1, -1, (offset, dest, destOffset, length) -> {})); + + RangeSeekableInputStream stream = createStream(new byte[1], 0, new ArrayList<>()); + assertThrows(IOException.class, () -> stream.seek(-1)); + } + + private static RangeSeekableInputStream createStream(byte[] content, int readBufferSize, List calls) { + return new RangeSeekableInputStream(content.length, readBufferSize, (offset, dest, destOffset, length) -> { + calls.add(new RangeCall(offset, length)); + System.arraycopy(content, Math.toIntExact(offset), dest, destOffset, length); + }); + } +} diff --git a/java/src/test/java/com/esamtrade/bucketbase/S3BucketRangeTest.java b/java/src/test/java/com/esamtrade/bucketbase/S3BucketRangeTest.java new file mode 100644 index 0000000..6e7bb38 --- /dev/null +++ b/java/src/test/java/com/esamtrade/bucketbase/S3BucketRangeTest.java @@ -0,0 +1,91 @@ +package com.esamtrade.bucketbase; + +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class S3BucketRangeTest { + @Test + void publicStreamUsesHeadOnceAndBoundedConditionalGets() throws IOException { + byte[] content = "0123456789abcdefghijklmnopqrstuvwxyz".getBytes(); + FakeS3Client client = new FakeS3Client(content); + S3Bucket bucket = new S3Bucket(client, null, "test-bucket", 8); + + try (SeekableInputStream stream = bucket.getObjectStream(PurePosixPath.from("object.bin"))) { + stream.seek(5); + assertArrayEquals("56".getBytes(), stream.readNBytes(2)); + stream.seek(8); + assertArrayEquals("89".getBytes(), stream.readNBytes(2)); + } + + assertEquals(1, client.headRequests.size()); + assertEquals(1, client.getRequests.size()); + GetObjectRequest request = client.getRequests.get(0); + assertEquals("bytes=5-12", request.range()); + assertEquals("test-etag", request.ifMatch()); + assertNotNull(request.range()); + } + + @Test + void fullReadIsStillOneBoundedGetAndBufferSizeIsValidated() throws IOException { + byte[] content = "0123456789abcdefghijklmnopqrstuvwxyz".getBytes(); + FakeS3Client client = new FakeS3Client(content); + S3Bucket bucket = new S3Bucket(client, null, "test-bucket", 8); + + assertArrayEquals(content, bucket.getObject(PurePosixPath.from("object.bin"))); + + assertEquals(1, client.getRequests.size()); + assertEquals("bytes=0-35", client.getRequests.get(0).range()); + assertThrows(IllegalArgumentException.class, () -> new S3Bucket(client, null, "test-bucket", -1)); + } + + private static final class FakeS3Client implements S3Client { + private final byte[] content; + private final List headRequests = new ArrayList<>(); + private final List getRequests = new ArrayList<>(); + + private FakeS3Client(byte[] content) { + this.content = content; + } + + @Override + public HeadObjectResponse headObject(HeadObjectRequest request) { + headRequests.add(request); + return HeadObjectResponse.builder().contentLength((long) content.length).eTag("test-etag").build(); + } + + @Override + public ResponseInputStream getObject(GetObjectRequest request) { + getRequests.add(request); + String[] bounds = request.range().substring("bytes=".length()).split("-"); + int start = Integer.parseInt(bounds[0]); + int endInclusive = Integer.parseInt(bounds[1]); + byte[] response = Arrays.copyOfRange(content, start, endInclusive + 1); + return new ResponseInputStream<>(GetObjectResponse.builder().build(), new ByteArrayInputStream(response)); + } + + @Override + public String serviceName() { + return "s3"; + } + + @Override + public void close() { + } + } +} diff --git a/java/src/test/java/com/esamtrade/bucketbase/S3BucketSDKv1RangeTest.java b/java/src/test/java/com/esamtrade/bucketbase/S3BucketSDKv1RangeTest.java new file mode 100644 index 0000000..8ee1dd5 --- /dev/null +++ b/java/src/test/java/com/esamtrade/bucketbase/S3BucketSDKv1RangeTest.java @@ -0,0 +1,81 @@ +package com.esamtrade.bucketbase; + +import com.amazonaws.services.s3.AbstractAmazonS3; +import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.S3Object; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class S3BucketSDKv1RangeTest { + @Test + void publicStreamUsesMetadataOnceAndBoundedConditionalGets() throws IOException { + byte[] content = "0123456789abcdefghijklmnopqrstuvwxyz".getBytes(); + FakeAmazonS3 client = new FakeAmazonS3(content); + S3BucketSDKv1 bucket = new S3BucketSDKv1(client, "test-bucket", 8); + + try (SeekableInputStream stream = bucket.getObjectStream(PurePosixPath.from("object.bin"))) { + stream.seek(5); + assertArrayEquals("56".getBytes(), stream.readNBytes(2)); + stream.seek(8); + assertArrayEquals("89".getBytes(), stream.readNBytes(2)); + } + + assertEquals(1, client.metadataCalls); + assertEquals(1, client.getRequests.size()); + GetObjectRequest request = client.getRequests.get(0); + assertArrayEquals(new long[]{5, 12}, request.getRange()); + assertEquals(List.of("test-etag"), request.getMatchingETagConstraints()); + } + + @Test + void fullReadIsStillOneBoundedGetAndBufferSizeIsValidated() throws IOException { + byte[] content = "0123456789abcdefghijklmnopqrstuvwxyz".getBytes(); + FakeAmazonS3 client = new FakeAmazonS3(content); + S3BucketSDKv1 bucket = new S3BucketSDKv1(client, "test-bucket", 8); + + assertArrayEquals(content, bucket.getObject(PurePosixPath.from("object.bin"))); + + assertEquals(1, client.getRequests.size()); + assertArrayEquals(new long[]{0, 35}, client.getRequests.get(0).getRange()); + assertThrows(IllegalArgumentException.class, () -> new S3BucketSDKv1(client, "test-bucket", -1)); + } + + private static final class FakeAmazonS3 extends AbstractAmazonS3 { + private final byte[] content; + private final List getRequests = new ArrayList<>(); + private int metadataCalls; + + private FakeAmazonS3(byte[] content) { + this.content = content; + } + + @Override + public ObjectMetadata getObjectMetadata(String bucketName, String key) { + metadataCalls++; + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setContentLength(content.length); + metadata.setHeader("ETag", "test-etag"); + return metadata; + } + + @Override + public S3Object getObject(GetObjectRequest request) { + getRequests.add(request); + int start = Math.toIntExact(request.getRange()[0]); + int endExclusive = Math.toIntExact(request.getRange()[1] + 1); + S3Object object = new S3Object(); + object.setObjectContent(new ByteArrayInputStream(Arrays.copyOfRange(content, start, endExclusive))); + return object; + } + } +} diff --git a/java/src/test/java/com/esamtrade/bucketbase/S3BucketSDKv1StreamingTest.java b/java/src/test/java/com/esamtrade/bucketbase/S3BucketSDKv1StreamingTest.java new file mode 100644 index 0000000..51c3eb5 --- /dev/null +++ b/java/src/test/java/com/esamtrade/bucketbase/S3BucketSDKv1StreamingTest.java @@ -0,0 +1,235 @@ +package com.esamtrade.bucketbase; + +import com.amazonaws.services.s3.AbstractAmazonS3; +import com.amazonaws.services.s3.model.AbortMultipartUploadRequest; +import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; +import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; +import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; +import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.PutObjectResult; +import com.amazonaws.services.s3.model.UploadPartRequest; +import com.amazonaws.services.s3.model.UploadPartResult; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Regression tests for streamed uploads through the AWS SDK v1 backend. + * + *

SDK v1's {@code putObject(bucket, key, stream, metadata)} buffers the entire stream + * into memory when the metadata carries no content length, so it can compute one. That defeats the + * whole point of {@link IBucket#openWrite(PurePosixPath)}, which exists to stream payloads too + * large to hold in memory. The backend must therefore upload in bounded parts, exactly as + * {@link S3Bucket} does.

+ */ +class S3BucketSDKv1StreamingTest { + + private static final int PART_SIZE = 5 * 1024 * 1024; + private static final PurePosixPath OBJECT = PurePosixPath.from("dir/streamed.bin"); + + @Test + void streamedUploadUsesMultipartAndNeverBuffersTheWholeStream() throws IOException { + // Larger than one part, so a correct implementation must issue several uploadPart calls. + byte[] payload = payload(12 * 1024 * 1024); + CountingInputStream source = new CountingInputStream(payload); + FakeMultipartS3 client = new FakeMultipartS3(); + client.source = source; + S3BucketSDKv1 bucket = new S3BucketSDKv1(client, "test-bucket"); + + bucket.putObjectStream(OBJECT, source); + + assertEquals(0, client.singleShotPutObjectCalls, + "a streamed upload must not go through the buffering single-shot putObject"); + assertTrue(client.uploadedParts.size() >= 2, + "expected a multipart upload, got " + client.uploadedParts.size() + " part(s)"); + assertEquals(1, client.completedUploads.size(), "the multipart upload must be completed"); + + // The heart of the regression: the first part must be sent before the source is drained. + assertTrue(client.bytesReadWhenFirstPartUploaded >= 0, "no part was uploaded at all"); + assertTrue(client.bytesReadWhenFirstPartUploaded < payload.length, + "the whole stream was read (" + client.bytesReadWhenFirstPartUploaded + " of " + payload.length + + " bytes) before the first part was uploaded, i.e. it was buffered in memory"); + + assertArrayEquals(payload, client.assembled(), "the reassembled object must match the source bytes"); + } + + @Test + void everyPartExceptTheLastIsAtLeastTheMinimumPartSize() throws IOException { + // S3 rejects a non-final part smaller than 5 MiB with EntityTooSmall, and a queue-backed + // sink hands over whatever chunk sizes the caller wrote, so parts must be coalesced. + byte[] payload = payload(11 * 1024 * 1024); + FakeMultipartS3 client = new FakeMultipartS3(); + S3BucketSDKv1 bucket = new S3BucketSDKv1(client, "test-bucket"); + + bucket.putObjectStream(OBJECT, new ChunkedInputStream(payload, 8 * 1024)); + + List parts = client.uploadedParts; + for (int i = 0; i < parts.size() - 1; i++) { + assertTrue(parts.get(i).length >= PART_SIZE, + "part " + (i + 1) + " is only " + parts.get(i).length + " bytes; S3 requires >= " + PART_SIZE); + } + assertArrayEquals(payload, client.assembled()); + } + + @Test + void emptyStreamStillStoresAnObject() throws IOException { + FakeMultipartS3 client = new FakeMultipartS3(); + S3BucketSDKv1 bucket = new S3BucketSDKv1(client, "test-bucket"); + + bucket.putObjectStream(OBJECT, new ByteArrayInputStream(new byte[0])); + + assertEquals(1, client.completedUploads.size(), "an empty body must still produce a stored object"); + assertEquals(0, client.assembled().length); + } + + @Test + void aFailureMidUploadAbortsAndDoesNotComplete() { + FakeMultipartS3 client = new FakeMultipartS3(); + client.failOnPart = 2; + S3BucketSDKv1 bucket = new S3BucketSDKv1(client, "test-bucket"); + + assertThrows(IOException.class, + () -> bucket.putObjectStream(OBJECT, new ByteArrayInputStream(payload(12 * 1024 * 1024)))); + + assertEquals(0, client.completedUploads.size(), "a failed upload must not be completed"); + assertEquals(1, client.abortedUploads.size(), "a failed upload must be aborted so no partial object lingers"); + } + + private static byte[] payload(int size) { + byte[] data = new byte[size]; + for (int i = 0; i < size; i++) { + data[i] = (byte) (i % 251); + } + return data; + } + + /** Records how many bytes the uploader has pulled from the source so far. */ + private static final class CountingInputStream extends InputStream { + private final byte[] data; + private int pos; + + CountingInputStream(byte[] data) { + this.data = data; + } + + int bytesRead() { + return pos; + } + + @Override + public int read() { + return pos < data.length ? data[pos++] & 0xff : -1; + } + + @Override + public int read(byte[] dest, int off, int len) { + if (pos >= data.length) { + return -1; + } + int n = Math.min(len, data.length - pos); + System.arraycopy(data, pos, dest, off, n); + pos += n; + return n; + } + } + + /** Returns at most {@code chunk} bytes per read, like a queue-backed sink does. */ + private static final class ChunkedInputStream extends InputStream { + private final byte[] data; + private final int chunk; + private int pos; + + ChunkedInputStream(byte[] data, int chunk) { + this.data = data; + this.chunk = chunk; + } + + @Override + public int read() { + return pos < data.length ? data[pos++] & 0xff : -1; + } + + @Override + public int read(byte[] dest, int off, int len) { + if (pos >= data.length) { + return -1; + } + int n = Math.min(Math.min(len, chunk), data.length - pos); + System.arraycopy(data, pos, dest, off, n); + pos += n; + return n; + } + } + + private static final class FakeMultipartS3 extends AbstractAmazonS3 { + private final List uploadedParts = new ArrayList<>(); + private final List completedUploads = new ArrayList<>(); + private final List abortedUploads = new ArrayList<>(); + private int singleShotPutObjectCalls; + private int bytesReadWhenFirstPartUploaded = -1; + private int failOnPart = -1; + private CountingInputStream source; + + byte[] assembled() { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + uploadedParts.forEach(out::writeBytes); + return out.toByteArray(); + } + + @Override + public PutObjectResult putObject(String bucketName, String key, InputStream input, ObjectMetadata metadata) { + singleShotPutObjectCalls++; + return new PutObjectResult(); + } + + @Override + public InitiateMultipartUploadResult initiateMultipartUpload(InitiateMultipartUploadRequest request) { + InitiateMultipartUploadResult result = new InitiateMultipartUploadResult(); + result.setUploadId("upload-1"); + return result; + } + + @Override + public UploadPartResult uploadPart(UploadPartRequest request) { + if (uploadedParts.size() + 1 == failOnPart) { + throw new IllegalStateException("simulated part upload failure"); + } + byte[] part; + try (InputStream in = request.getInputStream()) { + part = in.readNBytes(Math.toIntExact(request.getPartSize())); + } catch (IOException e) { + throw new RuntimeException(e); + } + uploadedParts.add(part); + if (bytesReadWhenFirstPartUploaded < 0 && source != null) { + bytesReadWhenFirstPartUploaded = source.bytesRead(); + } + UploadPartResult result = new UploadPartResult(); + result.setPartNumber(request.getPartNumber()); + result.setETag("etag-" + request.getPartNumber()); + return result; + } + + @Override + public CompleteMultipartUploadResult completeMultipartUpload(CompleteMultipartUploadRequest request) { + completedUploads.add(request.getUploadId()); + return new CompleteMultipartUploadResult(); + } + + @Override + public void abortMultipartUpload(AbortMultipartUploadRequest request) { + abortedUploads.add(request.getUploadId()); + } + } +} diff --git a/java/src/test/java/com/esamtrade/bucketbase/S3BucketSDKv1Test.java b/java/src/test/java/com/esamtrade/bucketbase/S3BucketSDKv1Test.java index 5317c32..b0f8dbd 100644 --- a/java/src/test/java/com/esamtrade/bucketbase/S3BucketSDKv1Test.java +++ b/java/src/test/java/com/esamtrade/bucketbase/S3BucketSDKv1Test.java @@ -1,32 +1,37 @@ package com.esamtrade.bucketbase; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIf; import java.io.IOException; +/** + * Functional tests for the AWS SDK v1 backend. See {@link S3BucketTest} for how the endpoint + * and credentials are configured. + */ +@EnabledIf("com.esamtrade.bucketbase.TestConfig#functionalTestsEnabled") class S3BucketSDKv1Test { private S3BucketSDKv1 bucket; private IBucketTester tester; - @BeforeAll - public static void setUpClass() { - - } - @BeforeEach public void setUp() { - String accessKey = System.getenv("MINIO_ACCESS_KEY"); - String secretKey = System.getenv("MINIO_SECRET_KEY"); - bucket = new S3BucketSDKv1("https://minio.esamtrade.vlada.ro", accessKey, secretKey, "minio-dev-tests"); + bucket = new S3BucketSDKv1( + TestConfig.MINIO_ENDPOINT, + TestConfig.MINIO_ACCESS_KEY, + TestConfig.MINIO_SECRET_KEY, + TestConfig.MINIO_BUCKET); + TestConfig.ensureBucketExists(bucket); tester = new IBucketTester(bucket); } @AfterEach void tearDown() throws IOException { - tester.cleanup(); + if (tester != null) { + tester.cleanup(); + } } @@ -40,6 +45,11 @@ void putObjectAndGetObjectStream() throws IOException { tester.testPutAndGetObjectStream(); } + @Test + void getObjectStreamIsSeekable() throws IOException { + tester.testGetObjectStreamIsSeekable(); + } + @Test void getListObjects() throws IOException { tester.testListObjects(); @@ -69,4 +79,4 @@ void testExists() throws IOException { void testRemoveObjects() throws IOException { tester.testRemoveObjects(); } -} \ No newline at end of file +} diff --git a/java/src/test/java/com/esamtrade/bucketbase/S3BucketTest.java b/java/src/test/java/com/esamtrade/bucketbase/S3BucketTest.java index 506523c..313ac55 100644 --- a/java/src/test/java/com/esamtrade/bucketbase/S3BucketTest.java +++ b/java/src/test/java/com/esamtrade/bucketbase/S3BucketTest.java @@ -1,33 +1,39 @@ package com.esamtrade.bucketbase; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIf; import java.io.IOException; +/** + * Functional tests against a live S3-compatible service. Endpoint and credentials come from + * {@link TestConfig}, which defaults to the public MinIO playground so this runs without any + * private secrets. Set {@code BUCKETBASE_SKIP_FUNCTIONAL_TESTS=1} to skip in offline builds. + */ +@EnabledIf("com.esamtrade.bucketbase.TestConfig#functionalTestsEnabled") class S3BucketTest { private S3Bucket bucket; private IBucketTester tester; - @BeforeAll - public static void setUpClass() { - - } - @BeforeEach public void setUp() { - String accessKey = System.getenv("MINIO_ACCESS_KEY"); - String secretKey = System.getenv("MINIO_SECRET_KEY"); - bucket = new S3Bucket("https://minio.esamtrade.vlada.ro", accessKey, secretKey, "minio-dev-tests"); + bucket = new S3Bucket( + TestConfig.MINIO_ENDPOINT, + TestConfig.MINIO_ACCESS_KEY, + TestConfig.MINIO_SECRET_KEY, + TestConfig.MINIO_BUCKET); + TestConfig.ensureBucketExists(bucket); tester = new IBucketTester(bucket); } @AfterEach void tearDown() throws IOException { - tester.cleanup(); + if (tester != null) { + tester.cleanup(); + } } @@ -41,6 +47,11 @@ void putObjectAndGetObjectStream() throws IOException { tester.testPutAndGetObjectStream(); } + @Test + void getObjectStreamIsSeekable() throws IOException { + tester.testGetObjectStreamIsSeekable(); + } + @Test void getListObjects() throws IOException { tester.testListObjects(); @@ -70,4 +81,4 @@ void testExists() throws IOException { void testRemoveObjects() throws IOException { tester.testRemoveObjects(); } -} \ No newline at end of file +} diff --git a/java/src/test/java/com/esamtrade/bucketbase/S3RemoveAndCloseTest.java b/java/src/test/java/com/esamtrade/bucketbase/S3RemoveAndCloseTest.java new file mode 100644 index 0000000..023e859 --- /dev/null +++ b/java/src/test/java/com/esamtrade/bucketbase/S3RemoveAndCloseTest.java @@ -0,0 +1,97 @@ +package com.esamtrade.bucketbase; + +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; +import software.amazon.awssdk.services.s3.model.DeletedObject; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; +import software.amazon.awssdk.services.s3.model.S3Error; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Verifies {@link S3Bucket} behaviour that needs a controllable S3 client rather than a live + * service: per-key delete-error reporting and client-ownership on close. + */ +class S3RemoveAndCloseTest { + + @Test + void removeObjectsReturnsServerReportedErrors() throws Exception { + FakeDeleteClient client = new FakeDeleteClient(); + // "denied.bin" fails; "ok.bin" and the missing key succeed silently. + client.errorKey = "denied.bin"; + S3Bucket bucket = new S3Bucket(client, null, "bucket"); + + List errors = bucket.removeObjects(List.of( + PurePosixPath.from("ok.bin"), + PurePosixPath.from("denied.bin"))); + + assertEquals(1, errors.size()); + DeleteError error = errors.get(0); + assertEquals("denied.bin", error.name()); + assertEquals("AccessDenied", error.code()); + assertEquals("nope", error.message()); + } + + @Test + void removeObjectsBatchesAtOneThousandKeys() throws Exception { + FakeDeleteClient client = new FakeDeleteClient(); + S3Bucket bucket = new S3Bucket(client, null, "bucket"); + + List names = java.util.stream.IntStream.range(0, 2025) + .mapToObj(i -> PurePosixPath.from("k" + i + ".bin")) + .toList(); + assertTrue(bucket.removeObjects(names).isEmpty()); + + // 2025 keys must be split into 1000 + 1000 + 25. + assertEquals(List.of(1000, 1000, 25), client.batchSizes); + } + + @Test + void closeDoesNotShutDownAnInjectedClient() throws Exception { + FakeDeleteClient client = new FakeDeleteClient(); + S3Bucket bucket = new S3Bucket(client, null, "bucket"); + + bucket.close(); + + assertFalse(client.closed, "a client passed in by the caller must not be closed by the bucket"); + } + + private static final class FakeDeleteClient implements S3Client { + private final List batchSizes = new java.util.ArrayList<>(); + private String errorKey; + private boolean closed; + + @Override + public DeleteObjectsResponse deleteObjects(DeleteObjectsRequest request) { + List objects = request.delete().objects(); + batchSizes.add(objects.size()); + DeleteObjectsResponse.Builder response = DeleteObjectsResponse.builder(); + List deleted = new java.util.ArrayList<>(); + List errors = new java.util.ArrayList<>(); + for (ObjectIdentifier object : objects) { + if (object.key().equals(errorKey)) { + errors.add(S3Error.builder().key(object.key()).code("AccessDenied").message("nope").build()); + } else { + deleted.add(DeletedObject.builder().key(object.key()).build()); + } + } + return response.deleted(deleted).errors(errors).build(); + } + + @Override + public String serviceName() { + return "s3"; + } + + @Override + public void close() { + closed = true; + } + } +} diff --git a/java/src/test/java/com/esamtrade/bucketbase/StreamPipeTest.java b/java/src/test/java/com/esamtrade/bucketbase/StreamPipeTest.java new file mode 100644 index 0000000..37f2ef2 --- /dev/null +++ b/java/src/test/java/com/esamtrade/bucketbase/StreamPipeTest.java @@ -0,0 +1,102 @@ +package com.esamtrade.bucketbase; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * {@link StreamPipe} documents bounded capacity, i.e. a slow reader back-pressures the writer + * instead of letting data pile up in memory. That bound must be expressed in bytes: if it + * counts write calls instead, a single large {@code write(byte[])} - which is exactly what a + * Parquet writer flushing a row group does - sails straight past it. + */ +class StreamPipeTest { + + @Test + void aSingleLargeWriteIsBackPressuredUntilTheReaderDrains() throws Exception { + StreamPipe pipe = new StreamPipe(); + InputStream in = pipe.inputStream(); + OutputStream out = pipe.outputStream(); + + byte[] payload = payload(8 * 1024 * 1024); + CountDownLatch writeReturned = new CountDownLatch(1); + AtomicReference writeFailure = new AtomicReference<>(); + + Thread writer = new Thread(() -> { + try { + out.write(payload); // one big call, no reader consuming yet + pipe.finish(); + } catch (Throwable t) { + writeFailure.set(t); + } finally { + writeReturned.countDown(); + } + }, "test-writer"); + writer.setDaemon(true); + writer.start(); + + assertFalse(writeReturned.await(500, TimeUnit.MILLISECONDS), + "an 8 MiB write completed with nothing consuming it, so the pipe buffered it all " + + "- capacity is bounded by call count, not by bytes"); + + byte[] received = in.readAllBytes(); + assertTrue(writeReturned.await(5, TimeUnit.SECONDS), "the writer never finished after the reader drained"); + assertNull(writeFailure.get(), String.valueOf(writeFailure.get())); + assertArrayEquals(payload, received, "every byte must survive the chunking"); + } + + @Test + void smallWritesRoundTripInOrder() throws Exception { + StreamPipe pipe = new StreamPipe(); + InputStream in = pipe.inputStream(); + + Thread writer = new Thread(() -> { + try (OutputStream out = pipe.outputStream()) { + for (int i = 0; i < 1_000; i++) { + out.write(("line-" + i + "\n").getBytes()); + } + pipe.finish(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }, "test-writer"); + writer.setDaemon(true); + writer.start(); + + String received = new String(in.readAllBytes()); + writer.join(5_000); + assertTrue(received.startsWith("line-0\n")); + assertTrue(received.endsWith("line-999\n")); + assertEquals(1_000, received.lines().count()); + } + + @Test + void abortMakesTheReaderFail() throws Exception { + StreamPipe pipe = new StreamPipe(); + InputStream in = pipe.inputStream(); + pipe.outputStream().write("partial".getBytes()); + pipe.abort(new IllegalStateException("caller gave up")); + + IOException failure = org.junit.jupiter.api.Assertions.assertThrows(IOException.class, in::readAllBytes); + assertTrue(failure.getMessage().contains("aborted"), failure.getMessage()); + } + + private static byte[] payload(int size) { + byte[] data = new byte[size]; + for (int i = 0; i < size; i++) { + data[i] = (byte) (i % 251); + } + return data; + } +} diff --git a/java/src/test/java/com/esamtrade/bucketbase/TestConfig.java b/java/src/test/java/com/esamtrade/bucketbase/TestConfig.java new file mode 100644 index 0000000..40d8f61 --- /dev/null +++ b/java/src/test/java/com/esamtrade/bucketbase/TestConfig.java @@ -0,0 +1,96 @@ +package com.esamtrade.bucketbase; + +import com.amazonaws.services.s3.model.AmazonS3Exception; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.model.HeadBucketRequest; +import software.amazon.awssdk.services.s3.model.S3Exception; + +/** + * Test configuration, mirroring {@code python/tests/base_config.py}. + * + *

Defaults point at the public MinIO playground with its well-known demo credentials, so the + * functional tests run against a real S3-compatible service without any private secrets. Every + * value can be overridden with an environment variable to target a private instance.

+ */ +public final class TestConfig { + + private TestConfig() { + } + + /** Endpoint URL of the S3-compatible service under test. */ + public static final String MINIO_ENDPOINT = + envOrDefault("MINIO_ENDPOINT", "https://play.min.io"); + + public static final String MINIO_ACCESS_KEY = + envOrDefault("MINIO_ACCESS_KEY", "Q3AM3UQ867SPQQA43P2F"); + + public static final String MINIO_SECRET_KEY = + envOrDefault("MINIO_SECRET_KEY", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"); + + public static final String MINIO_BUCKET = + envOrDefault("MINIO_DEV_TESTS_BUCKET", "bucketbase-test"); + + /** + * Functional tests are opt-out rather than opt-in: they run by default against the public + * playground, and can be disabled with {@code BUCKETBASE_SKIP_FUNCTIONAL_TESTS=1} for + * offline or air-gapped builds. + */ + public static boolean functionalTestsEnabled() { + String skip = System.getenv("BUCKETBASE_SKIP_FUNCTIONAL_TESTS"); + return skip == null || skip.isBlank() || skip.equals("0") || skip.equalsIgnoreCase("false"); + } + + /** + * Ensures the configured bucket exists before an SDK v2 functional test uses it. + * + *

Creation is deliberately test-fixture behavior; constructing a production + * {@link S3Bucket} must not mutate S3 infrastructure.

+ */ + static void ensureBucketExists(S3Bucket bucket) { + HeadBucketRequest headRequest = HeadBucketRequest.builder() + .bucket(bucket.bucketName) + .build(); + try { + bucket.s3Client.headBucket(headRequest); + return; + } catch (S3Exception exception) { + if (exception.statusCode() != 404) { + throw exception; + } + } + + try { + bucket.s3Client.createBucket(CreateBucketRequest.builder() + .bucket(bucket.bucketName) + .build()); + } catch (S3Exception exception) { + // Another concurrently running test process may have created the bucket. + if (exception.statusCode() != 409) { + throw exception; + } + } + bucket.s3Client.headBucket(headRequest); + } + + /** Ensures the configured bucket exists before an SDK v1 functional test uses it. */ + static void ensureBucketExists(S3BucketSDKv1 bucket) { + if (!bucket.s3Client.doesBucketExistV2(bucket.bucketName)) { + try { + bucket.s3Client.createBucket(bucket.bucketName); + } catch (AmazonS3Exception exception) { + // Another concurrently running test process may have created the bucket. + if (exception.getStatusCode() != 409) { + throw exception; + } + } + } + if (!bucket.s3Client.doesBucketExistV2(bucket.bucketName)) { + throw new IllegalStateException("S3 test bucket is not accessible: " + bucket.bucketName); + } + } + + private static String envOrDefault(String name, String fallback) { + String value = System.getenv(name); + return value == null || value.isBlank() ? fallback : value; + } +} diff --git a/python/tests/test_pure_posix_path_stdlib_contract.py b/python/tests/test_pure_posix_path_stdlib_contract.py new file mode 100644 index 0000000..b855f45 --- /dev/null +++ b/python/tests/test_pure_posix_path_stdlib_contract.py @@ -0,0 +1,146 @@ +"""Executable oracle for the Java ``PurePosixPath`` port. + +Yes, this deliberately tests Python's standard-library ``pathlib.PurePosixPath``, +which would normally be absurd in a BucketBase test suite. Its purpose is to keep +the expectations in Java's ``PurePosixPathTest`` tied to the actual stdlib +behavior of the Python version running the BucketBase tests. +""" + +import sys +import unittest +from pathlib import PurePosixPath + +# Python 3.14 left-strips leading dots from the name before looking for the extension +# separator, which changes `suffix`/`suffixes`/`stem` for names that end in a dot or start +# with several. See CPython `pathlib`: `name.rfind('.')` guarded by `0 < i < len(name) - 1` +# (<= 3.13) versus `name.lstrip('.').rfind('.')` (>= 3.14). +DOTS_STRIPPED_BEFORE_SUFFIX = sys.version_info >= (3, 14) + +# Python 3.12 switched path ordering from the `parts` tuple (`_cparts`) to the path string +# split on "/" (`_parts_normcase`). That reverses how a preserved "//" root sorts against "/": +# as parts, "//" > "/"; as split strings, ("", "", "a") < ("", "a"). +ORDERING_SPLITS_THE_PATH_STRING = sys.version_info >= (3, 12) + + +class TestPurePosixPathStdlibContract(unittest.TestCase): + def test_construction_and_parts_match_stdlib(self) -> None: + absolute = PurePosixPath("/home/user") + self.assertEqual("/home/user", str(absolute)) + self.assertEqual(("/", "home", "user"), absolute.parts) + + relative = PurePosixPath("home", "user") + self.assertEqual("home/user", str(relative)) + self.assertEqual(("home", "user"), relative.parts) + + empty = PurePosixPath("") + self.assertEqual(".", str(empty)) + self.assertEqual((), empty.parts) + self.assertEqual(empty, PurePosixPath()) + + def test_redundant_separators_trailing_separators_and_dot_components_collapse(self) -> None: + self.assertEqual("/home/user", str(PurePosixPath("/home/user/"))) + self.assertEqual("home/user", str(PurePosixPath("home//./user/."))) + self.assertEqual(".", str(PurePosixPath("./"))) + self.assertEqual("/", str(PurePosixPath(".", "./home/./user", "/."))) + + def test_dot_dot_components_remain_lexical(self) -> None: + self.assertEqual("/home/user/..", str(PurePosixPath("/home/user/.."))) + self.assertEqual("/../x", str(PurePosixPath("/../x"))) + self.assertEqual("/a/../../x", str(PurePosixPath("/a/../../x"))) + self.assertEqual("a/..", str(PurePosixPath("a/.."))) + self.assertEqual("../x", str(PurePosixPath("../x"))) + + def test_later_absolute_segment_replaces_earlier_segments(self) -> None: + self.assertEqual("/b", str(PurePosixPath("/a", "/b"))) + self.assertEqual("/", str(PurePosixPath("/home/user", "/"))) + + base = PurePosixPath("/a") + self.assertEqual("/b", str(base.joinpath("/b"))) + self.assertEqual("/b", str(base.joinpath(PurePosixPath("/b")))) + + def test_parent_matches_stdlib(self) -> None: + self.assertEqual("/home", str(PurePosixPath("/home/user").parent)) + self.assertEqual("/", str(PurePosixPath("/home").parent)) + self.assertEqual("/", str(PurePosixPath("/").parent)) + self.assertEqual(".", str(PurePosixPath("leaf").parent)) + self.assertEqual(".", str(PurePosixPath("").parent)) + self.assertEqual("/home/user", str(PurePosixPath("/home/user/..").parent)) + + def test_name_matches_stdlib(self) -> None: + self.assertEqual("file.txt", PurePosixPath("/home/user/file.txt").name) + self.assertEqual("..", PurePosixPath("/home/user/..").name) + self.assertEqual("", PurePosixPath("/").name) + self.assertEqual("", PurePosixPath("").name) + + def test_absolute_detection_matches_stdlib(self) -> None: + self.assertTrue(PurePosixPath("/home/user").is_absolute()) + self.assertTrue(PurePosixPath("//server/share").is_absolute()) + self.assertFalse(PurePosixPath("home/user").is_absolute()) + self.assertFalse(PurePosixPath(".").is_absolute()) + + def test_is_relative_to_matches_stdlib(self) -> None: + self.assertTrue(PurePosixPath("/home/user").is_relative_to(PurePosixPath("/home"))) + self.assertFalse(PurePosixPath("/home/user").is_relative_to(PurePosixPath("/etc"))) + self.assertFalse(PurePosixPath("/a").is_relative_to(PurePosixPath(""))) + self.assertTrue(PurePosixPath("a").is_relative_to(PurePosixPath("."))) + self.assertTrue(PurePosixPath("../a").is_relative_to(PurePosixPath(".."))) + + def test_suffix_suffixes_and_stem_match_stdlib(self) -> None: + archive = PurePosixPath("archive.tar.gz") + self.assertEqual(".gz", archive.suffix) + self.assertEqual([".tar", ".gz"], archive.suffixes) + self.assertEqual("archive.tar", archive.stem) + + dot_file = PurePosixPath(".bashrc") + self.assertEqual("", dot_file.suffix) + self.assertEqual([], dot_file.suffixes) + self.assertEqual(".bashrc", dot_file.stem) + + final_dot = PurePosixPath("name.") + self.assertEqual("name.", final_dot.name) + if DOTS_STRIPPED_BEFORE_SUFFIX: + self.assertEqual(".", final_dot.suffix) + self.assertEqual(["."], final_dot.suffixes) + self.assertEqual("name", final_dot.stem) + else: + self.assertEqual("", final_dot.suffix) + self.assertEqual([], final_dot.suffixes) + self.assertEqual("name.", final_dot.stem) + + multiple_leading_dots = PurePosixPath("..bashrc") + self.assertEqual([], multiple_leading_dots.suffixes) + if DOTS_STRIPPED_BEFORE_SUFFIX: + self.assertEqual("", multiple_leading_dots.suffix) + self.assertEqual("..bashrc", multiple_leading_dots.stem) + else: + self.assertEqual(".bashrc", multiple_leading_dots.suffix) + self.assertEqual(".", multiple_leading_dots.stem) + + def test_exactly_two_leading_separators_are_preserved(self) -> None: + self.assertEqual("//server/share", str(PurePosixPath("//server/share"))) + self.assertEqual(("//", "server", "share"), PurePosixPath("//server/share").parts) + self.assertEqual("/server/share", str(PurePosixPath("///server/share"))) + + def test_equality_hashing_and_ordering_match_canonical_stdlib_paths(self) -> None: + empty = PurePosixPath("") + dot = PurePosixPath(".") + self.assertEqual(empty, dot) + self.assertEqual(hash(empty), hash(dot)) + self.assertEqual(PurePosixPath("a"), PurePosixPath("a/")) + + self.assertLess(PurePosixPath("a/b"), PurePosixPath("a-b")) + self.assertLess(PurePosixPath("a"), PurePosixPath("a/b")) + self.assertLess(PurePosixPath("/a"), PurePosixPath("a")) + self.assertLess(PurePosixPath("\ue000"), PurePosixPath("\U0001f600")) + + # A preserved "//" root is a distinct path from "/" on every version, but the two sort + # in opposite orders depending on how the interpreter compares paths. + self.assertNotEqual(PurePosixPath("//a"), PurePosixPath("/a")) + if ORDERING_SPLITS_THE_PATH_STRING: + self.assertLess(PurePosixPath("//a"), PurePosixPath("/a")) + else: + self.assertLess(PurePosixPath("/a"), PurePosixPath("//a")) + + +if __name__ == "__main__": + unittest.main()