Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions .github/workflows/gradle.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@ name: Java CI with Gradle

on:
push:
branches:
- main
pull_request:

jobs:
build:
Expand Down Expand Up @@ -36,9 +39,9 @@ jobs:
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
run: ./gradlew build -PossrhUsername=null -PossrhPassword=null --exclude-task spark-aws-messaging:publish --info --rerun-tasks
run: ./gradlew build --info --rerun-tasks
- name: Execute Sonar analysis
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
run: ./gradlew jacocoTestReport sonar -PossrhUsername=null -PossrhPassword=null --exclude-task spark-aws-messaging:build --info
run: ./gradlew jacocoTestReport sonar --exclude-task spark-aws-messaging:build --info
49 changes: 49 additions & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
name: Release to Maven Central

on:
push:
tags:
- 'v*'
workflow_dispatch:

jobs:
release:
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0

- name: Set up JDK 21
uses: actions/setup-java@v4
with:
java-version: 21
distribution: zulu
cache: gradle

- name: Grant execute permission for gradlew
run: chmod +x gradlew

- name: Build library
run: ./gradlew build --exclude-task spark-aws-messaging-example:test --info

- name: Upload deployment to Maven Central
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
JRELEASER_MAVENCENTRAL_STAGE: UPLOAD
JRELEASER_GPG_SECRET_KEY: ${{ secrets.JRELEASER_GPG_SECRET_KEY }}
JRELEASER_GPG_PASSPHRASE: ${{ secrets.JRELEASER_GPG_PASSPHRASE }}
MAVEN_CENTRAL_USERNAME: ${{ secrets.MAVEN_CENTRAL_USERNAME }}
MAVEN_CENTRAL_PASSWORD: ${{ secrets.MAVEN_CENTRAL_PASSWORD }}
run: ./gradlew :spark-aws-messaging:publishToMavenCentral --stacktrace

- name: Upload JReleaser metadata
uses: actions/upload-artifact@v4
with:
name: maven-central-upload-metadata
path: |
spark-aws-messaging/build/jreleaser/output.properties
spark-aws-messaging/build/staging-deploy
if-no-files-found: error
22 changes: 22 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,28 @@ This library is available at Maven Central repository, so you can reference it i

The IAM permissions needed for this library to write on a SQS queue are *sqs:GetQueueUrl* and *sqs:SendMessage*.

## Releasing to Maven Central

The `spark-aws-messaging` module is configured to stage artifacts locally and deploy them to Maven Central with JReleaser.

Create a release tag and run:

```bash
./gradlew :spark-aws-messaging:publishToMavenCentral
```

The release flow expects these environment variables:

- `GITHUB_TOKEN`
- `MAVEN_CENTRAL_USERNAME`
- `MAVEN_CENTRAL_PASSWORD`
- `JRELEASER_GPG_SECRET_KEY`
- `JRELEASER_GPG_PASSPHRASE`

GitHub Actions mirrors the same flow in `.github/workflows/release.yml`.

For GitHub Actions, the workflow uploads the deployment to Maven Central with `JRELEASER_MAVENCENTRAL_STAGE=UPLOAD` and leaves the final publication confirmation to be done manually in the Central Portal.

## Messaging delivery semantics and error handling

The sink is at least once so some messages might be duplicated. If something wrong happens when the data is being written by a worker node, Spark will retry the task in another node. Messages that have already been sent could be sent again.
Expand Down
38 changes: 26 additions & 12 deletions spark-aws-messaging/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,18 @@ plugins {
id 'java'
id 'java-library'
id 'maven-publish'
id 'signing'
id 'jacoco'
id "org.sonarqube" version "4.4.1.3373"
id 'org.jreleaser' version '1.22.0'
}

group = 'com.fabiogouw'
archivesBaseName = 'spark-aws-messaging'
version = '1.2.0'

base {
archivesName = 'spark-aws-messaging'
}

repositories {
mavenCentral()
}
Expand Down Expand Up @@ -55,7 +57,7 @@ java {
}

tasks.register('copyToLib', Copy) {
into "${buildDir}/libs/deps"
into layout.buildDirectory.dir('libs/deps')
from configurations.runtimeClasspath
}

Expand All @@ -75,7 +77,7 @@ test {
publishing {
publications {
mavenJava(MavenPublication) {
artifactId = archivesBaseName
artifactId = base.archivesName.get()
from components.java
versionMapping {
usage('java-api') {
Expand Down Expand Up @@ -112,21 +114,30 @@ publishing {

repositories {
maven {
credentials {
username = "$ossrhUsername"
password = "$ossrhPassword"
}

url = 'https://ossrh-staging-api.central.sonatype.com/service/local/staging/deploy/maven2/'
name = 'staging'
url = layout.buildDirectory.dir('staging-deploy')
}
}
}

signing {
sign publishing.publications.mavenJava
tasks.register('publishToMavenCentral') {
group = 'publishing'
description = 'Stages the Maven publication and deploys it to Maven Central with JReleaser.'
dependsOn 'publishAllPublicationsToStagingRepository', 'jreleaserDeploy'
}

jreleaser {
gitRootSearch = true
release {
github {
enabled = true
repoOwner = 'fabiogouw'
name = 'spark-aws-messaging'
skipTag = true
skipRelease = true
token = System.getenv('GITHUB_TOKEN')
}
}
signing {
pgp {
active = 'ALWAYS'
Expand All @@ -138,8 +149,11 @@ jreleaser {
mavenCentral {
sonatype {
active = 'ALWAYS'
namespace = 'com.fabiogouw'
url = 'https://central.sonatype.com/api/v1/publisher'
stagingRepository('build/staging-deploy')
username = System.getenv('MAVEN_CENTRAL_USERNAME')
password = System.getenv('MAVEN_CENTRAL_PASSWORD')
}
}
}
Expand Down
Loading