diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 6fc810ae9..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,372 +0,0 @@ -job-defaults: &job-defaults - working_directory: &working_directory - ~/starlarky - environment: - DIST_PATH: /tmp/dist - TEST_RESULTS_PATH: /tmp/test-results - MAVEN_OPTS: -Xmx6400m - GRAALVM_HOME: /home/circleci/starlarky/.graalvm - HOMEBREW_REQUIRED_RUBY_VERSION: 2.4.6 - LARKY_LIB_HOME: /home/circleci/.larky/lib - -machine-linux: &machine-linux - machine: - docker_layer_caching: false - image: ubuntu-2204:2023.10.1 - -install-java: &install-java - run: - name: Install OpenJDK17 and Maven - command: | - sudo apt-get update && sudo apt-get install openjdk-17-jdk maven - sudo update-alternatives --set java /usr/lib/jvm/java-17-openjdk-amd64/bin/java - sudo update-alternatives --set javac /usr/lib/jvm/java-17-openjdk-amd64/bin/javac - -install-gh: &install-gh - run: - name: Install Go and ghr - command: | - sudo apt-get update && sudo apt-get install golang -y - go get -u github.com/tcnksm/ghr - -install-graalvm: &install-graalvm - run: - name: Install GraalVM - command: | - [ ! -d $GRAALVM_HOME ] \ - && curl https://github.com/graalvm/graalvm-ce-builds/releases/download/jdk-20.0.2/graalvm-community-jdk-20.0.2_linux-x64_bin.tar.gz -O -J -L \ - && tar xfz graalvm-community-jdk-20.0.2_linux-x64_bin.tar.gz \ - && mv graalvm-community-openjdk-20.0.2+9.1 .graalvm \ - && rm graalvm-community-jdk-20.0.2_linux-x64_bin.tar.gz \ - && $GRAALVM_HOME/bin/gu install native-image - -build-quarkus-runnable: &build-quarkus-runnable - run: - name: Build binary for runlarky - command: | - if [ "${CIRCLE_TAG}" != "" ] ; - then - mvn versions:set-property -Dproperty=starlarky.version -DnewVersion=${CIRCLE_TAG} -pl runlarky - mvn versions:set -DnewVersion=${CIRCLE_TAG} -pl runlarky - fi - mvn package -Pnative -DskipTests - cp ./runlarky/target/larky-runner ./pylarky - no_output_timeout: 30m - -# Template to save Maven dependency cache to restore it in another jobs -save-maven-cache: &save-maven-cache - save_cache: - key: starlarky-{{ checksum "pom.xml" }}-{{ checksum "libstarlark/pom.xml" }}-{{ checksum "larky/pom.xml" }} - paths: - - ~/.m2 - -restore-maven-cache: &restore-maven-cache - restore_cache: - key: starlarky-{{ checksum "pom.xml" }}-{{ checksum "libstarlark/pom.xml" }}-{{ checksum "larky/pom.xml" }} - -save-graalvm-cache: &save-graalvm-cache - save_cache: - key: starlarky-{{ checksum ".circleci/config.yml" }} - paths: - - $GRAALVM_HOME - -restore-graalvm-cache: &restore-graalvm-cache - restore_cache: - key: starlarky-{{ checksum ".circleci/config.yml" }} - -setup-maven-distributions: &setup-maven-distributions - run: - name: Setup distrubution servers for larky - command: | - mkdir -p ~/.m2 - cat \< ~/.m2/settings.xml - - - - github - $GITHUB_USERNAME - $GITHUB_API_TOKEN - - - - EOF - -install_python: &install_python - run: - name: Install Python - command: | - git clone https://github.com/pyenv/pyenv.git ~/.pyenv - echo 'export PYENV_ROOT="$HOME/.pyenv"' >> $BASH_ENV - echo 'export PATH="$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH"' >> $BASH_ENV - ~/.pyenv/bin/pyenv install 3.10.5 - ~/.pyenv/bin/pyenv local 3.10.5 - curl -sSL https://install.python-poetry.org | python - - echo 'export PATH="$HOME/.local/bin:$PATH"' >> $BASH_ENV - -build_test_pylarky: &build_test_pylarky - run: - name: Build and test pylarky - command: | - set -x - poetry version ${CIRCLE_TAG} - poetry install > poetry_install.log - poetry run pytest pylarky/tests - poetry build - -persist-dist: &persist-dist - persist_to_workspace: - root: /tmp/dist - paths: - - "*" - -attach-dist: &attach-dist - attach_workspace: - at: /tmp/dist - -parameters: - nightly-security-scan: - type: boolean - default: false - -# === JOBS === -version: 2.1 -jobs: - build-and-test: - <<: *job-defaults - <<: *machine-linux - steps: - - checkout - - <<: *install-java - - <<: *restore-maven-cache - - run: mkdir -p $TEST_RESULTS_PATH/junit/ - - run: mkdir -p $TEST_RESULTS_PATH/coverage/ - - run: - name: Maven Build/Test - command: | - mvn clean install dependency:go-offline -T 2.0C -B - - <<: *save-maven-cache - - run: find . -type f -regex ".*/target/surefire-reports/.*xml" -exec cp {} $TEST_RESULTS_PATH/junit/ \; - - run: find . -type f -regex ".*/target/surefire-reports/.*-output.txt" -exec cp {} $TEST_RESULTS_PATH/junit/ \; - - run: find . -type f -regex ".*/target/site/.*" -exec cp --parents {} $TEST_RESULTS_PATH/coverage/ \; - - store_artifacts: - path: /tmp/test-results - - store_test_results: - path: /tmp/test-results - - build-and-test-docker: - <<: *job-defaults - machine: true - steps: - - checkout - - <<: *restore-maven-cache - - run: mkdir -p $TEST_RESULTS_PATH/junit/ - - run: mkdir -p $TEST_RESULTS_PATH/coverage/ - - run: docker-compose build - - run: docker-compose run local bash /src/build-and-test-java.sh - - <<: *save-maven-cache - - run: find . -type f -regex ".*/target/surefire-reports/.*xml" -exec cp {} $TEST_RESULTS_PATH/junit/ \; - - run: find . -type f -regex ".*/target/surefire-reports/.*-output.txt" -exec cp {} $TEST_RESULTS_PATH/junit/ \; - - run: find . -type f -regex ".*/target/site/.*" -exec cp --parents {} $TEST_RESULTS_PATH/coverage/ \; - - run: docker-compose run local bash /src/build-and-test-python.sh - - store_artifacts: - path: /tmp/test-results - - store_test_results: - path: /tmp/test-results - - build-dist-maven: - <<: *job-defaults - <<: *machine-linux - steps: - - checkout - - <<: *install-java - - <<: *restore-maven-cache - - <<: *setup-maven-distributions - - run: - name: Build and deploy libstarlark - command: | - mvn versions:set -DnewVersion=${CIRCLE_TAG} -pl libstarlark - mvn clean dependency:go-offline package -pl libstarlark -U -Dmaven.test.skip.exec -T 2.0C -B - mvn deploy -DskipTests -pl libstarlark -Pgithub - - run: - name: Build and deploy larky - command: | - mvn versions:set-property -Dproperty=libstarlark.version -DnewVersion=${CIRCLE_TAG} - mvn versions:set -DnewVersion=${CIRCLE_TAG} -pl larky - mvn clean dependency:go-offline package -pl larky -U -Dmaven.test.skip.exec -T 2.0C -B - mvn deploy -DskipTests -pl larky -Pgithub - - <<: *save-maven-cache - - build-dist-linux: - <<: *job-defaults - <<: *machine-linux - steps: - - checkout - - <<: *install-java - - <<: *setup-maven-distributions - - <<: *restore-maven-cache - - <<: *restore-graalvm-cache - - <<: *install-graalvm - - <<: *save-graalvm-cache - - <<: *build-quarkus-runnable - - <<: *install_python - - <<: *build_test_pylarky - - run: - name: Copy dist files - command: | - mkdir ${DIST_PATH} - cp ./runlarky/target/larky-runner ${DIST_PATH}/larky-linux - cp dist/*.whl ${DIST_PATH} - - <<: *persist-dist - - build-dist-macos: - <<: *job-defaults - macos: - xcode: 10.2.1 - steps: - - checkout - - run: - name: Install dev - command: | - brew install openjdk@17 - brew install maven - - run: - name: Install GRAALVM - command: | - brew install --cask graalvm/tap/graalvm-ce-java17 - # Brew is designed to pull latest versions for the formula - xattr -r -d com.apple.quarantine /Library/Java/JavaVirtualMachines/graalvm-ce-java17-* - # We can use java_home exe to set GRAALVM_HOME - echo 'export GRAALVM_HOME="$(/usr/libexec/java_home)"' >> $BASH_ENV - source $BASH_ENV - # add gu to PATH - ln -s "$GRAALVM_HOME/bin/gu" /usr/local/bin - gu install native-image - - <<: *setup-maven-distributions - - <<: *build-quarkus-runnable - - <<: *install_python - - <<: *build_test_pylarky - - run: - name: Copy dist files - command: | - mkdir -p ${DIST_PATH} - cp ./runlarky/target/larky-runner ${DIST_PATH}/larky-macos - cp dist/*.whl ${DIST_PATH} - - <<: *persist-dist - - publish-to-gh: - <<: *job-defaults - docker: - - image: cibuilds/github:0.13 - steps: - - <<: *attach-dist - - run: - name: Create GH release - command: | - mv ${DIST_PATH}/larky-linux ${DIST_PATH}/larky-${CIRCLE_TAG}-linux - # mv ${DIST_PATH}/larky-macos ${DIST_PATH}/larky-${CIRCLE_TAG}-macos - ghr -draft -t ${GITHUB_API_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -n ${CIRCLE_TAG} ${CIRCLE_TAG} ${DIST_PATH} - - publish-to-pypi: - <<: *job-defaults - docker: - - image: circleci/python - steps: - - <<: *attach-dist - - run: - name: Install twine - command: pip install twine - - run: - name: Upload packages - command: twine upload -u __token__ -p ${PYPI_TOKEN} ${DIST_PATH}/*.whl - scan-sast-pr: - <<: *machine-linux - <<: *job-defaults - environment: - SEMGREP_REPO_URL: << pipeline.project.git_url >> - SEMGREP_BRANCH: << pipeline.git.branch >> - SEMGREP_BASELINE_REF: master - SEMGREP_PR_ID: $CIRCLE_PR_NUMBER - steps: - - checkout - - <<: *restore-maven-cache - - run: - name: Semgrep PR Scan - command: docker-compose run --rm semgrep - scan-sast-full: - <<: *machine-linux - <<: *job-defaults - environment: - SEMGREP_REPO_URL: << pipeline.project.git_url >> - SEMGREP_BRANCH: << pipeline.git.branch >> - steps: - - checkout - - <<: *restore-maven-cache - - run: - name: Semgrep Full Scan - command: docker-compose run --rm semgrep - -workflows: - version: 2 - build_and_tests: - when: - not: << pipeline.parameters.nightly-security-scan >> - jobs: - - build-and-test - - build-dist-linux: - requires: - - build-and-test - - scan-sast-pr: - context: - - security-tools - - circleci - requires: - - build-and-test - filters: - tags: - ignore: /.*/ - - release: - when: - not: << pipeline.parameters.nightly-security-scan >> - jobs: - - build-dist-maven: - context: github - filters: - tags: - only: /^\d+\.\d+\.\d+(?:-[a-zA-Z0-9_.]+){0,1}$/ - branches: - ignore: /.*/ - - build-dist-linux: - context: github - filters: - tags: - only: /^\d+\.\d+\.\d+(?:-[a-zA-Z0-9_.]+){0,1}$/ - branches: - ignore: /.*/ - requires: - - build-dist-maven - - publish-to-gh: - context: github - filters: - tags: - only: /^\d+\.\d+\.\d+(?:-[a-zA-Z0-9_.]+){0,1}$/ - branches: - ignore: /.*/ - requires: - - build-dist-linux - - publish-to-pypi: - filters: - tags: - only: DEPRECATED - branches: - ignore: /.*/ - requires: - - build-dist-linux - scheduled-security-scan: - when: << pipeline.parameters.nightly-security-scan >> - jobs: - - scan-sast-full: - context: - - security-tools - - circleci - diff --git a/.github/.release-please-manifest.json b/.github/.release-please-manifest.json new file mode 100644 index 000000000..2557a1077 --- /dev/null +++ b/.github/.release-please-manifest.json @@ -0,0 +1,3 @@ +{ + ".": "0.15.2" +} \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index fbf5ead74..000000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,18 +0,0 @@ -# To get started with Dependabot version updates, you'll need to specify which -# package ecosystems to update and where the package manifests are located. -# Please see the documentation for all configuration options: -# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates - -version: 2 - -updates: - - package-ecosystem: "maven" - directory: "/" - schedule: - interval: "daily" - rebase-strategy: "disabled" - open-pull-requests-limit: 200 - - package-ecosystem: "docker" - directory: "/" - schedule: - interval: "daily" diff --git a/.github/release-please-config.json b/.github/release-please-config.json new file mode 100644 index 000000000..c4cf00ff2 --- /dev/null +++ b/.github/release-please-config.json @@ -0,0 +1,66 @@ +{ + "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json", + "packages": { + "." : { + "release-type": "simple" + } + }, + "changelog-sections": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "feature", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "perf", + "section": "Performance Improvements" + }, + { + "type": "revert", + "section": "Reverts" + }, + { + "type": "docs", + "section": "Documentation", + "hidden": false + }, + { + "type": "style", + "section": "Styles", + "hidden": false + }, + { + "type": "chore", + "section": "Miscellaneous Chores", + "hidden": false + }, + { + "type": "refactor", + "section": "Code Refactoring", + "hidden": false + }, + { + "type": "test", + "section": "Tests", + "hidden": false + }, + { + "type": "build", + "section": "Build System", + "hidden": false + }, + { + "type": "ci", + "section": "Continuous Integration", + "hidden": false + } + ], + "draft": false +} \ No newline at end of file diff --git a/.github/workflows/build-test-publish.yaml b/.github/workflows/build-test-publish.yaml new file mode 100644 index 000000000..4a9a03b2d --- /dev/null +++ b/.github/workflows/build-test-publish.yaml @@ -0,0 +1,161 @@ +--- +## This workflow implements the following jobs +## - detect on PR which libs are updated +## - build libs +## - for libs: it uses the shared ci lib workflow +## - package the jar with unit tests +## - run integrations tests +## - publish the jar with github tag if event is push.tag + +name: build test and publish + +on: + push: + tags: + - "**" + pull_request: + +permissions: + contents: write + pull-requests: read + id-token: write + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + HOST_M2_REPO: ./root-m2 + +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 + + - name: Cache GraalVM Home + uses: actions/cache@8b402f58fbc84540c8b491a91e594a4576fec3d7 # v5 + id: cache-graalvm + with: + path: | + ./.graalvm + key: ${{ runner.os }}-graalvm-${{ hashFiles('**/install_graalvm.sh') }} + restore-keys: | + ${{ runner.os }}-graalvm- + + - name: Cache M2 Repository + uses: actions/cache@8b402f58fbc84540c8b491a91e594a4576fec3d7 # v5 + with: + path: | + ${{ env.HOST_M2_REPO }} + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Package jars with tests + run: | + # we create the cache folder for the 1st run when action/cache won't restore it + mkdir -p "$HOST_M2_REPO" + make test_and_package + env: + GITHUB_TOKEN: ${{ secrets.VERYGOODBOT_CIAPPBUILDER_PAT }} + + - name: Upload target directory + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: maven-target + path: "**/target/" + retention-days: 1 + overwrite: "true" + + - name: Publish Test Report + uses: mikepenz/action-junit-report@a294a61c909bd8a4b563024a2faa28897fd53ebc # v6.1.0 + if: ${{ ! cancelled() }} + with: + report_paths: '**/target/surefire-reports/*.xml' + detailed_summary: true + group_suite: true + fail_on_failure: true + truncate_stack_traces: false + annotate_only: true + + build-py: + name: Build py + runs-on: ubuntu-latest + needs: build + steps: + - name: Checkout + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 + + - name: Infos + id: infos + if: ${{ github.ref_type == 'tag' }} + run: | + VERSION=${TAG##*-v} # remove everything up to and including -v + VERSION=${VERSION#v} # remove initial v if it's just v1.2.3 + echo "VERSION=$VERSION" >> $GITHUB_ENV + env: + TAG: ${{ github.ref_name }} + + - name: Download target directory + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 + with: + name: maven-target + + - name: Run tests for pylarky + run: | + make py_test_and_package + + - name: Upload dist directory + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: python-target + path: "**/dist/*.whl" + retention-days: 1 + overwrite: "true" + + attach-assets: + needs: [build, build-py] + if: ${{ github.ref_type == 'tag' }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 + + - name: Download java target directory + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 + with: + name: maven-target + + - name: Download python target directory + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 + with: + name: python-target + + - name: Infos + id: infos + run: | + VERSION=${TAG##*-v} # remove everything up to and including -v + VERSION=${VERSION#v} # remove initial v if it's just v1.2.3 + echo "VERSION=$VERSION" >> $GITHUB_ENV + env: + TAG: ${{ github.ref_name }} + + - name: Add version to distribution + run: | + mv runlarky/target/larky-linux runlarky/target/larky-${VERSION}-linux + + - name: Upload to Release + run: | + # Check if a release exists for this tag + if gh release view ${{ github.ref_name }} &>/dev/null; then + echo "Release exists, uploading artifact..." + gh release upload ${{ github.ref_name }} dist/pylarky-*.whl --clobber + gh release upload ${{ github.ref_name }} runlarky/target/larky-*-linux --clobber + else + echo "No release found for tag ${{ github.ref_name }}" + fi + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release-please.yaml b/.github/workflows/release-please.yaml new file mode 100644 index 000000000..c29597634 --- /dev/null +++ b/.github/workflows/release-please.yaml @@ -0,0 +1,25 @@ +--- +name: release-please + +on: + push: + branches: + - master + +jobs: + release-please: + runs-on: ubuntu-latest + steps: + - uses: actions/create-github-app-token@29824e69f54612133e76f7eaac726eef6c875baf # v2 + id: app-token + with: + app-id: ${{ secrets.VGS_GITHUB_CI_BOT_APP_ID }} + private-key: ${{ secrets.VGS_GITHUB_CI_BOT_APP_PEM_FILE }} + + - uses: googleapis/release-please-action@16a9c90856f42705d54a6fda1823352bdc62cf38 # v4 + id: release + with: + config-file: .github/release-please-config.json + manifest-file: .github/.release-please-manifest.json + include-component-in-tag: false + token: ${{ steps.app-token.outputs.token }} diff --git a/.mvn/maven.config b/.mvn/maven.config new file mode 100644 index 000000000..4313f27a9 --- /dev/null +++ b/.mvn/maven.config @@ -0,0 +1,2 @@ +--settings +./.mvn/settings.xml \ No newline at end of file diff --git a/.mvn/settings.xml b/.mvn/settings.xml new file mode 100644 index 000000000..23e3c53ed --- /dev/null +++ b/.mvn/settings.xml @@ -0,0 +1,23 @@ + + + + repo1 + central + Maven Central + https://repo1.maven.org/maven2/ + + + repo2 + central + Maven Central 2 + https://repo2.maven.org/maven2/ + + + + + github + ${env.GITHUB_ACTOR} + ${env.GITHUB_TOKEN} + + + \ No newline at end of file diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index b169044f1..000000000 --- a/Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -FROM ubuntu:lunar - -WORKDIR /build/ -ENV PYENV_ROOT="/root/.pyenv" -ENV PATH="/root/.poetry/bin:$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH" - -# java -RUN apt-get update -qq \ - && JVM_ARCH=$([ `lscpu | grep -o "aarch64"` ] && echo "arm64" || echo "amd64") \ - && apt-get install -y curl openjdk-17-jdk maven \ - && update-alternatives --set java /usr/lib/jvm/java-17-openjdk-${JVM_ARCH}/bin/java \ - && update-alternatives --set javac /usr/lib/jvm/java-17-openjdk-${JVM_ARCH}/bin/javac - -# ghr -RUN apt-get install golang git -y \ - && go install github.com/tcnksm/ghr@latest - -# graalvm -RUN GRAALVM_ARCH=$([ `lscpu | grep -o "aarch64"` ] && echo "aarch64" || echo "x64") \ - && curl https://github.com/graalvm/graalvm-ce-builds/releases/download/jdk-17.0.9/graalvm-community-jdk-17.0.9_linux-${GRAALVM_ARCH}_bin.tar.gz -O -J -L \ - && tar xfz graalvm-community-jdk-17.0.9_linux-${GRAALVM_ARCH}_bin.tar.gz \ - && mv graalvm-community-openjdk-17.0.9+9.1 .graalvm \ - && rm graalvm-community-jdk-17.0.9_linux-${GRAALVM_ARCH}_bin.tar.gz \ - && /build/.graalvm/bin/gu install native-image - -# python -RUN git clone https://github.com/pyenv/pyenv.git ~/.pyenv \ - && apt-get install -y gcc libbz2-dev libsqlite3-dev libssl-dev make zlib1g-dev libffi-dev \ - && ~/.pyenv/bin/pyenv install 3.10.5 \ - && ~/.pyenv/bin/pyenv local 3.10.5 \ - && curl -sSL https://install.python-poetry.org | python3 - - -WORKDIR /src diff --git a/Dockerfile.maven b/Dockerfile.maven new file mode 100644 index 000000000..64d2a87b8 --- /dev/null +++ b/Dockerfile.maven @@ -0,0 +1,10 @@ +FROM maven:3.8.5-openjdk-17 + +RUN microdnf install -y \ + gcc \ + gcc-c++ \ + make \ + glibc-devel \ + zlib-devel + +WORKDIR /app \ No newline at end of file diff --git a/Dockerfile.python b/Dockerfile.python new file mode 100644 index 000000000..af6e31cc9 --- /dev/null +++ b/Dockerfile.python @@ -0,0 +1,5 @@ +FROM python:3.10.5-slim-buster + +RUN pip install poetry twine + +WORKDIR /app \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..6491c8518 --- /dev/null +++ b/Makefile @@ -0,0 +1,13 @@ +test_and_package: + @echo "Running tests and packaging" + docker compose run --rm maven /bin/sh -c "./install_graalvm.sh && ./build-and-test-java.sh" + +publish: + @echo "Deploying jar" + docker compose run --rm maven /bin/sh -c "mvn -B -e versions:set -DnewVersion=$(VERSION) && mvn deploy -DskipTests" + +py_test_and_package: + @echo "Running tests and packaging: pylarky" + docker compose run --rm \ + -e VERSION \ + python /bin/sh -c "./build-and-test-python.sh" \ No newline at end of file diff --git a/README.md b/README.md index 265d612c1..d5e836441 100644 --- a/README.md +++ b/README.md @@ -82,8 +82,8 @@ _Pylarky_ is pip lib-wrapper for runlarky to make larky calls conveniently from ```bash docker-compose build -docker-compose run local bash /src/build-and-test-java.sh -docker-compose run local bash /src/build-and-test-python.sh +docker compose run --rm maven /bin/sh -c "./install_graalvm.sh && ./build-and-test-java.sh" +docker compose run --rm python /bin/sh -c "./build-and-test-python.sh" ``` ### Run individual larky stdlib test @@ -122,10 +122,4 @@ In addition to having Maven installed, it must be configured to retrieve artifac ``` ## Deployment process - -To rollout a new verion of libstarlark/larky/larky-api create a new tag -``` -git tag x.x.x -git push origin x.x.x -``` -Than, after CircleCI build, publish the draft release +Release-please plugin manages creation of tag&release. \ No newline at end of file diff --git a/build-and-test-java.sh b/build-and-test-java.sh index c6dabbe8e..663379384 100755 --- a/build-and-test-java.sh +++ b/build-and-test-java.sh @@ -1,14 +1,11 @@ -TEST_RESULTS_PATH=${TEST_RESULTS_PATH:-/tmp/test-results} +#!/usr/bin/env bash +cd "$(dirname "$0")" -mkdir -p $TEST_RESULTS_PATH/junit/ -mkdir -p $TEST_RESULTS_PATH/coverage/ -mvn clean install dependency:go-offline -T 2.0C -B -find . -type f -regex ".*/target/surefire-reports/.*xml" -exec cp {} $TEST_RESULTS_PATH/junit/ \; -find . -type f -regex ".*/target/surefire-reports/.*-output.txt" -exec cp {} $TEST_RESULTS_PATH/junit/ \; -find . -type f -regex ".*/target/site/.*" -exec cp --parents {} $TEST_RESULTS_PATH/coverage/ \; +OS_TYPE=${OS_TYPE:-$(uname)} +OS_TYPE_LOWER=$(echo "$OS_TYPE" | tr '[:upper:]' '[:lower:]') -# package it up to deliver -mvn package -Pnative -DskipTests -mkdir ${DIST_PATH} -cp ./runlarky/target/larky-runner ${DIST_PATH}/larky-linux -# cp dist/*.whl ${DIST_PATH} +# test and package +mvn package -Pnative + +# tag distribution +mv ./runlarky/target/larky-runner ./runlarky/target/larky-${OS_TYPE_LOWER} \ No newline at end of file diff --git a/build-and-test-python.sh b/build-and-test-python.sh index 51bfb942d..8973880e3 100755 --- a/build-and-test-python.sh +++ b/build-and-test-python.sh @@ -1,10 +1,15 @@ +#!/usr/bin/env bash set -ex -pyenv local 3.10.5 -PATH=${PATH}:~/.local/bin +cd "$(dirname "$0")" + +OS_TYPE=${OS_TYPE:-$(uname)} +OS_TYPE_LOWER=$(echo "$OS_TYPE" | tr '[:upper:]' '[:lower:]') +VERSION=${VERSION:-0.0.1} # You must run ./build-and-test-java.sh to build this -cp ./runlarky/target/larky-runner ./pylarky +cp ./runlarky/target/larky-${OS_TYPE_LOWER} ./pylarky/larky-runner +chmod +x ./pylarky/larky-runner # Run tests poetry version ${VERSION} diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 000000000..27db5989a --- /dev/null +++ b/compose.yaml @@ -0,0 +1,29 @@ +services: + + maven: + build: + context: . + dockerfile: Dockerfile.maven + platform: linux/amd64 + working_dir: /app + environment: + GITHUB_ACTOR: + GITHUB_TOKEN: + GRAALVM_HOME: /app/.graalvm + MAVEN_OPTS: -Xmx6400m + volumes: + - .:/app/ + - ${HOST_M2_REPO:-~/.m2}:/root/.m2 + - /var/run/docker.sock:/var/run/docker.sock + + python: + build: + context: . + dockerfile: Dockerfile.python + working_dir: /app + environment: + GITHUB_ACTOR: + GITHUB_TOKEN: + volumes: + - .:/app/ + - /var/run/docker.sock:/var/run/docker.sock diff --git a/docker-compose.yaml b/docker-compose.yaml deleted file mode 100644 index f3cb0d466..000000000 --- a/docker-compose.yaml +++ /dev/null @@ -1,39 +0,0 @@ -x-common: - image: &maven-jdk-image maven:3.9-eclipse-temurin-17 -services: - - local: - build: - context: . - dockerfile: Dockerfile - environment: - VERSION: ${CIRCLE_TAG:-0.0.1} - TEST_RESULTS_PATH: /tmp/test-results/ - DIST_PATH: /tmp/dist/ - MAVEN_OPTS: -Xmx6400m - GRAALVM_HOME: /build/.graalvm - PYENV_ROOT: /root/.pyenv - volumes: - - ./:/src/ - - ./test-results:/tmp/test-results/ - - ~/.m2:/root/.m2 - - semgrep: - image: *maven-jdk-image - command: bash -c "mvn dependency:tree -DoutputFile=maven_dep_tree.txt && - apt-get update && - apt-get -y install python3 && - apt-get -y install python3-pip && - python3 -m pip install semgrep --break-system-packages && - git config --global --add safe.directory $$(pwd) && - semgrep ci --no-suppress-errors" - working_dir: /app - environment: - SEMGREP_REPO_NAME: $CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME - SEMGREP_REPO_URL: ${SEMGREP_REPO_URL} - SEMGREP_BRANCH: ${SEMGREP_BRANCH} - SEMGREP_PR_ID: $CIRCLE_PR_NUMBER - SEMGREP_APP_TOKEN: ${SEMGREP_APP_TOKEN} - volumes: - - .:/app/ - - ~/.m2/:/root/.m2/ diff --git a/install_graalvm.sh b/install_graalvm.sh new file mode 100755 index 000000000..9240baa0a --- /dev/null +++ b/install_graalvm.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -e + +cd "$(dirname "$0")" +GRAALVM_DIR=${GRAALVM_DIR:=./.graalvm} + +OS_TYPE=${OS_TYPE:-$(uname)} +if [ "$OS_TYPE" == "Darwin" ]; then + # For local development. + echo "macOS detected." + GRAALVM_URL="https://github.com/graalvm/graalvm-ce-builds/releases/download/jdk-20.0.2/graalvm-community-jdk-20.0.2_macos-aarch64_bin.tar.gz" + GRAALVM_PACKAGE="graalvm-community-openjdk-20.0.2+9.1" + GRAALVM_BIN=${GRAALVM_DIR}/Contents/Home/bin +elif [ "$OS_TYPE" == "Linux" ]; then + # For CI/CD builds and MakeFile usage + echo "Linux detected." + GRAALVM_URL="https://github.com/graalvm/graalvm-ce-builds/releases/download/jdk-20.0.2/graalvm-community-jdk-20.0.2_linux-x64_bin.tar.gz" + GRAALVM_PACKAGE="graalvm-community-openjdk-20.0.2+9.1" + GRAALVM_BIN=${GRAALVM_DIR}/bin +else + echo "Unsupported OS: $OS_TYPE" + exit 1 +fi + +if [ ! -d "$GRAALVM_DIR" ]; then + echo "GraalVM exists in '$(pwd)'. Updating..." + curl -o graalvm.tar.gz -J -L "$GRAALVM_URL" + tar xfz graalvm.tar.gz + mv $GRAALVM_PACKAGE .graalvm + rm graalvm.tar.gz + $GRAALVM_BIN/gu install native-image +fi \ No newline at end of file diff --git a/larky/pom.xml b/larky/pom.xml index ab3d82c3c..6c107d2de 100644 --- a/larky/pom.xml +++ b/larky/pom.xml @@ -11,6 +11,13 @@ jar + + + github + https://maven.pkg.github.com/verygoodsecurity/starlarky + + + maven @@ -310,16 +317,4 @@ - - - - github - - - github - https://maven.pkg.github.com/verygoodsecurity/starlarky - - - - diff --git a/libstarlark/pom.xml b/libstarlark/pom.xml index 779c70944..19d5ba269 100644 --- a/libstarlark/pom.xml +++ b/libstarlark/pom.xml @@ -11,6 +11,13 @@ jar + + + github + https://maven.pkg.github.com/verygoodsecurity/starlarky + + + 3.14.0 3.3.1 @@ -207,16 +214,4 @@ - - - - github - - - github - https://maven.pkg.github.com/verygoodsecurity/starlarky - - - -