diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..be89180 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,14 @@ +# Only container runtime sources belong in the build context. +** +!Dockerfile +!LICENSE +!docker/ +!docker/*.php +!docker/*.sh +!docker/php-fpm.conf +!docker/nginx.conf.template +!docker/health.txt +!moodle/ +!moodle/.config/ +!moodle/.config/php/ +!moodle/.config/php/mooship.ini diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..1caa4ca --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +# Container entry points and Linux lifecycle scripts must retain Unix line endings. +*.sh text eol=lf +Dockerfile text eol=lf diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 73ce359..9fec496 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -15,6 +15,20 @@ updates: commit-message: prefix: ci + - package-ecosystem: pip + directory: / + cooldown: + default-days: 7 + schedule: + interval: weekly + day: monday + groups: + package-build-tools: + patterns: ['*'] + open-pull-requests-limit: 5 + commit-message: + prefix: build + - package-ecosystem: pip directory: /.github cooldown: @@ -31,6 +45,7 @@ updates: - package-ecosystem: docker directories: + - / - /moodle/.docker/php - /moodle/.docker/nginx - /nginxproxy @@ -48,7 +63,9 @@ updates: prefix: build - package-ecosystem: docker-compose - directory: /moodle + directories: + - /moodle + - /docker # Database branch changes require coordinated migration and restore checks. ignore: - dependency-name: mysql diff --git a/.github/scripts/build_release.py b/.github/scripts/build_release.py new file mode 100644 index 0000000..72b03cd --- /dev/null +++ b/.github/scripts/build_release.py @@ -0,0 +1,59 @@ +"""Build a deterministic native toolkit archive from Git's runtime-file inventory.""" + +import argparse +import gzip +import hashlib +import io +from pathlib import Path +import re +import subprocess +import tarfile + +ROOT = Path(__file__).resolve().parents[2] +FILES = {"VERSION", "LICENSE", "README.md", "deploy-moodle.sh", "activate.sh", "deactivate.sh", + "remove.sh", "list-projects.sh", "backup.sh", "restore.sh", "schedule-backup.sh", "doctor.sh"} +DIRECTORIES = {"lib", "moodle", "nginxproxy"} + + +def build(root: Path, output: Path) -> Path: + version = (root / "VERSION").read_text(encoding="ascii").strip() + if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", version): + raise ValueError("VERSION must contain an exact X.Y.Z release") + inventory = subprocess.run(["git", "-C", str(root), "ls-files", "-z"], + check=True, capture_output=True, timeout=30).stdout + names = sorted(name for name in inventory.decode("utf-8").split("\0") if name + and (name in FILES or name.split("/", 1)[0] in DIRECTORIES)) + if not FILES.issubset(names): + raise ValueError("Required toolkit files are missing from Git's inventory") + output.mkdir(parents=True, exist_ok=True) + archive = output / f"mooship-{version}.tar.gz" + # Exclusive output prevents an accidental local overwrite; retries use a fresh directory. + with archive.open("xb") as destination, gzip.GzipFile(fileobj=destination, mode="wb", mtime=0, filename="") as gz: + with tarfile.open(fileobj=gz, mode="w|") as tar: + for name in names: + path = root / name + if path.is_symlink() or not path.is_file(): + raise ValueError(f"Toolkit source must be a regular file: {name}") + content = path.read_bytes() + if path.suffix == ".sh": + content = content.replace(b"\r\n", b"\n") + member = tarfile.TarInfo(f"mooship-{version}/{name}") + member.size = len(content) + member.mode = 0o755 if path.suffix == ".sh" else 0o644 + tar.addfile(member, io.BytesIO(content)) + digest = hashlib.sha256(archive.read_bytes()).hexdigest() + archive.with_name(archive.name + ".sha256").write_text( + f"{digest} {archive.name}\n", encoding="ascii", + ) + return archive + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output", type=Path) + args = parser.parse_args() + print(build(ROOT, args.output)) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/check_image_tags.py b/.github/scripts/check_image_tags.py new file mode 100644 index 0000000..7871efd --- /dev/null +++ b/.github/scripts/check_image_tags.py @@ -0,0 +1,29 @@ +"""Refuse to overwrite existing Docker Hub release tags before building/pushing.""" + +import os +import re +from urllib.error import HTTPError +from urllib.request import urlopen + + +def require_new_tag(tag: str) -> None: + if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+-moodle[0-9.]+-(?:php8\.3|nginx)", tag): + raise ValueError("Invalid Mooship release image tag") + try: + # Fixed HTTPS origin and validated tag prevent caller-supplied URL schemes. + with urlopen(f"https://hub.docker.com/v2/repositories/shellharbor/mooship/tags/{tag}", timeout=30): # nosec B310 + pass + except HTTPError as exc: + if exc.code == 404: + return + raise + raise ValueError(f"Docker Hub tag already exists; refusing to overwrite {tag}") + + +def main() -> None: + for suffix in ("php8.3", "nginx"): + require_new_tag(f"{os.environ['VERSION']}-moodle{os.environ['MOODLE_VERSION']}-{suffix}") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/check_release.py b/.github/scripts/check_release.py new file mode 100644 index 0000000..4bd9ac8 --- /dev/null +++ b/.github/scripts/check_release.py @@ -0,0 +1,49 @@ +"""Require matching release metadata and successful checks on this exact commit.""" + +import json +import os +from pathlib import Path +import re +import subprocess + +REQUIRED = {"CI", "Configuration", "Component integration", "Security", "CodeQL", "Documentation", + "Moodle images", "Python package"} + + +def validate_tag(tag: str, version: str) -> None: + if not re.fullmatch(r"v[0-9]+\.[0-9]+\.[0-9]+", tag) or tag != f"v{version}": + raise ValueError("Release tag must be vX.Y.Z and match VERSION") + + +def check_runs(runs: list, commit: str) -> None: + latest = {} + for run in sorted(runs, key=lambda item: item["id"], reverse=True): + if run["head_sha"] == commit: + latest.setdefault(run["name"], run) + failed = sorted(name for name in REQUIRED if name not in latest + or latest[name]["status"] != "completed" or latest[name]["conclusion"] != "success") + if failed: + raise ValueError("Missing successful exact-commit workflows: " + ", ".join(failed)) + + +def command(*args: str) -> str: + return subprocess.run(list(args), check=True, capture_output=True, text=True, timeout=60).stdout.strip() + + +def main() -> None: + version = Path("VERSION").read_text(encoding="ascii").strip() + validate_tag(os.environ["RELEASE_TAG"], version) + commit = command("git", "rev-parse", "HEAD") + tagged = command("git", "rev-parse", "--verify", f"refs/tags/{os.environ['RELEASE_TAG']}^{{commit}}") + if tagged != commit: + raise ValueError("Checkout does not match the exact release tag commit") + runs = json.loads(command("gh", "api", f"repos/{os.environ['GITHUB_REPOSITORY']}/actions/runs?head_sha={commit}&per_page=100")) + check_runs(runs["workflow_runs"], commit) + source = Path("Dockerfile").read_text(encoding="utf-8") + moodle = re.search(r"^ARG MOODLE_VERSION=([0-9.]+)$", source, re.MULTILINE).group(1) + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: + output.write(f"commit={commit}\nversion={version}\nmoodle={moodle}\n") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/publish_assets.py b/.github/scripts/publish_assets.py new file mode 100644 index 0000000..a947c13 --- /dev/null +++ b/.github/scripts/publish_assets.py @@ -0,0 +1,35 @@ +"""Upload missing native release assets and refuse conflicting existing content.""" + +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile + + +def command(*args: str) -> str: + return subprocess.run(list(args), capture_output=True, text=True, check=True, timeout=120).stdout + + +def main() -> None: + tag = os.environ["RELEASE_TAG"] + assets = json.loads(command("gh", "release", "view", tag, "--json", "assets"))["assets"] + existing = {asset["name"] for asset in assets} + for path in sorted(Path(sys.argv[1]).iterdir()): + if not path.is_file() or not (path.name.endswith(".tar.gz") or path.name.endswith(".sha256")): + continue + if path.name in existing: + with tempfile.TemporaryDirectory() as directory: + command("gh", "release", "download", tag, "--pattern", path.name, "--dir", directory) + actual = hashlib.sha256((Path(directory) / path.name).read_bytes()).digest() + if actual != hashlib.sha256(path.read_bytes()).digest(): + raise ValueError(f"Existing release asset differs; refusing to replace {path.name}") + else: + command("gh", "release", "upload", tag, str(path)) + print(f"Verified release asset: {path.name}") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e846801..68a4ce5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,9 +42,9 @@ jobs: - name: Install pinned CI tools run: python -m pip install -r .github/requirements.txt - name: Python lint - run: python -m ruff check lib tests .github/scripts + run: python -m ruff check lib src tests .github/scripts - name: YAML syntax and duplicate keys - run: python -m yamllint .github + run: python -m yamllint .github docker deploy/k8s - name: GitHub Actions syntax run: docker run --rm --read-only --network none -v "$PWD:/repo:ro" -w /repo rhysd/actionlint:1.7.12 -color - name: GitHub Actions security diff --git a/.github/workflows/configuration.yml b/.github/workflows/configuration.yml index 110cdf2..eb83f39 100644 --- a/.github/workflows/configuration.yml +++ b/.github/workflows/configuration.yml @@ -56,10 +56,10 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - name: Check every PHP and nginx Dockerfile + - name: Check every Dockerfile # Existing package-version/apt style warnings remain visible; errors fail the job. run: | while IFS= read -r -d '' file; do echo "Checking $file" docker run --rm --read-only --network none -i hadolint/hadolint:v2.15.1 hadolint --failure-threshold error - < "$file" - done < <(find moodle/.docker nginxproxy -name '*.Dockerfile' -print0) + done < <(git ls-files -z '*Dockerfile') diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 86d9f42..a2d73ed 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -42,15 +42,43 @@ jobs: - name: Deploy run: > + gateway=$(docker network inspect bridge --format '{{(index .IPAM.Config 0).Gateway}}') && sudo bash deploy-moodle.sh --slug "$SLUG" --domain "$DOMAIN" --no-ssl --db-type "${{ matrix.db }}" --moodle-version "${{ matrix.moodle }}" --profile small - --smtp-host 172.17.0.1:1025 --smtp-secure none --noreply-email "noreply@${DOMAIN}" + --smtp-host "${gateway}:1025" --smtp-secure none --noreply-email "noreply@${DOMAIN}" - name: Login page through nginxproxy run: | curl --fail --silent --show-error --max-time 60 "http://${DOMAIN}/login/index.php" -o login.html grep -q 'name="username"' login.html + - name: Resolve the synthetic site domain for Moodle self-checks + # The runner's /etc/hosts is not inherited by containers. Moodle 5.3's + # router check makes HTTP requests from PHP to the public wwwroot. + run: | + docker network create --internal mooship-e2e-self-checks + docker network connect --alias "$DOMAIN" mooship-e2e-self-checks nginxproxy + docker network connect mooship-e2e-self-checks "${SLUG}_php" + docker exec -e MOOSHIP_TEST_DOMAIN="$DOMAIN" "${SLUG}_php" php -r ' + $host = getenv("MOOSHIP_TEST_DOMAIN"); + exit(gethostbyname($host) === $host ? 1 : 0); + ' + + - name: Moodle router HTTP responses + if: matrix.moodle == '5.3' + run: | + curl --fail --silent --show-error --max-time 60 "http://${DOMAIN}/core/check/controller/test" -o /dev/null + curl --fail-with-body --silent --show-error --max-time 60 "http://${DOMAIN}/api/rest/v2/openapi.json" -o openapi-response.json || { + cat openapi-response.json + exit 1 + } + code=$(curl --silent --show-error --max-time 60 -o /dev/null -w '%{http_code}' "http://${DOMAIN}/not/a/valid/request") + test "$code" = 404 + for path in /lib/exampleshimroute.php /lib/exampleshimroute2.php; do + code=$(curl --silent --show-error --max-time 60 -o /dev/null -w '%{http_code}' "http://${DOMAIN}${path}") + test "$code" = 302 + done + # The project directory is private to root (mode 700) - name: Moodle configuration, sessions in Redis, and cron run: | @@ -63,9 +91,21 @@ jobs: password=$(grep "^REDIS_PASSWORD=" .env | cut -d= -f2-) docker compose exec -T -e REDISCLI_AUTH="$password" redis \ redis-cli --scan --pattern "mdl_session_*" | grep -q . - docker compose exec -T php php /var/www/html/admin/cli/cron.php > /dev/null + docker compose exec -T php php /var/www/html/admin/cli/cron.php --keep-alive=0 > /dev/null ' + - name: Native DB routing and installation-state checks + # Native host routing uses disposable DB containers, never the runner's own databases. + # The normal deployment above has already exercised the container-DB path. + env: + MOOSHIP_NATIVE_PREFLIGHT_TESTS: '1' + MOOSHIP_PREFLIGHT_DB: ${{ matrix.db }} + PYTHONPATH: tests + run: | + export MOOSHIP_PREFLIGHT_PHP_IMAGE + MOOSHIP_PREFLIGHT_PHP_IMAGE=$(sudo docker inspect --format '{{.Image}}' e2e_php) + python3 -B -m unittest test_native_preflight -v + - name: Mail delivery through SMTP run: | sudo docker exec -i e2e_php php <<'PHP' @@ -88,6 +128,15 @@ jobs: test ! -e "/etc/systemd/system/mooship-backup-${SLUG}.timer" # Warnings are expected on a test runner (no certificate, Moodle notices); failures are not + - name: Wait for services resumed after backups + run: | + sudo bash -euo pipefail -c ' + source ./deploy-moodle.sh + SLUG=e2e; PROJECT_DIR=/var/www/e2e + cd "$PROJECT_DIR" + for service in php nginx redis cron certbot_renew; do mooship_wait_service "$service"; done + ' + - name: Doctor run: | status=0 @@ -101,3 +150,22 @@ jobs: sudo bash -c 'cd /var/www/e2e && docker compose logs --tail 200' || true sudo docker logs --tail 100 nginxproxy || true docker logs --tail 50 mailpit || true + sudo docker exec e2e_php php /var/www/html/admin/cli/checks.php --type=status -v || true + sudo docker exec e2e_php php -r ' + define("CLI_SCRIPT", true); + require("/var/www/html/config.php"); + $check = new \core\check\environment\router(); + if (method_exists($check, "get_results")) { + foreach ($check->get_results() as $result) { + echo $result->get_summary(), "\n", $result->get_details(), "\n"; + } + } + if (class_exists(\core\router\apidocs::class)) { + try { + (new \core\router\apidocs())->openapi_docs(new \GuzzleHttp\Psr7\Response()); + } catch (\Throwable $error) { + fwrite(STDERR, $error->getMessage() . "\n" . $error->getTraceAsString() . "\n"); + exit(1); + } + } + ' || true diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml new file mode 100644 index 0000000..1590a0d --- /dev/null +++ b/.github/workflows/image.yml @@ -0,0 +1,53 @@ +name: Moodle images + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: image-${{ github.ref }} + cancel-in-progress: true + +jobs: + runtime: + name: Build and install Moodle with PostgreSQL and MySQL + runs-on: ubuntu-24.04 + timeout-minutes: 45 + env: + MOOSHIP_IMAGE_TESTS: '1' + PYTHONPATH: tests + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Validate Kubernetes 1.34 schemas + run: > + docker run --rm --read-only + --volume "$PWD/deploy/k8s:/manifests:ro" + ghcr.io/yannh/kubeconform:v0.7.0@sha256:85dbef6b4b312b99133decc9c6fc9495e9fc5f92293d4ff3b7e1b30f5611823c + -strict -summary -kubernetes-version 1.34.0 /manifests + - name: Build PHP and nginx images locally + run: | + docker build --target app --tag shellharbor/mooship:dev-moodle4.5.15-php8.3 . + docker build --target web --tag shellharbor/mooship:dev-moodle4.5.15-nginx . + - name: PHP syntax + run: | + for file in docker/*.php; do + docker run --rm --read-only --network none --volume "$PWD/$file:/check.php:ro" shellharbor/mooship:dev-moodle4.5.15-php8.3 php -l /check.php + done + - name: Validate both Compose examples + env: + MOODLE_ADMIN_EMAIL: admin@example.org + run: | + docker compose -f docker/compose.yml -f docker/compose.postgres.yml config --quiet + MOODLE_DB_TYPE=mysqli docker compose -f docker/compose.yml -f docker/compose.mysql.yml config --quiet + - name: Real installation, HTTP, secrets, readiness, and shutdown + run: | + docker pull postgres:17-alpine + docker pull mysql:8.4 + python3 -B -m unittest test_runtime_image -v diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml new file mode 100644 index 0000000..08b498e --- /dev/null +++ b/.github/workflows/python-package.yml @@ -0,0 +1,64 @@ +name: Python package + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: python-package-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + - run: python -m pip install -r requirements-python-build.txt + - run: python -B -m unittest discover -s tests -p test_python_cli.py -v + - run: python -m build + - run: python -m twine check --strict dist/* + - name: Rebuild wheel from source distribution + run: python -m pip wheel --no-deps --wheel-dir rebuilt dist/*.tar.gz + - name: Verify the native toolkit archive + run: | + python .github/scripts/build_release.py native + PYTHONPATH=src python -c 'from pathlib import Path; from tempfile import TemporaryDirectory; from mooship.install import unpack_release, verify_checksum; from mooship import __version__; a = next(Path("native").glob("*.tar.gz")); verify_checksum(a, a.with_name(a.name + ".sha256"), a.name); t = TemporaryDirectory(); unpack_release(a, Path(t.name) / "unpacked", __version__); t.cleanup()' + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: python-distributions + path: rebuilt/*.whl + if-no-files-found: error + retention-days: 7 + + install: + needs: build + runs-on: ubuntu-24.04 + timeout-minutes: 10 + strategy: + matrix: + python: ['3.8', '3.12', '3.14'] + steps: + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python }} + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: python-distributions + path: dist/ + - run: python -m pip install --no-deps dist/*.whl + - run: | + mooship --version + mooship --help + mooship install --directory "$RUNNER_TEMP/mooship-preview" --dry-run + python -m mooship --version diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..7919f0c --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,126 @@ +name: Release + +on: + release: + types: [published] + workflow_dispatch: + inputs: + tag: + description: Existing release tag matching VERSION (vX.Y.Z) + required: true + type: string + +permissions: + contents: read + +concurrency: + group: release-${{ inputs.tag || github.event.release.tag_name }} + cancel-in-progress: false + +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + outputs: + commit: ${{ steps.release.outputs.commit }} + version: ${{ steps.release.outputs.version }} + moodle: ${{ steps.release.outputs.moodle }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: refs/tags/${{ inputs.tag || github.event.release.tag_name }} + persist-credentials: false + - name: Verify tag, version and exact-commit CI + id: release + env: + RELEASE_TAG: ${{ inputs.tag || github.event.release.tag_name }} + GH_TOKEN: ${{ github.token }} + run: python3 .github/scripts/check_release.py + + distributions: + needs: verify + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.verify.outputs.commit }} + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + - run: python -m pip install -r requirements-python-build.txt + - run: python -m build + - run: python -m twine check --strict dist/* + - run: python .github/scripts/build_release.py native + - name: Publish verified native assets without overwriting existing files + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ inputs.tag || github.event.release.tag_name }} + run: python .github/scripts/publish_assets.py native + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: release-python-distributions + path: dist/* + if-no-files-found: error + retention-days: 14 + + pypi: + needs: distributions + if: vars.PYPI_PUBLISH == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + environment: + name: pypi + url: https://pypi.org/project/mooship/ + permissions: + id-token: write + steps: + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: release-python-distributions + path: dist/ + - name: Publish through PyPI Trusted Publishing + uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 + with: + skip-existing: true + + dockerhub: + needs: [verify, distributions] + if: vars.DOCKERHUB_PUBLISH == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 45 + env: + VERSION: ${{ needs.verify.outputs.version }} + REVISION: ${{ needs.verify.outputs.commit }} + MOODLE_VERSION: ${{ needs.verify.outputs.moodle }} + IMAGE: shellharbor/mooship + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.verify.outputs.commit }} + persist-credentials: false + - name: Build the exact release pair + run: | + docker build --platform linux/amd64 --target app --build-arg "MOOSHIP_VERSION=$VERSION" --build-arg "VCS_REF=$REVISION" -t "$IMAGE:$VERSION-moodle$MOODLE_VERSION-php8.3" . + docker build --platform linux/amd64 --target web --build-arg "MOOSHIP_VERSION=$VERSION" --build-arg "VCS_REF=$REVISION" -t "$IMAGE:$VERSION-moodle$MOODLE_VERSION-nginx" . + - name: Test both database installations before publication + run: | + docker pull postgres:17-alpine + docker pull mysql:8.4 + MOOSHIP_APP_IMAGE="$IMAGE:$VERSION-moodle$MOODLE_VERSION-php8.3" \ + MOOSHIP_WEB_IMAGE="$IMAGE:$VERSION-moodle$MOODLE_VERSION-nginx" \ + MOOSHIP_IMAGE_TESTS=1 PYTHONPATH=tests python3 -B -m unittest test_runtime_image -v + - name: Sign in and publish the tested images + env: + DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + run: | + test -n "$DOCKERHUB_USERNAME" && test -n "$DOCKERHUB_TOKEN" + python3 .github/scripts/check_image_tags.py + printf '%s' "$DOCKERHUB_TOKEN" | docker login --username "$DOCKERHUB_USERNAME" --password-stdin + docker push "$IMAGE:$VERSION-moodle$MOODLE_VERSION-php8.3" + docker push "$IMAGE:$VERSION-moodle$MOODLE_VERSION-nginx" + docker image inspect --format '{{join .RepoDigests "\n"}}' "$IMAGE:$VERSION-moodle$MOODLE_VERSION-php8.3" "$IMAGE:$VERSION-moodle$MOODLE_VERSION-nginx" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 0aaba41..7739e0e 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -30,7 +30,7 @@ jobs: - name: Install pinned CI tools run: python -m pip install -r .github/requirements.txt - name: Bandit - run: python -m bandit -r lib .github/scripts -c .github/bandit.yml + run: python -m bandit -r lib src .github/scripts -c .github/bandit.yml secrets: name: Secrets in files and Git history diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b05ece..9e03d81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ After every task with file changes, briefly record what changed and why: feature ### AI-KIT +- 2026-10-15: require image impact assessment for behavior, dependency and configuration changes, synchronized runtime/build assets, and affected image rebuilds with relevant tests. Added a verified build-input map and reporting rules to the runtime guide and synchronized public/local AI instructions so host and standalone changes are assessed together without unnecessary rebuilds or implied publication. + +- 2026-10-15: synchronized the isolated host-preflight branch/PR, local verification evidence, E2E run identity and separate Dependency Graph limitation in project context, navigation and the local continuity profile without merging the earlier image work. + +- 2026-10-15: synchronized the public Mooship skill, context, wiki, and local continuity profile with the separate standalone runtime contract, its tests, storage requirements, and publication limits. + +- 2026-10-15: synchronized preflight implementation pointers and verification boundaries in the public skill/context/wiki and local continuity profile while preserving the separate uncommitted image work. + - 2026-10-15: synchronized public context/wiki availability and clean-checkout documentation validation in the project skill, local registry, and continuity profile; optional AI-KIT paths are described without requiring unpublished resources. - 2026-10-15: recorded the tested database branch/migration contract in the public skill and synchronized the reproduced CI failure, successful Compose/database/regression checks, and remaining deployment limits in local context. - 2026-10-15: added a portable local `SKILL.md` for Mooship development and operations, replaced stale Claude instructions with an adapter to the same guide, and synchronized discovery, documentation, and context. Preserved ignore rules and avoided global installation. @@ -33,6 +41,29 @@ After every task with file changes, briefly record what changed and why: feature ### Project +- 2026-10-28: enabled repository Dependabot alerts/Dependency Graph and verified Dependency Review now passes. Added strict release tag-to-checkout verification and a Docker Hub overwrite guard; configured the user-confirmed PyPI pending publisher and GitHub environment/variable for tokenless publication. +- 2026-10-28: fixed Linux image-test secret fixtures: their bind-mounted temporary directory now permits traversal by runtime UID 1000. Docker Desktop's Windows bind handling had hidden the directory-permission failure reproduced by GitHub's Linux runner. +- 2026-10-28: prepared Mooship 1.2.0 distribution: a dependency-free Python CLI modeled on LaraShip, exact-release HTTPS/checksum/archive validation, native command forwarding, reproducible toolkit assets, wheel/sdist checks, Python compatibility CI, and gated Docker Hub/PyPI release workflows. Extended lint/security/dependency checks to the Python package and documented account setup and publication evidence. Existing application images remain separate from host administration. +- 2026-10-28: integrated the previously isolated PR #6 preflight/readiness/router/dependency fixes into the distribution branch, resolving documentation conflicts and preserving standalone image work. The combined Linux snapshot passed 130 regressions (145 discovered, 15 opt-in skipped); wheel/sdist metadata and actual native release archive verification passed. + +- 2026-10-15: verified the host preflight implementation in GitHub E2E run 38787890321 at ff1fbf2: all four Moodle 4.5/5.3 and PostgreSQL/MySQL cases passed, including router/API/shims, SMTP, cron, backups, systemd schedules and doctor. Local validation passed 108 regressions (120 discovered, 12 opt-in skipped), both real host-gateway DB probes, HTTP/HTTPS and Compose/proxy checks. Dependency Review remains separately blocked by disabled repository Dependency Graph. + +- 2026-10-15: implemented host port preflight, configurable native DB routing, PHP-side connectivity/state probes and bounded service readiness on the separate `codex/deployment-preflight-readiness` branch (draft PR #6), preserving the uncommitted image work in this checkout. Added regression/native-routing tests and expanded full E2E; branch documentation records rollout and archive compatibility. + +- 2026-10-15: added buildable Moodle 4.5.15/PHP 8.3 and matching nginx images for later Docker Hub publication, with pinned source/base digests, non-root execution, immutable code, file secrets, explicit locked installation/cron, DB-aware readiness, process liveness, and graceful shutdown. Added PostgreSQL/MySQL Compose overlays, minimal Kubernetes manifests, operational/publishing documentation and ADR-0004. Added real Moodle image integration tests and a build/test-only CI workflow; expanded Dockerfile/YAML linting and Dependabot discovery. Added LF rules for portable Linux entrypoints. Existing host deployment scripts remain independent. + +- 2026-10-15: install locked production dependencies when Moodle declares league/oauth2-server, fixing the verified missing ScopeTrait fatal error on fresh Moodle 5.3 API requests. Run Composer as the PHP container user with a pinned, SHA-256-verified PHAR, without developer packages, scripts or plugins, and verify the required trait before Moodle initialization. + +- 2026-10-15: route removed PHP script paths through Moodle's router while preserving FastCGI slash arguments, original URLs and query parameters. Added real HTTP/HTTPS regressions and E2E assertions for both Moodle 5.3 shim redirects; a missing router returns 404 rather than looping. + +- 2026-10-15: resolve the synthetic E2E site domain inside PHP through a dedicated internal Docker network alias, allowing Moodle 5.3 router self-checks to reach the shared proxy. Added explicit routed HTTP status checks and detailed router diagnostics without weakening doctor. + +- 2026-10-15: preserve the inode when selecting Moodle's public document root so the running nginx file bind mount sees the change before reload. Exercise the actual updater with a single-file mount and verify inode/protocol preservation; the second full E2E exposed the stale mount missed by a directory-mounted fixture. + +- 2026-10-15: full E2E exposed stale running nginx configuration after Moodle 5.1+ switches to the public/ document root. Validate/reload nginx after generating Moodle configuration, add a real HTTP regression, and wait for services resumed by backup before E2E doctor checks instead of treating transient starting health as a permanent failure. + +- 2026-10-15: validate all explicit deployment ports before side effects, reject duplicates/occupied ports and shared proxy reservations, and exclude selected ports from automatic allocation. Added configurable native DB routing with host-gateway mapping and real PHP-side connectivity/authentication checks. Replaced fixed startup sleep with bounded PHP/nginx/Redis health checks, deferred cron until installation, and replaced installer-help parsing with structured DB state checks. Added regressions and disposable host-gateway DB checks to full E2E; preserve native endpoints in restored archives. + - 2026-10-15: fixed published context/wiki links to unavailable local AI-KIT resources. Documentation validation now checks targets against the Git file inventory, preventing local ignored files from masking broken links; added regressions for ignored files/directories, pending public files, tracked ignored documents, and command failure. - 2026-10-15: restored MySQL 8.4 LTS and PostgreSQL 17 in the new-project template after Dependabot branch upgrades broke the Compose contract. Restricted automated database branch changes and added PostgreSQL image/data-mount assertions; documented that existing deployments need a separate migration rather than an image downgrade. - 2026-10-15: explicitly allowed the root `SKILL.md` in `.gitignore` and linked it from README so other AI assistants can read the project guide on GitHub or in a clone. Corrected skill availability instructions while keeping nested agent skills and local AI-KIT files ignored. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..391d585 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,97 @@ +ARG PHP_IMAGE=php:8.3-fpm-bookworm@sha256:f768bd9142444f6126a690697b028c23f30878c3b628a7c355dfce75e29cc045 +ARG NGINX_IMAGE=nginx:1.31.0-alpine@sha256:2f07d83bf561b506400dc183b1b2003803e39efbd22451f848adaba14d28c7c7 +ARG MOODLE_VERSION=4.5.15 +ARG MOODLE_COMMIT=0a694ba9b8b4183bf7a45094cfa7211030db3192 +ARG MOODLE_SHA256=19d8ab675769cd631b733c63914d1e318acad575763c75ef7e7e238f4180ed70 + +FROM ${PHP_IMAGE} AS extensions +ARG REDIS_VERSION=6.3.0 +RUN apt-get update && apt-get install -y --no-install-recommends \ + libicu-dev libpq-dev libzip-dev libpng-dev libjpeg62-turbo-dev \ + libfreetype6-dev libonig-dev libxml2-dev libwebp-dev \ + && docker-php-ext-configure gd --with-freetype --with-jpeg --with-webp \ + && docker-php-ext-install -j2 intl mysqli pgsql pdo_mysql pdo_pgsql zip gd soap exif opcache bcmath sockets mbstring \ + && pecl install "redis-${REDIS_VERSION}" $release, "version" => $version]));' \ + && chmod -R a-w /opt/moodle + +FROM ${NGINX_IMAGE} AS web +ARG MOODLE_VERSION +ARG MOOSHIP_VERSION=dev +ARG VCS_REF=unknown +LABEL org.opencontainers.image.title="Mooship Moodle web server" \ + org.opencontainers.image.source="https://github.com/shellharbor/mooship" \ + org.opencontainers.image.version="${MOOSHIP_VERSION}-moodle${MOODLE_VERSION}-nginx" \ + org.opencontainers.image.revision="${VCS_REF}" \ + org.opencontainers.image.licenses="GPL-3.0-or-later AND MIT" +RUN addgroup -g 1000 mooship && adduser -D -H -u 1000 -G mooship mooship \ + && mkdir -p /var/moodledata /opt/mooship \ + && chown 1000:1000 /var/moodledata +COPY --from=source /opt/moodle /opt/moodle +COPY docker/nginx.conf.template docker/health.txt /opt/mooship/ +COPY --chmod=755 docker/web-entrypoint.sh /usr/local/bin/mooship-web +COPY LICENSE /opt/mooship/LICENSE +USER 1000:1000 +EXPOSE 8080 +STOPSIGNAL SIGQUIT +HEALTHCHECK --interval=15s --timeout=5s --start-period=10s --retries=3 \ + CMD wget -q -O - http://127.0.0.1:8080/healthz | grep -qx ok +ENTRYPOINT ["/usr/local/bin/mooship-web"] + +FROM ${PHP_IMAGE} AS app +ARG MOODLE_VERSION +ARG MOOSHIP_VERSION=dev +ARG VCS_REF=unknown +LABEL org.opencontainers.image.title="Mooship Moodle runtime" \ + org.opencontainers.image.source="https://github.com/shellharbor/mooship" \ + org.opencontainers.image.version="${MOOSHIP_VERSION}-moodle${MOODLE_VERSION}-php8.3" \ + org.opencontainers.image.revision="${VCS_REF}" \ + org.opencontainers.image.licenses="GPL-3.0-or-later AND MIT" +RUN apt-get update && apt-get install -y --no-install-recommends \ + libicu72 libpq5 libzip4 libpng16-16 libjpeg62-turbo libfreetype6 \ + libonig5 libxml2 libwebp7 libfcgi-bin tini \ + && rm -rf /var/lib/apt/lists/* \ + && groupadd --gid 1000 mooship \ + && useradd --uid 1000 --gid 1000 --home-dir /var/moodledata --no-create-home --shell /usr/sbin/nologin mooship \ + && mkdir -p /var/moodledata /opt/mooship \ + && chown 1000:1000 /var/moodledata \ + && chmod 2770 /var/moodledata \ + && cp /usr/local/etc/php/php.ini-production /usr/local/etc/php/php.ini \ + && rm /usr/local/etc/php-fpm.d/*.conf +COPY --from=extensions /usr/local/lib/php/extensions/ /usr/local/lib/php/extensions/ +COPY --from=extensions /usr/local/etc/php/conf.d/ /usr/local/etc/php/conf.d/ +COPY --from=source /opt/moodle /opt/moodle +COPY --from=source /opt/mooship/version.json /opt/mooship/version.json +COPY docker/config.php /opt/moodle/config.php +COPY docker/environment.php docker/validate.php docker/readiness.php docker/install.php /opt/mooship/ +COPY --chmod=755 docker/entrypoint.sh /usr/local/bin/mooship +COPY --chmod=755 docker/healthcheck.sh /usr/local/bin/mooship-healthcheck +COPY --chmod=755 docker/cron.sh /opt/mooship/cron.sh +COPY docker/php-fpm.conf /usr/local/etc/php-fpm.d/mooship.conf +COPY moodle/.config/php/mooship.ini /usr/local/etc/php/conf.d/zz-mooship.ini +COPY LICENSE /opt/mooship/LICENSE +ENV PHP_FPM_MAX_CHILDREN=4 +USER 1000:1000 +WORKDIR /opt/moodle +EXPOSE 9000 +STOPSIGNAL SIGQUIT +HEALTHCHECK --interval=15s --timeout=5s --start-period=10s --retries=3 CMD ["/usr/local/bin/mooship-healthcheck"] +ENTRYPOINT ["/usr/bin/tini", "-g", "--", "/usr/local/bin/mooship"] +CMD ["fpm"] diff --git a/PROJECT_CONTEXT.md b/PROJECT_CONTEXT.md index 0258e48..d5fa9d6 100644 --- a/PROJECT_CONTEXT.md +++ b/PROJECT_CONTEXT.md @@ -1,11 +1,11 @@ # PROJECT_CONTEXT -Last verified: 2026-10-15. Sources: the user's instructions, local Git metadata, GitHub API, README, scripts, and templates. Test results below retain their execution dates. +Last verified: 2026-10-28. Sources: the user's instructions, local Git metadata, GitHub/Docker Hub/PyPI APIs, README, scripts, and templates. Test results below retain their execution dates. ## Project overview - **Mooship** automates Moodle installation on Ubuntu with Docker Compose and a shared reverse proxy. -- This directory is a Git repository with `origin` set to **shellharbor/mooship**, `https://github.com/shellharbor/mooship.git`. The latest locally inspected commit on 2026-10-15 is `35aa256` (subject `v1.1.2`); earlier implementation baselines include `b44b4e1` (`v1.1.2`), `d553e0d` (`v1.1.0`), and `aafd118` (`v1.0.0`). On 2026-10-15, the GitHub API confirmed `main` as the default branch, MIT as the license, tags `v1.1.0` / `v1.0.0`, and `v1.1.0` as the latest published release; a commit subject is not evidence of a tagged release. +- This directory is a Git repository with `origin` set to **shellharbor/mooship**, `https://github.com/shellharbor/mooship.git`. The latest locally inspected commit on 2026-10-15 is `cddc89b` (subject `v1.1.2`); earlier implementation baselines include `b44b4e1` (`v1.1.2`), `d553e0d` (`v1.1.0`), and `aafd118` (`v1.0.0`). On 2026-10-15, the GitHub API confirmed `main` as the default branch, MIT as the license, tags `v1.1.0` / `v1.0.0`, and `v1.1.0` as the latest published release; a commit subject is not evidence of a tagged release. - Workflows: creating a site, activation/deactivation, explicit removal, full backups, and restoration into an unused directory. - Project language: English throughout documentation, code comments, CLI help, prompts, log/error messages, configuration templates, examples, and local agent instructions. Keep future additions in English, as requested by the user. - Chat language: Russian, according to the user's current language policy; project artifacts shown in chat retain English. @@ -26,8 +26,23 @@ Last verified: 2026-10-15. Sources: the user's instructions, local Git metadata, - Shared checks: `lib/common.sh`; environment files, manifests, and archive extraction: `lib/state.py`; HTTP nginx configuration: `lib/web.py`; proxy lifecycle Compose edits: `lib/proxy.py`; templates: `moodle/` and `nginxproxy/`; examples: `examples/`. - Tests: `tests/` (Python unittest, Bash, and mocked external operations). Moodle core is downloaded; there are no local Moodle PHPUnit/Behat suites or migrations. Moodle CLI installs the database schema. +## Standalone Moodle runtime (2026-10-15) + +- User chose Moodle/PHP in an application image, with nginx and DB separate, for later Docker Hub publication. Root `Dockerfile` builds `app` (default) and `web` targets; local tags are `shellharbor/mooship:dev-moodle4.5.15-php8.3` and `shellharbor/mooship:dev-moodle4.5.15-nginx`. Both build successfully on Linux amd64. Publication, Git pushes and a release were not performed. +- Moodle 4.5.15 is pinned to commit `0a694ba9b8b4183bf7a45094cfa7211030db3192` and archive SHA-256; PHP 8.3 Bookworm and nginx 1.31.0 Alpine bases are digest-pinned. Both images contain the same immutable source. Moodle and dependency licenses remain bundled; wrapper code remains MIT. +- UID/GID 1000; writable `/var/moodledata` and `/tmp`, read-only roots, no capabilities/Docker socket/source mount required. Credentials use environment or absolute `_FILE` secrets; only the install workload receives the admin secret. Commands: `fpm`, explicit locked `install`, `cron`, `cron-once`, `check`, and passthrough `php ...`. Normal startup does not migrate schemas and refuses an absent/mismatching installed version. +- `docker/compose.yml` plus the chosen `compose.postgres.yml` or `compose.mysql.yml` overlay supplies PostgreSQL 17/MySQL 8.4 and a separate scheduler. Only nginx publishes HTTP, on loopback 8080 by default. Runtime settings/secrets, backup/upgrade differences from the host toolkit and later publication commands are in `docs/DOCKER.md` (ADR-0004). +- FPM liveness is a FastCGI ping; nginx liveness is `/healthz`; `/readyz`/`mooship check` query DB/schema readiness. Docker shutdown uses SIGQUIT. Kubernetes examples use stable APIs, non-root security contexts, DB/admin Secret references, external DB, RWX data PVC, PHP/nginx Deployment, explicit install Job and independent CronJob. Resource values are estimates. All six resources pass strict Kubernetes 1.34 schema validation; no cluster deployment/server dry-run was performed. +- `.github/workflows/image.yml` adds a ninth workflow: local image builds, PHP syntax, Compose/schema checks and opt-in real Moodle tests. It never publishes. Dependabot now discovers the root Dockerfile and runtime Compose; Dockerfile/YAML lint includes these paths. Existing host deployment/lifecycle/backup scripts are independent. +- Validation outside the opt-in image suite: 89 Linux regressions passed (101 discovered, 12 skipped); PHP syntax, ShellCheck, Ruff, YAML, actionlint, zizmor, documentation links, both runtime Compose variants, and strict Kubernetes 1.34 schema checks passed. Hadolint reported no errors; package-version/pipeline-style warnings remain visible. No GitHub-hosted run was claimed. +- Final image validation: all three `MOOSHIP_IMAGE_TESTS=1` tests passed on Docker Desktop Linux amd64 (351.508 seconds). Real Moodle installs passed on PostgreSQL 17 and MySQL 8.4, with file-based app/admin secrets, HTTP login/protected paths, cron, repeated-install and version refusal, DB outage/recovery, independent liveness and graceful app/web shutdown. Initial MySQL fixtures exposed upstream initializer SQL quoting limits; final DB fixtures avoid quote/backslash initialization while the secret reader/admin password still exercise them. Both image builds remain available locally. Additional architectures, production TLS/SMTP/Redis, host installer full E2E and a live Kubernetes deployment were not tested by this suite. + ## Working with the project +- Distribution completion (2026-10-28): user requested Docker Hub publication and a Python CLI following neighboring LaraShip. Current work is on `codex/distribution-and-publishing`, preserving earlier uncommitted image sources. VERSION proposes 1.2.0; `src/mooship/` provides dependency-free Python 3.8+ installation and native script forwarding. Exact-release installation verifies HTTPS, SHA-256, archive safety/limits and VERSION, refuses existing destinations and marks failed copies. Root `pyproject.toml`, package/release workflows and `docs/PYTHON.md`/`docs/RELEASING.md` are added. The runtime Docker images remain application images, without a host-controller backend. Docker daemon is available; both earlier dev images exist locally. Docker Hub repository exists but has no tags; PyPI mooship returns 404. Local Docker credentials exist for tigusigalpa; GitHub Actions publishing secrets/variables and PyPI Trusted Publisher are not yet configured. User corrected an initial setup confirmation and explicitly said PyPI is not configured. Registry publication must be verified separately; no 1.2.0 release or upload is claimed yet. + +- Host preflight implementation originated on `codex/deployment-preflight-readiness`, in `.ci-reports/deployment-preflight`, [PR #6](https://github.com/shellharbor/mooship/pull/6). On 2026-10-28 it was merged into `codex/distribution-and-publishing` at `d864b7f`, preserving the image and package work; primary main is not yet updated. The implementation adds port/native DB/readiness gates, structured installation checks, deferred cron and archive compatibility; fixes nginx file identity/reload/PHP shims; and installs declared locked production dependencies with a pinned verified Composer tool. Earlier local checks passed 108 regressions (120 discovered, 12 opt-in skipped), Compose/proxy checks, real PostgreSQL/MySQL host-gateway probes, all six HTTP/HTTPS scenarios and linters/docs. [Full E2E run 38787890321](https://github.com/shellharbor/mooship/actions/runs/38787890321) tested `ff1fbf2` and passed all four Moodle 4.5/5.3 x PostgreSQL/MySQL cases. Dependency Review separately reported disabled repository Dependency Graph. On 2026-10-28 the combined committed snapshot passed 130 Linux regressions (145 discovered, 15 opt-in skipped); package wheel/sdist metadata and actual native archive checksum/extraction/VERSION checks passed. New registry uploads and GitHub runs remain separate checks. + - Installation: `sudo bash deploy-moodle.sh --slug lms --domain lms.example.com --db-type postgres --ssl-email admin@example.com`; see `README.md` for the full guide. - Deployment requires Docker. Check daemon availability before container-based validation; daemon status is a temporary fact. - Checks: `bash -n` for `.sh` files, `python3 -B -m unittest discover -s tests -v`, ShellCheck, and `docker compose config` for generated variants. Report the results for each task. @@ -41,7 +56,7 @@ Last verified: 2026-10-15. Sources: the user's instructions, local Git metadata, - Database branch repair validation (2026-10-15): reproduced the reported MySQL 9.6/8.4 assertion failure, then passed the exact two-test Configuration command in Ubuntu WSL with Compose v5.6.0 (four generated database variants and six proxy states). PostgreSQL 17/MySQL 8.4 real backup/restore round trips passed separately in isolated Docker containers. The full Python 3.14 regression run in a temporary Linux copy passed 85 tests (94 discovered, 9 optional checks skipped). Raw template parsing, Ruff on the changed tests, and Dependabot YAML lint passed. These local checks do not establish a new GitHub workflow run or a complete Moodle deployment. - Documentation repair validation (2026-10-15): a clean local clone reproduced all 15 missing AI-KIT links from the reported GitHub failure. Overlaying the pending public changes passed the 12-document/endpoint JSON check without `AGENTS.md`, `ai-kit/`, `.agents/`, `.claude/`, or local decision templates. The same clean Linux copy passed 89 regressions (98 discovered, 9 optional checks skipped); all 10 documentation tests also passed separately. Ruff passed on the changed checker/tests; Bandit passed on the checker. GitHub-hosted execution was not rerun in this task. - Dependabot checks Actions, CI Python requirements, Dockerfiles, and Compose weekly with a seven-day version-update cooldown; PHP minor/major image updates require coordinated template/version-selection changes. Actions are pinned to verified commits; scanner containers use explicit version tags. Gitleaks permits only exact synthetic database fixture passwords in their test file and local tool/report directories. Its job trusts only the mounted checkout and checks for readable, nonempty Git history before scanning to avoid silent ownership failures on Linux runners. -- GitHub verification (2026-10-15): CI, Configuration, Component integration, Security, CodeQL, and Documentation completed successfully on `main` at `b44b4e1`. Dependabot update jobs also completed successfully. The API showed failed dependency-review runs on Dependabot PR branches; the cause was not inspected in this documentation task. E2E has no runs on `main`. Default CodeQL setup could not previously be inspected because the token lacks repository-administration scope; no extra token permissions were requested. Full Moodle E2E, live SMTP/systemd, production DNS, and Let's Encrypt remain unverified. Scanner containers are short-lived CI commands without published ports; deployment templates still describe the existing workloads. No Kubernetes manifests. +- GitHub verification (2026-10-15): CI, Configuration, Component integration, Security, CodeQL, and Documentation completed successfully on `main` at `b44b4e1`. Dependabot update jobs also completed successfully. The API showed failed dependency-review runs on Dependabot PR branches; the cause was not inspected in this documentation task. E2E has no runs on `main`. Default CodeQL setup could not previously be inspected because the token lacks repository-administration scope; no extra token permissions were requested. The later preflight-branch E2E result is recorded above; production DNS and Let's Encrypt remain unverified. Scanner containers are short-lived CI commands without published ports; deployment templates still describe the existing workloads. Standalone runtime manifests were added later on 2026-10-15; see the runtime section. ## Constraints and conventions @@ -55,16 +70,17 @@ Last verified: 2026-10-15. Sources: the user's instructions, local Git metadata, - Full archive v1: `/var/backups/mooship/_.tar.gz`, containing the project directory, database dump, SSL data, and manifest. The application and certificate renewal pause during the snapshot; previously running services resume even on failure. Optional retention (`--keep N`) removes only `_YYYYMMDD_HHMMSS.tar.gz` archives; hooks must be root-owned and not group/other-writable; schedules are `mooship-backup-.timer` units. - `restore.sh` accepts a trusted v1 archive created by Mooship and an unused slug matching the archive; the user must type the slug to confirm. Native databases/users must not already exist. `remove.sh` preserves backups; native DROP requires a matching `.mooship-native-database` marker, written after successful database/user creation. Older native projects without the marker retain their databases for manual review. - Local AI-KIT container convention: new Dockerfiles, containers, and microservices must be Kubernetes-ready under the local `ai-kit/CONTAINERS.md` standard when available in the checkout. Kubernetes availability in this project must be verified separately. +- Image synchronization agreement (2026-10-15): assess image impact for behavior, dependency and configuration changes across host installation and standalone runtime. Synchronize affected sources/build definitions, configuration, manifests, tests and documentation in the same task, rebuild affected targets, and run relevant runtime checks. Shared Moodle source/plugins must match in `app` and `web`; preserve each deployment option's supported versions. Verified host-only/documentation changes need no rebuild. Report impact and actual checks, distinguishing source updates, builds, publication and deployment. See `docs/DOCKER.md#keeping-images-in-sync` and root `SKILL.md`; this agreement changes instructions only and does not imply a new build or publication. - Architecture decisions: see `docs/DECISIONS.md` and `docs/adr/`. ## Open questions -- The production server, DNS, complete Moodle installation, and actual SSL issuance have not been verified. +- The production server, DNS, host installer full E2E, and actual SSL issuance have not been verified. Standalone runtime checks are recorded separately. - The six `phpXX.ini` templates keep stock values (`upload_max_filesize=2M`, `memory_limit=128M`); new projects override them with `mooship.ini`, while projects created earlier still use the stock limits. - Redis stores Moodle sessions in new projects (sprint 2); Moodle's application cache (MUC) is not configured automatically. Compatibility of older Moodle releases with the fixed database images needs separate validation. -- Sprint 2 behavior verified only with mocks and component checks: real Moodle sessions in Redis, SMTP delivery, systemd timers, and `doctor.sh` against a live site are covered by `e2e.yml`, which has no runs on `main` as of 2026-10-15. +- Full host E2E verified on 2026-10-15: run 38787890321 passed Moodle 4.5/5.3 with PostgreSQL/MySQL at implementation ff1fbf2 on the separate preflight branch. It checked HTTP login, routed/API/shim responses for 5.3, Redis sessions, cron, PHP host-gateway DB probes, SMTP to Mailpit, backup retention, systemd timers and doctor. Synthetic DNS/HTTP and disposable databases do not verify production DNS/TLS or host-installed DB administration. No main-branch merge is implied. - The proxy lifecycle parser defects reproduced on 2026-10-06 were fixed and regression-tested on 2026-10-14. Shared nginx configuration validation still follows application/restart; deployment cleanup or recovery is not implemented. -- Hardening still pending: localhost PHP-FPM publication, `moodledata` mode 777, fixed native DB address `172.17.0.1`, and password-file input. SMTP values avoid the child `docker exec` argument list, but an explicitly supplied `--smtp-password` still appears in the original deployment command. Resource profiles tune workers/buffers without enforcing container memory or CPU budgets. +- Hardening still pending: `moodledata` mode 777 and password-file input. SMTP values avoid the child `docker exec` argument list, but an explicitly supplied `--smtp-password` still appears in the original deployment command. Resource profiles tune workers/buffers without enforcing container memory or CPU budgets. - Backups require space for a temporary project copy and the archive; older zip files are not full v1 backups. ## Update rule diff --git a/README.md b/README.md index 8a172e1..7a33f8c 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,12 @@ Automated Moodle deployment on Ubuntu. Chosen GitHub repository: [shellharbor/mo The PHP version is selected automatically based on the Moodle version. Passwords, database names, and available ports are generated unless supplied explicitly. +For a portable application image, see the [standalone Moodle Docker guide](docs/DOCKER.md). The root Dockerfile builds Moodle 4.5.15/PHP 8.3 and a matching nginx companion, with separate PostgreSQL/MySQL and persistent data. Compose examples, Kubernetes manifests, installation tests, and later Docker Hub publication instructions are included. Build these images locally until published; the host installer below is an independent deployment option. Bundled Moodle retains its GPL license. + +The [Python CLI](docs/PYTHON.md) provides a dependency-free `mooship` command for verified toolkit installation and the existing deployment, lifecycle, backup/restore and diagnostics scripts. Version 1.2.0 introduces packaging for pip/pipx; use `pipx install .` from a reviewed checkout until publication. See [release publication](docs/RELEASING.md) for Docker Hub and PyPI setup and verification. + +Deployment validates ports and service readiness before publishing a site. For native databases, `--db-host HOST` selects the container-reachable address (default `host.docker.internal` with Linux host-gateway mapping). See [preflight, native database access, and readiness](docs/PREFLIGHT.md) for requirements and checks. + The `mooship/` folder is self-contained. It includes the deployment script, project and proxy templates, management utilities, examples, and an [AI assistant skill](SKILL.md). Copy the entire folder to a server or place it in a separate Git repository; it does not require files from outside the folder. --- @@ -164,8 +170,7 @@ Client ──► nginxproxy :80 / :443 ──(Docker network )──► _db` | `5432` | | Containerized MySQL | `mysqli` | `_db` | `3306` | -| Native PostgreSQL | `pgsql` | `172.17.0.1` | `--port-postgres` or 5432 | -| Native MySQL | `mysqli` | `172.17.0.1` | `--port-mysql` or 3306 | +| Native PostgreSQL | `pgsql` | `--db-host` or `host.docker.internal` | `--port-postgres` or 5432 | +| Native MySQL | `mysqli` | `--db-host` or `host.docker.internal` | `--port-mysql` or 3306 | `wwwroot` uses `https://` by default and `http://` with `--no-ssl`. The same protocol is saved as `SITE_SCHEME` in `.env` and preserved during backup restoration. @@ -633,13 +639,13 @@ docker compose exec db sh -c 'mysql -u"$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_ ### Native database (`--db-native`) -- No database container is created in Compose. Moodle connects to `dbhost = 172.17.0.1`, the host IP on the `docker0` interface. +- No DB container is created. PHP/cron map `host.docker.internal` to Docker's host gateway by default; `--db-host HOST` selects a custom container-reachable address of the same locally managed DB. Its value is saved as `DB_HOST`. - The port comes from `--port-postgres`/`--port-mysql`, defaulting to 5432/3306. - Project containers use their own `` network. Its subnet is usually `172.18.0.0/16` or above, rather than `172.17.0.0/16`. Allow access from all relevant Docker subnets, for example `172.16.0.0/12`. Find the exact subnet with `docker network inspect -f '{{(index .IPAM.Config 0).Subnet}}'`. **PostgreSQL on the host:** -1. In `postgresql.conf`, the server must listen on the Docker bridge, for example `listen_addresses = 'localhost,172.17.0.1'` (or `'*'`). +1. Set PostgreSQL `listen_addresses` to the actual host-side Docker gateway interface as well as localhost. Determine the gateway from your Docker configuration; no fixed bridge IP is assumed. See [native preflight](docs/PREFLIGHT.md#native-databases). 2. Allow Docker subnets in `pg_hba.conf`: ``` host all all 172.16.0.0/12 scram-sha-256 @@ -652,7 +658,7 @@ docker compose exec db sh -c 'mysql -u"$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_ **MySQL on the host:** 1. `--db-root-password` is required. Root must authenticate **with a password**: on Ubuntu, root uses `auth_socket` by default, so a password-based `mysql -u root -p…` check will fail. -2. `bind-address` in `/etc/mysql/mysql.conf.d/mysqld.cnf` must include `172.17.0.1` (or `0.0.0.0`). Restart MySQL after changing it. +2. MySQL's `bind-address` must accept traffic on the actual Docker-reachable host interface. Restrict access with the firewall and restart MySQL after configuration changes. 3. The user is created as `'user'@'%'` to allow connections from containers. 4. If ufw is enabled, allow port 3306 from `172.16.0.0/12`. @@ -663,7 +669,7 @@ sudo bash /opt/mooship/deploy-moodle.sh \ --slug mylms --domain mylms.example.com \ --db-type mysql --db-native --db-root-password 'ChangeMe_Root_2026' \ --ssl-email admin@example.com -# Result: database and 'user'@'%' created in host MySQL; config.php → 172.17.0.1:3306 +# Result: database and 'user'@'%' created in host MySQL; config.php → host.docker.internal:3306 ``` --- @@ -807,14 +813,14 @@ sudo bash /opt/mooship/deploy-moodle.sh \ ### 7. Native PostgreSQL ```bash -# Prerequisites: host PostgreSQL listens on 172.17.0.1; pg_hba.conf allows 172.16.0.0/12 +# Prerequisites: PostgreSQL accepts connections on the Docker-reachable host interface and project subnet sudo bash /opt/mooship/deploy-moodle.sh \ --slug nativepg \ --domain nativepg.example.com \ --db-type postgres \ --db-native \ --ssl-email admin@example.com -# Result: database and user created in host PostgreSQL; config.php → 172.17.0.1:5432, +# Result: database and user created in host PostgreSQL; config.php → host.docker.internal:5432, # no db container ``` diff --git a/SKILL.md b/SKILL.md index bc7c09e..c886c2f 100644 --- a/SKILL.md +++ b/SKILL.md @@ -1,11 +1,11 @@ --- name: mooship -description: Develop, review, test, and operate Mooship, the Bash and Python toolkit for deploying Moodle on Ubuntu with Docker Compose. Use for this repository's deployment scripts, templates, site lifecycle, backup and restore, diagnostics, and CI; not for general Moodle plugin development. +description: Develop, review, test, and operate Mooship's Moodle deployment toolkit and standalone runtime images. Use for its scripts, templates, container builds, site lifecycle, backup and restore, diagnostics, and CI; not for general Moodle plugin development. --- # Mooship -Work on `shellharbor/mooship`. This is a deployment toolkit, not a Moodle core checkout: Moodle source is downloaded during deployment. Runtime Python helpers use Python 3.8+ and the standard library. Server operations target Ubuntu 20.04+ with root privileges, Docker Engine, and the Compose plugin. +Work on `shellharbor/mooship`. The host installer downloads Moodle during deployment; the separate root Dockerfile downloads pinned Moodle during build and bakes it into portable PHP/nginx images. There is no vendored Moodle core checkout. Runtime Python helpers use Python 3.8+ and the standard library. Host installer operations target Ubuntu 20.04+ with root privileges, Docker Engine, and the Compose plugin. Resolve paths below against the Mooship repository root, even if this skill is loaded from another location. Keep the checkout available; this file is a guide to the project, not a replacement for its scripts and templates. A root `SKILL.md` can be supplied directly to another AI; automatic discovery depends on that AI's integration. @@ -26,21 +26,37 @@ Resolve paths below against the Mooship repository root, even if this skill is l | Backup operations | `backup.sh`, `restore.sh`, `schedule-backup.sh`; archive contract in `docs/BACKUP.md` | | Diagnostics | `doctor.sh`: read-only health checks; exit codes 0 OK, 1 WARNING, 2 CRITICAL | | Shared shell contracts | `lib/common.sh`: paths, identifiers, database ownership/readiness, schedules | +| Deployment preflight | `lib/preflight.sh`, `lib/check-database.php`, `lib/install-dependencies.sh`, `docs/PREFLIGHT.md`: ports, native routing, PHP DB/dependency probes, bounded readiness | | State and archives | `lib/state.py`: environment parsing, manifests, checksums, safe extraction | | HTTP configuration | `lib/web.py`, `moodle/.config/nginx/_site.conf`, `nginxproxy/site-template.conf` | | Proxy lifecycle edits | `lib/proxy.py`, exercised by `tests/test_proxy_lifecycle.py` | | Project templates | `moodle/docker-compose.yml`, `moodle/.docker/`, `moodle/.config/php/` | +| Standalone images | Root `Dockerfile`, `docker/`, `deploy/k8s/`, `docs/DOCKER.md`, `tests/test_runtime_image.py`, `.github/workflows/image.yml` | +| Python CLI and releases | `src/mooship/`, `pyproject.toml`, `VERSION`, `docs/PYTHON.md`, `docs/RELEASING.md`, `.github/workflows/python-package.yml`, `.github/workflows/release.yml` | | Shared proxy template | `nginxproxy/`; deployed separately under `/var/www/nginxproxy` | | Validation | `tests/`, `.github/scripts/`, `.github/workflows/`, `.github/requirements.txt`, `.coveragerc` | Prefer existing shared helpers over new inline parsers. When editing generated configuration, check both the template and every script that transforms it. A template change does not automatically update an already deployed site or an existing proxy's copied template. +## Assess image impact + +- Before changing behavior, dependencies, or configuration, follow [the image update procedure](docs/DOCKER.md#keeping-images-in-sync). Trace the actual build inputs and assess both host installation and standalone runtime behavior; their source layouts and supported versions differ. +- If affected, synchronize image sources/build definitions, runtime configuration, manifests, tests and documentation in the same task. Rebuild affected `app`/`web` targets and run relevant runtime tests; shared Moodle source or plugins require matching changes in both images. An already copied source needs no cosmetic Dockerfile edit. Host-only or documentation changes need no rebuild when verified to have no image impact. +- Report the impact, affected targets, builds and tests actually run, and any unavailable checks. Source changes, local image builds, registry publication and deployment are separate outcomes; publication/deployment stay within the user's authorized scope. + ## Preserve operational contracts +- **Python distribution:** follow [the CLI guide](docs/PYTHON.md). Package installation does not deploy or install Docker. Native toolkit installation requires an exact official release, checksum and safe archive validation, a new destination and matching VERSION. Existing scripts retain their permissions, prompts, signals and exit codes. Application images are not a Docker-controller backend. Release publication requires matching VERSION/tag and successful exact-commit checks; report actual registry evidence separately. + +- **Standalone images:** consult [the runtime guide](docs/DOCKER.md). Preserve the matching pinned source in PHP/nginx targets, non-root UID/GID 1000, immutable code and external data/config/secrets. Installation and upgrades stay explicit; process liveness is independent of DB readiness. Host installer lifecycle/backup utilities do not manage this stack. Image tests opt in with `MOOSHIP_IMAGE_TESTS=1`; build both tags and pull the documented DB images first. A local build is not Docker Hub publication or a verified Kubernetes deployment. + - **New installations:** reject occupied project directories and leftover resources. `nginxproxy` is a reserved slug. Keep resolved paths under `/var/www`, reject project-directory symlinks, and match slugs exactly: an operation on `lms` must not affect `lms2`. - **Native databases:** do not reuse existing databases/users or drop them based only on a name. Removal requires a matching `.mooship-native-database` ownership marker. Failed creation or an older project without that marker does not establish ownership. +- **Deployment gates:** validate ports before creating resources; exempt the selected native DB listener from free-port checks. Native `--db-host` controls the application's route to the locally managed DB. Preserve PHP-side TCP/authentication probes, bounded service waits, refusal to initialize unknown/partial DB states, nginx reload after document-root changes, and cron startup after installation/restore. See `docs/PREFLIGHT.md`. - **Database branches:** keep templates and integration checks on the tested branches documented in `docs/CI.md`. A branch change requires coordinated data-volume, migration, and backup/restore validation. Updating a template never authorizes downgrading an existing project's database files. -- **Secrets:** preserve `umask 077`, mode `600` for `.env`, `config.php`, and archives, and localhost-only database, Redis, and PHP-FPM publications. Use the existing environment parser rather than executing `.env` as shell code. Never print credentials or private certificate material in reports; endpoints receiving secrets require HTTPS. +- **Host installer secrets:** preserve `umask 077`, mode `600` for `.env`, `config.php`, and archives, and localhost-only database, Redis, and PHP-FPM publications. Use the existing environment parser rather than executing `.env` as shell code. The standalone image's baked `config.php` contains no secrets; mounted secret files must be readable by UID 1000. Never print credentials or private certificate material in reports; endpoints receiving secrets require HTTPS. + +- **Locked PHP dependencies:** preserve the conditional Composer gate for Moodle's declared OAuth2 dependency. Verify the pinned tool's checksum before execution, install from the lockfile without dev packages/scripts/plugins, preserve platform checks and verify autoloading. Do not replace install with update or hide dependency failures. - **Shared proxy:** other sites depend on `/var/www/nginxproxy`. Use `lib/proxy.py` for lifecycle edits; it targets Mooship's generated configuration format, not arbitrary YAML. Preserve neighboring entries, comments, disabled declarations, and valid empty collections. Deactivation must apply Compose changes so the site's network and certificate volume detach. - **Routing:** preserve per-request Docker DNS resolution of project upstreams so a stopped site cannot prevent proxy startup. Trust the client-address entry appended by the shared proxy. Keep ACME reachable when Basic Auth is enabled, and retain Moodle routing/X-Sendfile behavior, including the `public/` layout for Moodle 5.1+. - **Protocol:** HTTPS is the default. `--no-ssl` must configure HTTP consistently through both nginx layers and Moodle's `wwwroot`. Persist `SITE_SCHEME`; restore older archives without it as HTTPS. Redeploying an occupied slug is not an HTTP-to-HTTPS migration or a Moodle upgrade. @@ -113,6 +129,7 @@ Use [docs/CI.md](docs/CI.md) and the matching workflow for optional checks: - Database round trips: `MOOSHIP_DOCKER_TESTS=1`. - HTTP/HTTPS routing: `MOOSHIP_HTTP_INTEGRATION=1`. - Real proxy lifecycle: `MOOSHIP_PROXY_INTEGRATION=1`. +- Native DB routing/state probes: `MOOSHIP_NATIVE_PREFLIGHT_TESTS=1`, with `MOOSHIP_PREFLIGHT_PHP_IMAGE` set to an existing PHP image with both PDO drivers and `MOOSHIP_PREFLIGHT_DB=postgres` or `mysql`. This uses disposable databases; see `docs/PREFLIGHT.md` and the E2E workflow. - Pre-pull the component images listed in `.github/workflows/integration.yml`; the tests themselves do not pull them. Use its exact test selectors and environment settings. - Install pinned development tools from `.github/requirements.txt` in a separate environment when needed. Keep runtime helpers dependency-free. The combined statement/branch coverage gate is 90% for `lib/`, including subprocesses; it does not measure Bash or Moodle/PHP. diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..26aaba0 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +1.2.0 diff --git a/WIKI.md b/WIKI.md index 606e98b..64b99ff 100644 --- a/WIKI.md +++ b/WIKI.md @@ -5,6 +5,10 @@ Project knowledge map. Public links target files included in the repository. Opt ## Getting started - [README and operations guide](README.md) +- [Standalone Moodle images, Compose, Kubernetes, and Docker Hub](docs/DOCKER.md) +- [Image impact assessment, synchronization, rebuilds, and checks](docs/DOCKER.md#keeping-images-in-sync) +- [Python CLI, pip/pipx installation, and PyPI](docs/PYTHON.md) +- [Release publication and account setup](docs/RELEASING.md) - [Public Mooship AI skill](SKILL.md) - [Project context](PROJECT_CONTEXT.md) - [Git exclusions](.gitignore) @@ -24,6 +28,10 @@ These local resources are not distributed with the repository. Use them when pro ## Decisions and history - [Published architecture decisions](docs/adr/) +- [Standalone Moodle runtime decision](docs/adr/0004-standalone-moodle-images.md) +- [Python CLI and release distribution decision](docs/adr/0006-python-cli-and-release-distribution.md) + +- [Deployment preflight and readiness decision](docs/adr/0005-deployment-preflight.md) - [Changelog](CHANGELOG.md) ## Mooship @@ -31,6 +39,10 @@ These local resources are not distributed with the repository. Use them when pro - [README, live workflow/release badges, and operations guide](README.md) - [Public Mooship AI skill](SKILL.md); optional local Claude Code adapter: `.claude/skills/deploy-moodle/SKILL.md`. - [Installation](deploy-moodle.sh) +- [Host deployment preflight implementation (draft PR #6)](https://github.com/shellharbor/mooship/pull/6) + +- [Port preflight, native DB access, and readiness](docs/PREFLIGHT.md) +- [Locked production dependency gate](lib/install-dependencies.sh) and [failure-path tests](tests/test_dependencies.py) - [Backup](backup.sh), [backup schedule](schedule-backup.sh), and [restore](restore.sh) - [Health check (doctor)](doctor.sh) and its [tests with profiles, mail, and retention](tests/test_operations.py) - [Full backup procedure](docs/BACKUP.md) and [data preservation decision](docs/adr/0001-safe-deploy-and-full-backups.md) diff --git a/deploy-moodle.sh b/deploy-moodle.sh index d5bba7f..25277d7 100644 --- a/deploy-moodle.sh +++ b/deploy-moodle.sh @@ -61,6 +61,7 @@ umask 077 # # Other options: # --redis-password PASS Redis password (also used for Moodle sessions) +# --db-host HOST Native DB address reachable from PHP (default host.docker.internal) # --create-dhparam Create dhparam.pem for SSL (takes several minutes) # --enable-basic-auth Enable HTTP Basic Authentication # --auth-user USER Basic Auth username (generated if omitted) @@ -80,6 +81,7 @@ APP_TYPE="moodle" TEMPLATE_DIR="${SCRIPT_DIR}/${APP_TYPE}" MOODLE_REPOSITORY="https://github.com/moodle/moodle" source "${SCRIPT_DIR}/lib/common.sh" +source "${SCRIPT_DIR}/lib/preflight.sh" # ======================== Colors ============================= RED='\033[0;31m' @@ -143,9 +145,8 @@ find_free_port() { while [[ $ATTEMPTS -lt $MAX_ATTEMPTS ]]; do local PORT=$(( RANDOM % RANGE + MIN )) - # Use ss to check that the port is available - if ! ss -tlnp 2>/dev/null | grep -q ":${PORT} " && \ - ! ss -ulnp 2>/dev/null | grep -q ":${PORT} "; then + # Exclude selected ports, occupied TCP sockets, and Docker publications. + if [[ "$PORT" != 80 && "$PORT" != 443 ]] && mooship_port_available "$PORT"; then echo "$PORT" return 0 fi @@ -338,6 +339,7 @@ DB_POSTGRES_NAME="" DB_POSTGRES_USER="" DB_POSTGRES_PASSWORD="" DB_NATIVE=false +NATIVE_DB_HOST="" DB_ROOT_PASSWORD="" CREATE_DHPARAM=false ENABLE_BASIC_AUTH=false @@ -409,6 +411,7 @@ parse_args() { --db-postgres-user) DB_POSTGRES_USER="$2"; shift 2 ;; --db-postgres-password) DB_POSTGRES_PASSWORD="$2"; shift 2 ;; --db-native) DB_NATIVE=true; shift 1 ;; + --db-host) NATIVE_DB_HOST="$2"; shift 2 ;; --db-root-password) DB_ROOT_PASSWORD="$2"; shift 2 ;; --create-dhparam) CREATE_DHPARAM=true; shift 1 ;; --enable-basic-auth) ENABLE_BASIC_AUTH=true; shift 1 ;; @@ -424,6 +427,10 @@ parse_args() { [[ -z "$DOMAIN" ]] && error "--domain is required" [[ -z "$DB_TYPE" ]] && error "--db-type is required" + if [[ -n "$NATIVE_DB_HOST" ]]; then + [[ "$DB_NATIVE" == true ]] || error "--db-host requires --db-native" + mooship_validate_db_host "$NATIVE_DB_HOST" + fi if [[ "$DB_TYPE" != "postgres" && "$DB_TYPE" != "mysql" ]]; then error "Database type must be 'postgres' or 'mysql', received: ${DB_TYPE}" @@ -616,24 +623,54 @@ select_resource_profile() { # Keep manually specified ports; select the remaining ports from their ranges assign_ports() { info "Selecting free ports..." + local variable port db_variable + local -a variables=(PORT_HTTP PORT_HTTPS PORT_PHP PORT_REDIS) + if [[ "$DB_TYPE" == mysql ]]; then db_variable=PORT_MYSQL; else db_variable=PORT_POSTGRES; fi + variables+=("$db_variable") + if [[ "$DB_NATIVE" == true && -z "${!db_variable}" ]]; then + if [[ "$DB_TYPE" == mysql ]]; then PORT_MYSQL=3306; else PORT_POSTGRES=5432; fi + fi + # Validate every explicit input before probing or creating anything, even unused DB flags. + for variable in PORT_HTTP PORT_HTTPS PORT_PHP PORT_REDIS PORT_MYSQL PORT_POSTGRES; do + port="${!variable}" + [[ -n "$port" ]] || continue + mooship_validate_port "$port" + printf -v "$variable" '%s' "$((10#$port))" + done + MOOSHIP_ASSIGNED_PORTS="" + MOOSHIP_DOCKER_PORTS=$(docker ps --format '{{.Ports}}' 2>/dev/null || true) + for variable in "${variables[@]}"; do + port="${!variable}" + [[ -n "$port" ]] || continue + [[ "$port" != 80 && "$port" != 443 ]] || error "Port $port is reserved for the shared proxy" + [[ " $MOOSHIP_ASSIGNED_PORTS " != *" $port "* ]] || error "Duplicate service port: $port" + if [[ "$DB_NATIVE" != true || "$variable" != "$db_variable" ]]; then + mooship_port_available "$port" || error "Port $port is already in use" + fi + MOOSHIP_ASSIGNED_PORTS+=" $port" + done if [[ -z "$PORT_HTTP" ]]; then PORT_HTTP=$(find_free_port 8100 8400 "HTTP") + MOOSHIP_ASSIGNED_PORTS+=" $PORT_HTTP" fi info " HTTP: ${PORT_HTTP}" if [[ -z "$PORT_HTTPS" ]]; then PORT_HTTPS=$(find_free_port 4100 4300 "HTTPS") + MOOSHIP_ASSIGNED_PORTS+=" $PORT_HTTPS" fi info " HTTPS: ${PORT_HTTPS}" if [[ -z "$PORT_PHP" ]]; then PORT_PHP=$(find_free_port 9100 9600 "PHP-FPM") + MOOSHIP_ASSIGNED_PORTS+=" $PORT_PHP" fi info " PHP-FPM: ${PORT_PHP}" if [[ -z "$PORT_REDIS" ]]; then PORT_REDIS=$(find_free_port 6500 6800 "Redis") + MOOSHIP_ASSIGNED_PORTS+=" $PORT_REDIS" fi info " Redis: ${PORT_REDIS}" @@ -754,11 +791,12 @@ configure_compose_db() { info "Configuring docker-compose.yml for database type ${DB_TYPE}..." fi - python3 - "$COMPOSE_FILE" "$SLUG" "$DB_TYPE" "$DB_NATIVE" <<'PYEOF' + python3 - "$COMPOSE_FILE" "$SLUG" "$DB_TYPE" "$DB_NATIVE" "${NATIVE_DB_HOST:-host.docker.internal}" <<'PYEOF' import re import sys path, slug, db_type, native = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] == 'true' +native_host = sys.argv[5] other = 'mysql' if db_type == 'postgres' else 'postgres' drop_services = {f'db_{other}'} @@ -826,6 +864,13 @@ if native: # --- 3. Rename the selected database: db_postgres → db, _db_postgres → _db --- result = [re.sub(rf'\bdb_{db_type}\b', 'db', l.replace(f'{slug}_db_{db_type}', f'{slug}_db')) for l in result] +if native and native_host == 'host.docker.internal': + mapped = [] + for line in result: + mapped.append(line) + if re.match(r'^ (php|cron):\s*$', line): + mapped.extend([' extra_hosts:', ' - "host.docker.internal:host-gateway"']) + result = mapped with open(path, 'w') as f: f.write('\n'.join(result) + '\n') @@ -870,6 +915,9 @@ write_project_env() { fi echo "" echo "DB_NATIVE=${DB_NATIVE}" + if [[ "$DB_NATIVE" == true ]]; then + echo "DB_HOST=${NATIVE_DB_HOST:-host.docker.internal}" + fi echo "PHP_VERSION=${PHP_VERSION}" echo "MOODLE_VERSION=${MOODLE_VERSION}" echo "" @@ -1338,15 +1386,13 @@ build_and_start_project() { ensure_project_docker_resources info "Running: docker compose up -d --build" - if ! docker compose up -d --build; then + if ! docker compose up -d --build php nginx redis certbot_renew; then error "Could not build/start the project. Check logs: cd ${PROJECT_DIR} && docker compose logs" fi info "The docker compose command completed successfully" - # Allow time for containers to start - info "Waiting for containers to start..." - sleep 10 + info "Waiting for PHP-FPM, nginx, Redis, and the database to become ready..." info "Checking container status..." local RUNNING @@ -1373,12 +1419,13 @@ build_and_start_project() { fi local SERVICE - for SERVICE in php nginx cron redis certbot_renew; do - [[ "$(docker inspect -f '{{.State.Running}}' "${SLUG}_${SERVICE}")" == true ]] || - error "Service ${SERVICE} is not running" + for SERVICE in php nginx redis certbot_renew; do + mooship_wait_service "$SERVICE" || error "Service ${SERVICE} is not ready" done if [[ "$DB_NATIVE" != true ]]; then mooship_wait_database || error "The database is not ready" + else + mooship_check_database tcp || error "Native DB is unreachable; check --db-host, database listeners and firewall" fi cd "${SCRIPT_DIR}" || true } @@ -1440,7 +1487,8 @@ install_moodle() { # Moodle 5.1+: web files moved to public/ (config.php, lib/, and admin/cli remain at the root) if [[ -f "${PROJECT_DIR}/public_html/public/version.php" ]]; then info "Moodle 5.1+: nginx document root → /var/www/html/public" - sed -i 's|^\(#\?\s*\)root /var/www/html;|\1root /var/www/html/public;|' "${PROJECT_DIR}/.config/nginx/_site.conf" + python3 "$SCRIPT_DIR/lib/web.py" --public-root "${PROJECT_DIR}/.config/nginx/_site.conf" || + error "Could not select the Moodle public document root" fi success "Moodle ${MOODLE_VERSION} installed successfully!" @@ -1475,7 +1523,7 @@ configure_moodle_config() { info "Creating config.php for Moodle..." local DB_TYPE_MOODLE="" - local DB_HOST="" + local DB_HOST_VALUE="" local DB_NAME="" local DB_USER="" local DB_PASS="" @@ -1493,13 +1541,13 @@ configure_moodle_config() { DB_PASS="$DB_MYSQL_PASSWORD" fi - # Native database: Docker bridge IP and host port. + # Native database: container-reachable host address and host port. # Container database: container name and internal port (the external .env port does not listen inside the network) if [[ "$DB_NATIVE" == true ]]; then - DB_HOST="172.17.0.1" + DB_HOST_VALUE="${NATIVE_DB_HOST:-host.docker.internal}" if [[ "$DB_TYPE" == "postgres" ]]; then DB_PORT_VALUE="$PORT_POSTGRES"; else DB_PORT_VALUE="$PORT_MYSQL"; fi else - DB_HOST="${SLUG}_db" + DB_HOST_VALUE="${SLUG}_db" if [[ "$DB_TYPE" == "postgres" ]]; then DB_PORT_VALUE="5432"; else DB_PORT_VALUE="3306"; fi fi @@ -1512,7 +1560,7 @@ global \$CFG; \$CFG->dbtype = '${DB_TYPE_MOODLE}'; \$CFG->dblibrary = 'native'; -\$CFG->dbhost = '${DB_HOST}'; +\$CFG->dbhost = '${DB_HOST_VALUE}'; \$CFG->dbname = '${DB_NAME}'; \$CFG->dbuser = '${DB_USER}'; \$CFG->dbpass = '${DB_PASS}'; @@ -1557,7 +1605,7 @@ EOF success "config.php created successfully!" info "Database settings saved in ${MOODLE_CONFIG}:" info " DB_TYPE: ${DB_TYPE_MOODLE}" - info " DB_HOST: ${DB_HOST}" + info " DB_HOST: ${DB_HOST_VALUE}" info " DB_PORT: ${DB_PORT_VALUE}" info " DB_NAME: ${DB_NAME}" info " DB_USER: ${DB_USER}" @@ -1579,14 +1627,14 @@ install_moodle_database() { info "Checking whether Moodle is installed..." cd "${PROJECT_DIR}" || error "Could not change to directory ${PROJECT_DIR}" - local CHECK_OUTPUT - CHECK_OUTPUT=$(docker compose exec -T php php /var/www/html/admin/cli/install_database.php --help 2>&1 || true) - - if echo "$CHECK_OUTPUT" | grep -q "Moodle.*already.*installed"; then + local CHECK_STATUS=0 + mooship_check_database installed || CHECK_STATUS=$? + if [[ "$CHECK_STATUS" == 0 ]]; then info "Moodle is already installed, skipping database installation" cd "${SCRIPT_DIR}" || true return fi + [[ "$CHECK_STATUS" == 3 ]] || error "Could not verify the Moodle installation state" info "Starting automatic Moodle installation through CLI..." info "This may take several minutes..." @@ -1980,12 +2028,16 @@ main() { create_dhparam create_project comment_ssl_blocks - create_native_database build_and_start_project + create_native_database + (cd "$PROJECT_DIR" && mooship_check_database connect) || error "PHP cannot authenticate to the selected database" install_moodle + install_moodle_dependencies configure_moodle_config + reload_project_nginx install_moodle_database configure_moodle_mail + start_moodle_cron # Publish upstreams only after the application and its Docker network exist. update_proxy_nginx_conf update_proxy_docker_compose diff --git a/deploy/k8s/config.yml b/deploy/k8s/config.yml new file mode 100644 index 0000000..f5aa910 --- /dev/null +++ b/deploy/k8s/config.yml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: mooship +data: + MOODLE_DB_TYPE: pgsql + MOODLE_DB_HOST: database.example.internal + MOODLE_DB_NAME: moodle + MOODLE_DB_USER: moodle + MOODLE_DB_PASSWORD_FILE: /run/secrets/db/password + MOODLE_WWWROOT: http://moodle.example.org +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: mooship-data +spec: + accessModes: [ReadWriteMany] + resources: + requests: + storage: 10Gi diff --git a/deploy/k8s/install.yml b/deploy/k8s/install.yml new file mode 100644 index 0000000..64f7d8d --- /dev/null +++ b/deploy/k8s/install.yml @@ -0,0 +1,49 @@ +# Run explicitly against an empty database before starting the Deployment. +apiVersion: batch/v1 +kind: Job +metadata: + name: mooship-install +spec: + backoffLimit: 0 + template: + spec: + automountServiceAccountToken: false + restartPolicy: Never + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + seccompProfile: {type: RuntimeDefault} + containers: + - name: install + image: shellharbor/mooship:dev-moodle4.5.15-php8.3 + args: [install] + envFrom: + - configMapRef: {name: mooship} + env: + - {name: MOODLE_AGREE_LICENSE, value: 'true'} + - {name: MOODLE_ADMIN_EMAIL, value: admin@example.org} + - {name: MOODLE_ADMIN_PASSWORD_FILE, value: /run/secrets/admin/password} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: [ALL]} + resources: + requests: {cpu: 250m, memory: 512Mi} + limits: {cpu: '2', memory: 2Gi} + volumeMounts: + - {name: data, mountPath: /var/moodledata} + - {name: tmp, mountPath: /tmp} + - {name: db-secret, mountPath: /run/secrets/db, readOnly: true} + - {name: admin-secret, mountPath: /run/secrets/admin, readOnly: true} + volumes: + - name: data + persistentVolumeClaim: {claimName: mooship-data} + - name: tmp + emptyDir: {} + - name: db-secret + secret: {secretName: mooship-db} + - name: admin-secret + secret: {secretName: mooship-admin} diff --git a/deploy/k8s/runtime.yml b/deploy/k8s/runtime.yml new file mode 100644 index 0000000..6406b9a --- /dev/null +++ b/deploy/k8s/runtime.yml @@ -0,0 +1,171 @@ +apiVersion: v1 +kind: Service +metadata: + name: mooship +spec: + selector: + app: mooship + ports: + - name: http + port: 80 + targetPort: http +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mooship +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: mooship + template: + metadata: + labels: + app: mooship + spec: + automountServiceAccountToken: false + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: app + image: shellharbor/mooship:dev-moodle4.5.15-php8.3 + args: [fpm] + envFrom: + - configMapRef: + name: mooship + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + # Initial estimates, tune after measuring the actual workload. + resources: + requests: {cpu: 250m, memory: 512Mi} + limits: {cpu: '2', memory: 2Gi} + startupProbe: + exec: + command: [mooship-healthcheck] + periodSeconds: 2 + failureThreshold: 30 + livenessProbe: + exec: + command: [mooship-healthcheck] + timeoutSeconds: 5 + readinessProbe: + exec: + command: [mooship, check] + timeoutSeconds: 8 + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - 'pid=$(cat /tmp/php-fpm.pid); kill -QUIT "$pid"; while kill -0 "$pid" 2>/dev/null; do sleep 1; done' + volumeMounts: + - {name: data, mountPath: /var/moodledata} + - {name: app-tmp, mountPath: /tmp} + - {name: db-secret, mountPath: /run/secrets/db, readOnly: true} + - name: web + image: shellharbor/mooship:dev-moodle4.5.15-nginx + env: + - name: MOOSHIP_FPM_UPSTREAM + value: localhost:9000 + ports: + - {name: http, containerPort: 8080} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + resources: + requests: {cpu: 50m, memory: 32Mi} + limits: {cpu: '1', memory: 256Mi} + startupProbe: + httpGet: {path: /healthz, port: http} + periodSeconds: 2 + failureThreshold: 30 + livenessProbe: + httpGet: {path: /healthz, port: http} + readinessProbe: + httpGet: {path: /readyz, port: http} + timeoutSeconds: 8 + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - 'pid=$(cat /tmp/nginx.pid); kill -QUIT "$pid"; while kill -0 "$pid" 2>/dev/null; do sleep 1; done' + volumeMounts: + - {name: data, mountPath: /var/moodledata, readOnly: true} + - {name: web-tmp, mountPath: /tmp} + volumes: + - name: data + persistentVolumeClaim: + claimName: mooship-data + - name: app-tmp + emptyDir: {} + - name: web-tmp + emptyDir: {} + - name: db-secret + secret: + secretName: mooship-db +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: mooship-cron +spec: + schedule: '* * * * *' + concurrencyPolicy: Forbid + startingDeadlineSeconds: 60 + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 2 + jobTemplate: + spec: + backoffLimit: 0 + template: + spec: + automountServiceAccountToken: false + restartPolicy: Never + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + seccompProfile: {type: RuntimeDefault} + containers: + - name: cron + image: shellharbor/mooship:dev-moodle4.5.15-php8.3 + args: [cron-once] + envFrom: + - configMapRef: {name: mooship} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: [ALL]} + resources: + requests: {cpu: 100m, memory: 256Mi} + limits: {cpu: '1', memory: 1Gi} + volumeMounts: + - {name: data, mountPath: /var/moodledata} + - {name: tmp, mountPath: /tmp} + - {name: db-secret, mountPath: /run/secrets/db, readOnly: true} + volumes: + - name: data + persistentVolumeClaim: {claimName: mooship-data} + - name: tmp + emptyDir: {} + - name: db-secret + secret: {secretName: mooship-db} diff --git a/docker/.env.example b/docker/.env.example new file mode 100644 index 0000000..57558d1 --- /dev/null +++ b/docker/.env.example @@ -0,0 +1,10 @@ +# Copy to docker/.env and supply separate secret files in .secrets/. +MOODLE_DB_TYPE=pgsql +MOODLE_WWWROOT=http://localhost:8080 +MOODLE_ADMIN_EMAIL=admin@example.org +MOODLE_SITE_FULLNAME=Mooship Moodle +MOODLE_SITE_SHORTNAME=Mooship +MOOSHIP_BIND_ADDRESS=127.0.0.1 +MOOSHIP_HTTP_PORT=8080 +# For TLS termination at a trusted reverse proxy, use an HTTPS WWWROOT +# and set MOODLE_SSL_PROXY=true. See docs/DOCKER.md before enabling proxy flags. diff --git a/docker/compose.mysql.yml b/docker/compose.mysql.yml new file mode 100644 index 0000000..96d5a5d --- /dev/null +++ b/docker/compose.mysql.yml @@ -0,0 +1,25 @@ +services: + db: + image: mysql:8.4 + command: [--character-set-server=utf8mb4, --collation-server=utf8mb4_unicode_ci] + restart: unless-stopped + environment: + MYSQL_DATABASE: moodle + MYSQL_USER: moodle + MYSQL_PASSWORD_FILE: /run/secrets/db_password + MYSQL_ROOT_PASSWORD_FILE: /run/secrets/mysql_root_password + secrets: [db_password, mysql_root_password] + volumes: + - database:/var/lib/mysql + healthcheck: + test: [CMD-SHELL, 'mysqladmin ping --silent -h 127.0.0.1'] + interval: 5s + timeout: 5s + retries: 30 + +volumes: + database: + +secrets: + mysql_root_password: + file: ../.secrets/mysql_root_password diff --git a/docker/compose.postgres.yml b/docker/compose.postgres.yml new file mode 100644 index 0000000..ab45224 --- /dev/null +++ b/docker/compose.postgres.yml @@ -0,0 +1,19 @@ +services: + db: + image: postgres:17-alpine + restart: unless-stopped + environment: + POSTGRES_DB: moodle + POSTGRES_USER: moodle + POSTGRES_PASSWORD_FILE: /run/secrets/db_password + secrets: [db_password] + volumes: + - database:/var/lib/postgresql/data + healthcheck: + test: [CMD-SHELL, 'pg_isready -U moodle -d moodle'] + interval: 5s + timeout: 5s + retries: 30 + +volumes: + database: diff --git a/docker/compose.yml b/docker/compose.yml new file mode 100644 index 0000000..e5fcc3e --- /dev/null +++ b/docker/compose.yml @@ -0,0 +1,89 @@ +name: mooship + +x-runtime: &runtime + image: ${MOOSHIP_APP_IMAGE:-shellharbor/mooship:dev-moodle4.5.15-php8.3} + build: + context: .. + target: app + read_only: true + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + tmpfs: + - /tmp:uid=1000,gid=1000,mode=1770 + volumes: + - moodledata:/var/moodledata + environment: &environment + MOODLE_DB_TYPE: ${MOODLE_DB_TYPE:-pgsql} + MOODLE_DB_HOST: db + MOODLE_DB_NAME: moodle + MOODLE_DB_USER: moodle + MOODLE_DB_PASSWORD_FILE: /run/secrets/db_password + MOODLE_WWWROOT: ${MOODLE_WWWROOT:-http://localhost:8080} + MOODLE_REVERSE_PROXY: ${MOODLE_REVERSE_PROXY:-false} + MOODLE_SSL_PROXY: ${MOODLE_SSL_PROXY:-false} + secrets: [db_password] + depends_on: + db: + condition: service_healthy + +services: + app: + <<: *runtime + restart: unless-stopped + stop_grace_period: 60s + + cron: + <<: *runtime + command: [cron] + restart: unless-stopped + stop_grace_period: 60s + healthcheck: + disable: true + + install: + <<: *runtime + profiles: [init] + command: [install] + restart: 'no' + environment: + <<: *environment + MOODLE_AGREE_LICENSE: 'true' + MOODLE_ADMIN_USER: ${MOODLE_ADMIN_USER:-admin} + MOODLE_ADMIN_EMAIL: ${MOODLE_ADMIN_EMAIL:?Set MOODLE_ADMIN_EMAIL} + MOODLE_ADMIN_PASSWORD_FILE: /run/secrets/admin_password + MOODLE_SITE_FULLNAME: ${MOODLE_SITE_FULLNAME:-Mooship Moodle} + MOODLE_SITE_SHORTNAME: ${MOODLE_SITE_SHORTNAME:-Mooship} + secrets: [db_password, admin_password] + healthcheck: + disable: true + + web: + image: ${MOOSHIP_WEB_IMAGE:-shellharbor/mooship:dev-moodle4.5.15-nginx} + build: + context: .. + target: web + restart: unless-stopped + read_only: true + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + stop_grace_period: 60s + tmpfs: + - /tmp:uid=1000,gid=1000,mode=1770 + volumes: + - moodledata:/var/moodledata:ro + environment: + MOOSHIP_FPM_UPSTREAM: app:9000 + ports: + - '${MOOSHIP_BIND_ADDRESS:-127.0.0.1}:${MOOSHIP_HTTP_PORT:-8080}:8080' + depends_on: + app: + condition: service_healthy + +volumes: + moodledata: + +secrets: + db_password: + file: ../.secrets/db_password + admin_password: + file: ../.secrets/admin_password diff --git a/docker/config.php b/docker/config.php new file mode 100644 index 0000000..f7b0a8d --- /dev/null +++ b/docker/config.php @@ -0,0 +1,36 @@ +dbtype = $configuration['type']; +$CFG->dblibrary = 'native'; +$CFG->dbhost = $configuration['host']; +$CFG->dbname = $configuration['name']; +$CFG->dbuser = $configuration['user']; +$CFG->dbpass = $configuration['password']; +$CFG->prefix = $configuration['prefix']; +$CFG->dboptions = ['dbpersist' => false, 'dbsocket' => false, 'dbport' => $configuration['port']]; +if ($configuration['type'] === 'mysqli') { + $CFG->dboptions['dbcollation'] = 'utf8mb4_unicode_ci'; +} +$CFG->wwwroot = $configuration['wwwroot']; +$CFG->dataroot = '/var/moodledata'; +$CFG->directorypermissions = 02770; +$CFG->admin = 'admin'; +$CFG->reverseproxy = $configuration['reverseproxy']; +$CFG->sslproxy = $configuration['sslproxy']; +$CFG->routerconfigured = true; +$CFG->xsendfile = 'X-Accel-Redirect'; +$CFG->xsendfilealiases = ['/dataroot/' => $CFG->dataroot]; +$CFG->disableupdateautodeploy = true; +if (\Mooship\value('MOODLE_REDIS_HOST') !== '') { + $CFG->session_handler_class = '\core\session\redis'; + $CFG->session_redis_host = \Mooship\value('MOODLE_REDIS_HOST'); + $CFG->session_redis_port = (int)\Mooship\value('MOODLE_REDIS_PORT', '6379'); + $CFG->session_redis_auth = \Mooship\secret('MOODLE_REDIS_PASSWORD'); + $CFG->session_redis_prefix = 'mooship_session_'; +} +require_once __DIR__ . '/lib/setup.php'; diff --git a/docker/cron.sh b/docker/cron.sh new file mode 100644 index 0000000..2fd7bee --- /dev/null +++ b/docker/cron.sh @@ -0,0 +1,22 @@ +#!/bin/sh +set -eu +child='' +finish() { + trap - TERM INT QUIT + if [ -n "$child" ]; then + kill -TERM "$child" 2>/dev/null || true + wait "$child" 2>/dev/null || true + fi + exit 0 +} +trap finish TERM INT QUIT +while :; do + php /opt/moodle/admin/cli/cron.php --keep-alive=0 & + child=$! + if ! wait "$child"; then exit 1; fi + child='' + sleep 60 & + child=$! + wait "$child" + child='' +done diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100644 index 0000000..9ab13b2 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,24 @@ +#!/bin/sh +set -eu +umask 077 +if [ "$#" -eq 0 ]; then set -- fpm; fi +case "$1" in + --help|-h) + printf '%s\n' 'Mooship Moodle runtime: fpm | install | cron | cron-once | check | php ...' + ;; + install) exec php /opt/mooship/install.php ;; + check) exec php /opt/mooship/readiness.php ;; + fpm|cron|cron-once) + php /opt/mooship/validate.php + if ! php /opt/mooship/readiness.php; then + printf '%s\n' '[ERROR] Install the database first, or verify its version and connectivity.' >&2 + exit 1 + fi + case "$1" in + fpm) exec php-fpm -F ;; + cron) exec /opt/mooship/cron.sh ;; + cron-once) exec php /opt/moodle/admin/cli/cron.php --keep-alive=0 ;; + esac + ;; + *) exec "$@" ;; +esac diff --git a/docker/environment.php b/docker/environment.php new file mode 100644 index 0000000..ebc7758 --- /dev/null +++ b/docker/environment.php @@ -0,0 +1,115 @@ + 65536) { + throw new \InvalidArgumentException("Cannot read the secret file for {$name}."); + } + $contents = @file_get_contents($file); + if ($contents === false) { + throw new \InvalidArgumentException("Cannot read the secret file for {$name}."); + } + $value = rtrim($contents, "\r\n"); + } + if ($required && $value === '') { + throw new \InvalidArgumentException("A secret is required for {$name}."); + } + return $value; +} + +function flag(string $name): bool { + $value = value($name, 'false'); + $result = filter_var($value, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE); + if ($result === null) { + throw new \InvalidArgumentException("Invalid boolean setting: {$name}."); + } + return $result; +} + +function settings(): array { + $type = value('MOODLE_DB_TYPE', 'pgsql'); + if (!in_array($type, ['pgsql', 'mysqli'], true)) { + throw new \InvalidArgumentException('MOODLE_DB_TYPE must be pgsql or mysqli.'); + } + $host = value('MOODLE_DB_HOST', 'db'); + $name = value('MOODLE_DB_NAME', 'moodle'); + $user = value('MOODLE_DB_USER', 'moodle'); + if (!preg_match('/^[a-zA-Z0-9_.:-]+$/D', $host)) { + throw new \InvalidArgumentException('Invalid MOODLE_DB_HOST.'); + } + foreach ([$name, $user] as $identifier) { + if (!preg_match('/^[a-zA-Z][a-zA-Z0-9_]{0,62}$/D', $identifier)) { + throw new \InvalidArgumentException('Invalid database name or user.'); + } + } + $port = value('MOODLE_DB_PORT', $type === 'pgsql' ? '5432' : '3306'); + if (!ctype_digit($port) || (int)$port < 1 || (int)$port > 65535) { + throw new \InvalidArgumentException('Invalid MOODLE_DB_PORT.'); + } + $prefix = value('MOODLE_DB_PREFIX', 'mdl_'); + if (!preg_match('/^[a-z][a-z0-9_]{0,9}$/D', $prefix)) { + throw new \InvalidArgumentException('MOODLE_DB_PREFIX must contain 1-10 safe characters.'); + } + $wwwroot = rtrim(value('MOODLE_WWWROOT'), '/'); + $url = parse_url($wwwroot); + if (!filter_var($wwwroot, FILTER_VALIDATE_URL) || !is_array($url) || + !in_array($url['scheme'] ?? '', ['http', 'https'], true) || + isset($url['user']) || isset($url['pass']) || isset($url['query']) || isset($url['fragment']) || + !empty($url['path'])) { + throw new \InvalidArgumentException('MOODLE_WWWROOT must be an HTTP(S) origin without a subdirectory.'); + } + return [ + 'type' => $type, 'host' => $host, 'name' => $name, 'user' => $user, + 'password' => secret('MOODLE_DB_PASSWORD'), 'port' => (int)$port, + 'prefix' => $prefix, 'wwwroot' => $wwwroot, + 'reverseproxy' => flag('MOODLE_REVERSE_PROXY'), 'sslproxy' => flag('MOODLE_SSL_PROXY'), + ]; +} + +function connection(array $configuration): \PDO { + if ($configuration['type'] === 'pgsql') { + $dsn = "pgsql:host={$configuration['host']};port={$configuration['port']};" . + "dbname={$configuration['name']};connect_timeout=5"; + } else { + $dsn = "mysql:host={$configuration['host']};port={$configuration['port']};" . + "dbname={$configuration['name']};charset=utf8mb4"; + } + return new \PDO($dsn, $configuration['user'], $configuration['password'], [ + \PDO::ATTR_ERRMODE => \PDO::ERRMODE_EXCEPTION, \PDO::ATTR_TIMEOUT => 5, + ]); +} + +function validate(array $configuration): void { + $workers = value('PHP_FPM_MAX_CHILDREN', '4'); + if (!ctype_digit($workers) || (int)$workers < 2 || (int)$workers > 256) { + throw new \InvalidArgumentException('PHP_FPM_MAX_CHILDREN must be between 2 and 256.'); + } + if (!is_writable('/var/moodledata') || !is_writable('/tmp')) { + throw new \RuntimeException('The Moodle data directory and /tmp must be writable by UID 1000.'); + } + if (!is_file('/opt/moodle/version.php')) { + throw new \RuntimeException('The image does not contain Moodle source.'); + } +} + +function ready(array $configuration): bool { + validate($configuration); + $metadata = json_decode(file_get_contents('/opt/mooship/version.json'), true, 512, JSON_THROW_ON_ERROR); + $database = connection($configuration); + $query = $database->prepare("SELECT value FROM {$configuration['prefix']}config WHERE name = :name"); + $query->execute(['name' => 'version']); + $version = $query->fetchColumn(); + return $version !== false && (float)$version === (float)$metadata['version']; +} diff --git a/docker/health.txt b/docker/health.txt new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/docker/health.txt @@ -0,0 +1 @@ +ok diff --git a/docker/healthcheck.sh b/docker/healthcheck.sh new file mode 100644 index 0000000..1097b9a --- /dev/null +++ b/docker/healthcheck.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +response=$(SCRIPT_NAME=/fpm-ping SCRIPT_FILENAME=/fpm-ping REQUEST_METHOD=GET \ + cgi-fcgi -bind -connect 127.0.0.1:9000 2>/dev/null) +printf '%s\n' "$response" | tr -d '\r' | grep -qx pong diff --git a/docker/install.php b/docker/install.php new file mode 100644 index 0000000..b3cd559 --- /dev/null +++ b/docker/install.php @@ -0,0 +1,46 @@ +query( + "SELECT pg_try_advisory_lock(hashtext(current_database()), hashtext('mooship-install'))" + )->fetchColumn(); + $tables = $mooshipInstallConnection->query( + 'SELECT tablename FROM pg_tables WHERE schemaname = current_schema()' + )->fetchAll(); + } else { + $lock = $mooshipInstallConnection->prepare('SELECT GET_LOCK(:name, 0)'); + $lock->execute(['name' => 'mooship-' . substr(hash('sha256', $configuration['name']), 0, 40)]); + $locked = $lock->fetchColumn(); + $tables = $mooshipInstallConnection->query('SHOW TABLES')->fetchAll(); + } + if (!$locked || $tables) { + throw new RuntimeException('Installation requires an empty database and an exclusive installer lock.'); + } + $argv = [ + '/opt/moodle/admin/cli/install_database.php', '--agree-license', + '--adminuser=' . \Mooship\value('MOODLE_ADMIN_USER', 'admin'), '--adminpass=' . $password, + '--adminemail=' . $email, '--fullname=' . \Mooship\value('MOODLE_SITE_FULLNAME', 'Mooship Moodle'), + '--shortname=' . \Mooship\value('MOODLE_SITE_SHORTNAME', 'Mooship'), '--lang=en', + ]; + $argc = count($argv); + $_SERVER['argv'] = $argv; + $_SERVER['argc'] = $argc; +} catch (Throwable $error) { + $message = $error instanceof PDOException ? 'Cannot connect to the installation database.' : $error->getMessage(); + fwrite(STDERR, '[ERROR] ' . $message . PHP_EOL); + exit(1); +} +require '/opt/moodle/admin/cli/install_database.php'; diff --git a/docker/nginx.conf.template b/docker/nginx.conf.template new file mode 100644 index 0000000..d10ceb7 --- /dev/null +++ b/docker/nginx.conf.template @@ -0,0 +1,46 @@ +pid /tmp/nginx.pid; +error_log /dev/stderr warn; +worker_processes auto; +events { worker_connections 1024; } +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + access_log /dev/stdout; + server_tokens off; + client_body_temp_path /tmp/client_body; + fastcgi_temp_path /tmp/fastcgi; + proxy_temp_path /tmp/proxy; + uwsgi_temp_path /tmp/uwsgi; + scgi_temp_path /tmp/scgi; + server { + listen 8080; + root /opt/moodle; + index index.php; + client_max_body_size 1100m; + location = /healthz { alias /opt/mooship/health.txt; } + location = /readyz { + include /etc/nginx/fastcgi_params; + fastcgi_param SCRIPT_FILENAME /opt/mooship/readiness.php; + fastcgi_pass ${MOOSHIP_FPM_UPSTREAM}; + } + location ~ ^/config\.php(?:/|$) { return 404; } + location ~ /\.(?!well-known) { return 404; } + location ~ ^/admin/cli/ { return 404; } + location ~ (/vendor/|/node_modules/|composer\.json|/readme|/README|\.lock$|/fixtures/|/behat/|db/install\.xml|environment\.xml) { + return 404; + } + location /dataroot/ { internal; alias /var/moodledata/; } + location / { try_files $uri $uri/ /r.php$is_args$args; } + location ~ \.php(?:/|$) { + fastcgi_split_path_info ^(.+\.php)(/.*)$; + set $saved_path_info $fastcgi_path_info; + try_files $fastcgi_script_name =404; + include /etc/nginx/fastcgi_params; + fastcgi_param PATH_INFO $saved_path_info; + fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; + fastcgi_param HTTP_PROXY ""; + fastcgi_read_timeout 300s; + fastcgi_pass ${MOOSHIP_FPM_UPSTREAM}; + } + } +} diff --git a/docker/php-fpm.conf b/docker/php-fpm.conf new file mode 100644 index 0000000..569af65 --- /dev/null +++ b/docker/php-fpm.conf @@ -0,0 +1,18 @@ +[global] +daemonize = no +error_log = /proc/self/fd/2 +pid = /tmp/php-fpm.pid + +[www] +listen = 0.0.0.0:9000 +pm = dynamic +pm.max_children = ${PHP_FPM_MAX_CHILDREN} +pm.start_servers = 2 +pm.min_spare_servers = 1 +pm.max_spare_servers = 2 +pm.max_requests = 500 +catch_workers_output = yes +decorate_workers_output = no +clear_env = no +ping.path = /fpm-ping +security.limit_extensions = .php diff --git a/docker/readiness.php b/docker/readiness.php new file mode 100644 index 0000000..7e910a9 --- /dev/null +++ b/docker/readiness.php @@ -0,0 +1,12 @@ +getMessage() . PHP_EOL); + exit(1); +} diff --git a/docker/web-entrypoint.sh b/docker/web-entrypoint.sh new file mode 100644 index 0000000..1f3a018 --- /dev/null +++ b/docker/web-entrypoint.sh @@ -0,0 +1,16 @@ +#!/bin/sh +set -eu +umask 077 +: "${MOOSHIP_FPM_UPSTREAM:=app:9000}" +export MOOSHIP_FPM_UPSTREAM +if ! printf '%s' "$MOOSHIP_FPM_UPSTREAM" | grep -Eq '^[A-Za-z0-9_.-]+:[0-9]{1,5}$'; then + printf '%s\n' '[ERROR] MOOSHIP_FPM_UPSTREAM must be a hostname:port.' >&2 + exit 1 +fi +port=${MOOSHIP_FPM_UPSTREAM##*:} +if [ "$port" -lt 1 ] || [ "$port" -gt 65535 ]; then + printf '%s\n' '[ERROR] Invalid FastCGI port.' >&2 + exit 1 +fi +envsubst '${MOOSHIP_FPM_UPSTREAM}' < /opt/mooship/nginx.conf.template > /tmp/nginx.conf +exec nginx -c /tmp/nginx.conf -g 'daemon off;' diff --git a/docs/CI.md b/docs/CI.md index 7953bc7..3535c9a 100644 --- a/docs/CI.md +++ b/docs/CI.md @@ -9,6 +9,7 @@ Mooship separates fast regression checks, component tests, and full Moodle deplo | [CI](../.github/workflows/ci.yml) | Bash syntax and ShellCheck including examples; Ruff; YAML syntax/duplicate keys; actionlint; zizmor; Python regression matrix; branch coverage | Push to `main`, PR, manual | | [Configuration](../.github/workflows/configuration.yml) | Raw Compose YAML/image references for Dependabot; generated PostgreSQL/MySQL Compose files with native/container databases; proxy lifecycle states; all PHP/nginx Dockerfiles with Hadolint | Push to `main`, PR, manual | | [Component integration](../.github/workflows/integration.yml) | Separate PostgreSQL 17, MySQL 8.4, HTTP/HTTPS, and proxy lifecycle jobs | Push to `main`, PR, manual | +| [Moodle images](../.github/workflows/image.yml) | Build PHP/nginx images, validate runtime Compose, install real Moodle on both DBs, check HTTP/secrets/health/shutdown | Push to `main`, PR, manual | | [Security](../.github/workflows/security.yml) | Bandit on Python helpers; Gitleaks on repository files and Git history, with redacted output | Push to `main`, PR, Wednesday, manual | | [CodeQL](../.github/workflows/codeql.yml) | Python and GitHub Actions security analysis | Push to `main`, PR, Wednesday, manual | | [Dependency review](../.github/workflows/dependency-review.yml) | Reject newly introduced high/critical dependency vulnerabilities reported by GitHub | PR | @@ -19,6 +20,8 @@ The regression matrix uses Python 3.8, 3.12, and 3.14 on Ubuntu 24.04, plus Pyth ## Coverage +The [Python package workflow](../.github/workflows/python-package.yml) checks CLI behavior, wheel/sdist metadata, wheel rebuilds from the sdist, safe native archive extraction and installed commands on Python 3.8/3.12/3.14. The [Release workflow](../.github/workflows/release.yml) requires successful exact-commit checks and matching tag/VERSION before uploading native assets or optionally publishing checked images and distributions. See [release configuration](RELEASING.md). + [`.coveragerc`](../.coveragerc) measures statement and branch coverage of `lib/`, including Python child processes launched through Bash. The gate requires at least 90% combined coverage. The locally verified starting level is 92%; coverage does not measure Bash or Moodle/PHP code. The workflow stores `coverage.xml` as an artifact for 14 days. ## Security and dependency updates @@ -31,7 +34,7 @@ The runtime uses Python's standard library. CI packages are installed separately CodeQL analyzes Python and workflow code. Bash is checked by ShellCheck and behavioral tests. Only CodeQL jobs request `security-events: write`; PR dependency review does not post comments. The public repository supports these GitHub features; local validation cannot confirm their server-side configuration or successful runs. -Hadolint errors fail CI. Existing package pinning and `apt` style warnings remain visible until the legacy Dockerfiles are improved. [Bandit configuration](../.github/bandit.yml) suppresses subprocess import/argument-list warnings for the fixed, shell-free repository inspection command; shell execution checks remain active. [Gitleaks configuration](../.gitleaks.toml) excludes local tool/report directories and exactly matches the deterministic database fixture passwords in their test file. Other findings fail CI. +Hadolint errors fail CI. Package pinning and shell/`apt` style warnings remain visible. [Bandit configuration](../.github/bandit.yml) suppresses subprocess import/argument-list warnings for the fixed, shell-free repository inspection command; shell execution checks remain active. [Gitleaks configuration](../.gitleaks.toml) excludes local tool/report directories and exactly matches the deterministic database fixture passwords in their test file. Other findings fail CI. Secret scanning explicitly trusts only the mounted checkout and checks that Git can read a nonempty history before scanning all refs. This prevents a runner/container ownership mismatch from producing a misleading successful scan of zero commits. @@ -46,7 +49,7 @@ python3 -m venv .venv . .venv/bin/activate python -m pip install -r .github/requirements.txt python -m ruff check lib tests .github/scripts -python -m yamllint .github +python -m yamllint .github docker deploy/k8s zizmor --offline --strict-collection .github python -m bandit -r lib .github/scripts -c .github/bandit.yml python -B .github/scripts/check_docs.py @@ -65,6 +68,6 @@ MOOSHIP_COMPOSE_TESTS=1 MOOSHIP_PROXY_COMPOSE_TESTS=1 PYTHONPATH=tests \ test_proxy_lifecycle.ProxyComposeTests -v ``` -Component tests require a running Docker daemon and locally available images. Workflows pull their component's images before enabling the opt-in tests; tests themselves never pull images or publish host ports. See the [development checks](../README.md#development-checks) and the integration workflow for exact image lists and commands. +Component tests require a running Docker daemon and locally available images. Workflows pull their component's images before enabling the opt-in tests; tests themselves never pull images. The regular component suites use internal networks without host publications. E2E's native-routing fixture intentionally publishes a disposable synthetic DB on a random host port to exercise PHP's host-gateway route; see [preflight tests](PREFLIGHT.md). See the [development checks](../README.md#development-checks) and the integration workflow for exact image lists and commands. Full deployments run on disposable GitHub runners and use HTTP and a synthetic domain. DNS and actual Let's Encrypt issuance require separate target-server checks. The workflow definitions and locally executed checks do not establish a production deployment or branch protection policy. diff --git a/docs/DOCKER.md b/docs/DOCKER.md new file mode 100644 index 0000000..03f0dee --- /dev/null +++ b/docs/DOCKER.md @@ -0,0 +1,158 @@ +# Standalone Moodle images + +The root Dockerfile builds a Moodle 4.5.15/PHP 8.3 runtime and a matching nginx companion for later publication as `shellharbor/mooship`. PostgreSQL 17 or MySQL 8.4 runs separately. The existing host installer is an independent deployment option. These application images do not manage Docker or host directories. + +Build these images locally until they are published. The `dev-*` tags below identify local builds, not an established release. Runtime verification covers Linux amd64; other architectures require their own build and installation checks. + +## Build and install with Compose + +Run from the repository root on a Linux Docker host with Compose v2+. Docker Desktop can build/test the images; production secret-file ownership examples assume Linux. + +```bash +docker build --target app -t shellharbor/mooship:dev-moodle4.5.15-php8.3 . +docker build --target web -t shellharbor/mooship:dev-moodle4.5.15-nginx . +cp docker/.env.example docker/.env +``` + +Set `MOODLE_ADMIN_EMAIL` and the public `MOODLE_WWWROOT` in `docker/.env`. The URL must be an HTTP(S) origin without a subdirectory. Generate distinct passwords in ignored files, only for an initial deployment: + +```bash +install -d -m 0700 .secrets +for name in db_password admin_password mysql_root_password; do + test ! -e ".secrets/$name" || exit 1 + printf 'Aa9!%s\n' "$(openssl rand -hex 32)" > ".secrets/$name" + chmod 0600 ".secrets/$name" +done +sudo chown 1000:1000 .secrets/* +``` + +Compose file secrets are bind mounts: their host permissions must allow the PHP process (UID/GID 1000) to read them. Do not make secrets world-readable as a production workaround. The database password is shared with the database; the admin password is mounted only into the installer. `mysql_root_password` is needed only for MySQL. Store the initial admin password securely before removing its file after installation. + +For PostgreSQL, always use the base file and PostgreSQL overlay together: + +```bash +docker compose --env-file docker/.env -f docker/compose.yml -f docker/compose.postgres.yml up -d db +docker compose --env-file docker/.env -f docker/compose.yml -f docker/compose.postgres.yml run --rm install +docker compose --env-file docker/.env -f docker/compose.yml -f docker/compose.postgres.yml up -d app cron web +``` + +For MySQL, set `MOODLE_DB_TYPE=mysqli` in `docker/.env` and replace the PostgreSQL overlay with `docker/compose.mysql.yml` in all three commands. Choose the engine before creating its volume; switching overlays does not migrate data. Installation requires an empty database and explicit license acceptance (`MOODLE_AGREE_LICENSE=true` in the installer service), takes an advisory DB lock, and refuses populated databases. Check installer success before starting services. Normal startup never installs or upgrades the schema. + +The example binds `http://localhost:8080` on loopback; DB and FastCGI publish no host ports. On a server, place a trusted TLS reverse proxy before nginx. Set an HTTPS origin and `MOODLE_SSL_PROXY=true` when the proxy terminates TLS, and restrict backend access. Enable `MOODLE_REVERSE_PROXY` only when the topology requires Moodle's reverse-proxy mode. This nginx example does not trust arbitrary incoming `X-Forwarded-For` headers; configure trusted client-address forwarding for your network. Certificate management is external. + +## Configuration and commands + +| Setting | Default / requirement | +| --- | --- | +| `MOODLE_WWWROOT` | Required HTTP(S) origin | +| `MOODLE_DB_TYPE` | `pgsql` (default) or `mysqli` | +| `MOODLE_DB_HOST`, `MOODLE_DB_PORT` | `db`; engine port 5432/3306 | +| `MOODLE_DB_NAME`, `MOODLE_DB_USER`, `MOODLE_DB_PREFIX` | `moodle`, `moodle`, `mdl_` | +| `MOODLE_DB_PASSWORD` / `MOODLE_DB_PASSWORD_FILE` | Required; choose exactly one | +| `MOODLE_REVERSE_PROXY`, `MOODLE_SSL_PROXY` | `false`; explicit booleans | +| `MOODLE_REDIS_HOST`, `MOODLE_REDIS_PORT` | Optional session store; default port 6379 | +| `MOODLE_REDIS_PASSWORD` / `MOODLE_REDIS_PASSWORD_FILE` | Required if Redis is enabled | +| `PHP_FPM_MAX_CHILDREN` | `4`; at least 2 with the bundled dynamic pool | +| `MOOSHIP_FPM_UPSTREAM` | nginx only; `app:9000`, configurable hostname:port | +| `MOODLE_AGREE_LICENSE` | `true` required for installation | +| `MOODLE_ADMIN_EMAIL`, `MOODLE_ADMIN_PASSWORD[_FILE]` | Required for installation | +| `MOODLE_ADMIN_USER` | `admin` | +| `MOODLE_SITE_FULLNAME`, `MOODLE_SITE_SHORTNAME` | `Mooship Moodle`, `Mooship` | + +Secret files must have absolute paths, be readable, and contain at most 64 KiB. Trailing line endings are removed; other characters are preserved. Passwords are not embedded in layers or generated PHP. The admin password is passed to Moodle in PHP memory, without adding it to the process command line. Environment passwords remain visible to container administrators; prefer files. + +| Command | Behavior | +| --- | --- | +| `fpm` (default) | Validate settings and installed DB version, serve FastCGI on 9000 | +| `install` | Install into an empty database with an exclusive lock | +| `cron` | Run cron, wait 60 seconds after completion, repeat; exit on error | +| `cron-once` | Run cron once; suitable for a Kubernetes CronJob | +| `check` | Readiness: writable data/tmp, DB connection, matching installed version | +| `php ...` | Explicit CLI operation, including controlled schema upgrades | +| `--help` | Display commands without credentials | + +Application code is immutable in `/opt/moodle`; `/var/moodledata` and `/tmp` are writable. Named Docker volumes inherit UID 1000; provision bind mounts/PVCs accordingly. Both images run as 1000:1000 with read-only roots and dropped capabilities, without privileged mode, source mounts, or a Docker socket. Logs go to stdout/stderr. nginx reads Moodle data only for internal X-Accel-Redirect responses; direct `/dataroot/` requests are denied. The images reuse the project's PHP/OPcache limits. + +FPM liveness (`mooship-healthcheck`) uses a real FastCGI ping, independent of the database. nginx liveness is `/healthz`; `/readyz` checks PHP/DB readiness and returns 503 during DB failure or a version mismatch. Compose disables the FPM health check for install/cron roles. PHP and nginx use SIGQUIT for graceful Docker shutdown; Kubernetes pre-stop hooks request graceful shutdown within a 60-second termination budget. + +File sessions persist in the data volume by default. Optional Redis sessions do not configure MUC. Multiple app replicas require shared data, compatible caches/sessions and capacity planning. Bake plugins into derived PHP/nginx images consistently; browser code installation is disabled. Uploaded files, caches and language packs remain runtime data. + +## Backups and upgrades + +Back up the database and `moodledata` together while writes/cron are stopped or under a consistent maintenance procedure. Retain exact image references, configuration, and separately protected secrets. The host installer's `backup.sh`/`restore.sh` expects a different layout and does not manage this stack. + +Test upgrades against a restored copy. Enable maintenance with the old image, stop web/cron/app, take consistent backups, select matching new image tags, and run `php admin/cli/upgrade.php --non-interactive` as a one-off app container. Start services and disable maintenance after checks. Ordinary startup rejects mismatching schema versions. Rollback after schema changes requires restoring both DB and data with the old image; changing tags alone is insufficient. Do not downgrade database engines by switching overlays. + +## Kubernetes + +[config.yml](../deploy/k8s/config.yml) provides ConfigMap and RWX PVC. [runtime.yml](../deploy/k8s/runtime.yml) provides Service, a PHP/nginx Deployment and a separate CronJob. [install.yml](../deploy/k8s/install.yml) provides an explicit initialization Job. They use stable `v1`, `apps/v1`, and `batch/v1` APIs, non-root execution, read-only roots, dropped capabilities, startup/liveness/readiness probes, and temporary volumes. Resource values are initial estimates. The DB is external; the default Deployment has one replica and uses Recreate for controlled upgrades. + +Verify the cluster/namespace, set available matching image tags (prefer release digests), edit the public origin and DB settings, and provision an RWX storage class supporting UID/GID 1000. The app and scheduler share the data PVC. Create Secrets in that namespace: + +```bash +kubectl create secret generic mooship-db --from-file=password=.secrets/db_password +kubectl create secret generic mooship-admin --from-file=password=.secrets/admin_password +``` + +Initialize before starting the runtime: + +```bash +kubectl apply -f deploy/k8s/config.yml +kubectl apply -f deploy/k8s/install.yml +kubectl wait --for=condition=complete --timeout=15m job/mooship-install +kubectl logs job/mooship-install +kubectl apply -f deploy/k8s/runtime.yml +``` + +Run the final command only after installation succeeds. Remove the admin Secret after success if no longer needed. A partial install needs investigation and an explicitly reset empty DB before retry; the Job has no automatic retry. Ingress/TLS, DB transport, network policy and backups depend on the target cluster. These manifests do not establish a tested production cluster deployment. + +## Keeping images in sync + +Assess image impact whenever behavior, dependencies, or configuration changes. The host installer and standalone runtime are separate implementations; inspect both when the same behavior applies to them, respecting their supported Moodle/PHP versions. + +| Changed inputs | Image impact | +| --- | --- | +| Root `Dockerfile`, its downloaded Moodle source, base images or extension pins | Trace affected stages; shared Moodle code changes require matching `app` and `web` builds. Update Moodle version, commit and archive hash together. | +| `docker/*.php`, app entrypoint, cron, health check, FPM settings, `moodle/.config/php/mooship.ini` | These are copied into `app`; rebuild it when these inputs change. | +| `docker/nginx.conf.template`, `docker/web-entrypoint.sh`, `docker/health.txt` | These are copied into `web`; rebuild it when these inputs change. | +| `.dockerignore` or newly added runtime files | Verify the required inputs enter the build context and are copied into the correct target. | +| Compose, Kubernetes manifests, environment examples | Usually external to the images; validate configuration and runtime compatibility. Rebuild only if image contents or their startup contract also need changes. | +| Host installer, lifecycle utilities, `lib/`, other host templates, documentation | Not copied into the standalone images. Check whether the behavior also needs a corresponding runtime change; otherwise no image rebuild is needed. | + +Update affected image sources, build definitions, configuration, manifests, tests and documentation in the same task. Editing a file already copied into an image can be sufficient at source level; do not change the Dockerfile merely to force a rebuild. Plugins and other shared Moodle code must stay consistent between PHP and nginx. + +Check Docker availability, rebuild the affected targets using the commands above, and run the relevant [image tests](#tests-and-later-publication) on the resulting pair. Add or adjust behavioral tests when needed. Use an unchanged companion only when its contents and interface remain compatible. Validate Compose/Kubernetes configuration when touched; host installer E2E does not replace standalone image tests. Report unavailable checks explicitly. Documentation-only changes need content/link checks, without rebuilding images. + +In the task report, state whether images are affected and why, which targets were rebuilt, and which checks ran. Distinguish a source update, a local build, runtime verification, registry publication and deployment. Existing published images do not acquire repository changes automatically; publication and rollout follow the user's authorized scope, with verified release tags/digests recorded separately. + +## Tests and later publication + +Build both tags, pull the test DB images, and run the isolated suite: + +```bash +docker pull postgres:17-alpine +docker pull mysql:8.4 +MOOSHIP_IMAGE_TESTS=1 PYTHONPATH=tests python3 -B -m unittest test_runtime_image -v +``` + +Tests use disposable uniquely labeled containers, a private network and volumes, publish no host ports, and clean up only their own resources. They cover secrets, non-root/immutable execution, invalid settings, real PostgreSQL/MySQL installs, repeated install/version refusal, HTTP login/protected paths, cron, DB failure/recovery, and graceful shutdown. The [image workflow](../.github/workflows/image.yml) builds/tests on PRs and `main`, without registry credentials or publication. + +For Docker Hub, choose a confirmed Mooship release identifier and build/test that exact pair. Keep the Moodle version in both tags. `MOOSHIP_VERSION` sets OCI version metadata; `VCS_REF` identifies the source commit. The Dockerfile pins PHP/nginx digests and the Moodle commit/archive SHA-256. A Moodle update must deliberately change and verify its version, commit and hash together, for both targets. + +```bash +read -r -p 'Confirmed Mooship release tag: ' RELEASE +REVISION=$(git rev-parse HEAD) +APP="shellharbor/mooship:${RELEASE}-moodle4.5.15-php8.3" +WEB="shellharbor/mooship:${RELEASE}-moodle4.5.15-nginx" +docker build --target app --build-arg "MOOSHIP_VERSION=$RELEASE" --build-arg "VCS_REF=$REVISION" -t "$APP" . +docker build --target web --build-arg "MOOSHIP_VERSION=$RELEASE" --build-arg "VCS_REF=$REVISION" -t "$WEB" . +MOOSHIP_APP_IMAGE="$APP" MOOSHIP_WEB_IMAGE="$WEB" MOOSHIP_IMAGE_TESTS=1 PYTHONPATH=tests \ + python3 -B -m unittest test_runtime_image -v +docker login --username shellharbor +docker push "$APP" +docker push "$WEB" +``` + +First create `shellharbor/mooship` on Docker Hub under an account you control. Use an access token at the login prompt. Publish both tags and record digests; set `MOOSHIP_APP_IMAGE`/`MOOSHIP_WEB_IMAGE` in Compose accordingly. Multi-platform publication needs explicit build/runtime tests per architecture; these commands publish the local platform only. Do not use the nginx tag as the PHP runtime. + +Mooship wrapper code is MIT licensed; bundled Moodle/dependencies retain their upstream licenses, including Moodle's GPL-3.0-or-later terms. Moodle's `COPYING.txt` and dependency notices remain under `/opt/moodle`; the wrapper license is `/opt/mooship/LICENSE`. Distribute corresponding sources and notices when publishing. See [requirements](https://moodledev.io/general/releases/4.5) and the [pinned Moodle source](https://github.com/moodle/moodle/tree/0a694ba9b8b4183bf7a45094cfa7211030db3192). diff --git a/docs/PREFLIGHT.md b/docs/PREFLIGHT.md new file mode 100644 index 0000000..189cd94 --- /dev/null +++ b/docs/PREFLIGHT.md @@ -0,0 +1,47 @@ +# Deployment preflight and readiness + +These checks apply to new host-installer deployments. Existing sites are not rewritten. + +## Ports + +Every explicit HTTP, HTTPS, PHP, Redis, PostgreSQL and MySQL port must be a decimal integer from 1 to 65535. Leading zeroes are normalized. Ports 80/443 are reserved for the shared proxy. Selected ports must be distinct; automatic allocation also excludes explicit assignments. + +Published service ports must be available on loopback. Checks use a real TCP bind and Docker's publication inventory, including NAT mappings without a listening host process. Occupied native DB ports are expected and exempt from free-port checks. The race between checking and binding a port remains; startup errors are fatal. + +## Native databases + +Use `--db-native --db-host HOST`; the default is `host.docker.internal`. On Linux Docker Engine 20.10+, PHP and cron map that name to `host-gateway`. Custom addresses are used without that mapping. The address is persisted as `DB_HOST` and written into Moodle configuration. + +Mooship creates the new database/user through the host's local administrative connection. `--db-host` selects the application's route to that same database; it is not remote provisioning. Do not select a different server from the one managed by the local database tools. + +The DB must listen on an address reachable from the project's Docker network. Configure PostgreSQL `listen_addresses` and a narrowly scoped `pg_hba.conf` rule, or MySQL `bind-address`/grants, plus the firewall. Do not expose DB ports publicly to solve routing. Mooship does not rewrite shared database configuration or firewall rules. + +Before native CREATE, PHP tests TCP reachability from its actual container. After CREATE, PHP authenticates with application credentials and executes `SELECT 1`. Containerized DBs use the same authentication probe. Errors do not print passwords or driver exceptions. Failure can leave a newly created owned DB/project for explicit recovery; automatic destructive cleanup is not performed. + +New native archives retain their endpoint. Older archives without `DB_HOST` retain `172.17.0.1`, matching their archived `config.php`; migration is a separate operation. + +## Readiness and installation + +PHP-FPM uses a FastCGI ping; nginx serves `/mooship-healthz` without Moodle/Basic Auth; Redis uses authenticated `PING`. The installer waits for health instead of a fixed ten-second sleep. DB readiness is bounded separately. Each service gets 90 checks spaced two seconds apart, plus probe execution time. `MOOSHIP_READY_ATTEMPTS` adjusts the count; Docker health checks have five-second execution timeouts. + +Process health is independent of DB availability. PHP's DB probe establishes authentication before initialization. Installation checks distinguish an empty DB (exit 3), a present `mdl_config.version` record (0), a populated DB without that record (4), and connection/probe failure (1). This classification is not a full schema audit. Partial/unknown state never silently skips or retries installation. The fixed `mdl_` prefix matches the host installer. + +When Moodle declares `league/oauth2-server` in its production requirements (including 5.3), run `lib/install-dependencies.sh` in the PHP container before schema initialization. A fresh GitHub source archive lacks that package: the core site can install while API requests fail with a missing ScopeTrait. Install the production graph from the existing composer.lock and verify that the trait autoloads. Older manifests without this requirement skip the step. Invalid metadata, a missing required lockfile, failed download, checksum mismatch or failed installation stops deployment. + +The helper downloads Composer 2.10.3 with the fixed SHA-256 published on the [official download page](https://getcomposer.org/download/), verifies it before execution and removes the temporary PHAR on exit. It runs as the PHP container user with temporary Composer home/cache directories, disables dev dependencies/scripts/plugins, and preserves platform checks. PHP needs outbound HTTPS to Composer, Packagist and package download hosts during this step. Updating the tool pin requires reviewing both the version and checksum. Full v1 archives retain installed vendor files along with the project. + +Cron starts after successful installation/mail configuration, or after restoring data. Older archives without health metadata use live FPM socket and nginx configuration checks. New templates include health checks; Compose parsing alone does not prove readiness. + +After Moodle code/config generation, validate and reload nginx so a newly selected `public/` document root is active before publishing the site. Update the mounted configuration in place: replacing its inode would leave nginx's single-file bind mount reading the old contents even after reload. Full E2E also waits for health after backup resumes services, before running doctor. Its one-off cron check uses `--keep-alive=0` to execute tasks without the administrator-configured idle polling period. + +## Tests + +`tests/test_preflight.py` covers invalid/occupied/duplicate ports, native endpoint selection, secret transport, bounded health transitions and installation refusal. Opt-in `tests/test_native_preflight.py` publishes a disposable synthetic PostgreSQL/MySQL DB on a random host port and checks real PHP host-gateway access, wrong credentials, empty/partial/installed states and an unavailable endpoint. Temporary host publications are needed for this test; these are not production databases. + +The full E2E matrix installs Moodle 4.5/5.3 with PostgreSQL/MySQL and then checks native routing using its built PHP image. It also checks HTTP, Redis sessions, SMTP, cron, backup retention, systemd schedules and doctor. TLS issuance is outside the synthetic HTTP runner setup. + +The synthetic E2E domain is resolved both on the runner and inside PHP. A dedicated internal test network gives nginxproxy that DNS alias, allowing Moodle 5.3's router status check to call its own wwwroot. Production domains must resolve from their application containers. E2E checks successful router/API responses and a real 404 before the final doctor gate. + +The PHP nginx location also falls back to `r.php` for removed scripts, preserving the original request/query and saved FastCGI PATH_INFO for existing scripts. This covers Moodle's shim redirects; a missing router returns 404. See the [official router-aware nginx example](https://moodledev.io/docs/5.3/guides/restructure#developer-configuration-examples). E2E asserts both physical and virtual shim responses, and component tests exercise HTTP/HTTPS, Basic Auth, public/legacy roots and slash arguments. + +The opt-in HTTP suite (`MOOSHIP_HTTP_INTEGRATION=1`) includes a real nginx reload regression: an application route under `public/` stays unavailable until the changed document root is loaded, then returns HTTP 200. Use the pre-pulled fixture images from the component integration workflow. diff --git a/docs/PYTHON.md b/docs/PYTHON.md new file mode 100644 index 0000000..6ff7feb --- /dev/null +++ b/docs/PYTHON.md @@ -0,0 +1,64 @@ +# Mooship Python CLI + +Install Mooship through pip or pipx, then use one command for its existing Moodle deployment, backup, restore and diagnostics scripts. The Python package needs Python 3.8+ and has no runtime dependencies. It follows LaraShip's verified-release installer design. Installing the package does not deploy Moodle, install Docker or modify server resources. + +## Install the CLI + +Version 1.2.0 introduces Python packaging. Until it is published, install a reviewed checkout with `pipx install .` or `python -m pip install .` in a virtual environment. After publication: + +```bash +pipx install mooship==1.2.0 +mooship --version +mooship --help +``` + +Help and installation previews work without root. Server operations target Ubuntu 20.04+ with Bash, Python 3 and Docker/Compose; use SSH from Windows/macOS. Do not use `sudo pip` or bypass OS-managed Python protections. + +## Install the native toolkit + +The CLI downloads an exact official GitHub release archive and its SHA-256 file over HTTPS, verifies the checksum, archive paths/types/limits and embedded VERSION, then installs into a new directory. Existing files/directories/symlinks are refused. A failed copy remains marked for inspection and cannot silently be reused. Checksums establish consistency with the release assets rather than an independent signature. + +```bash +mooship install --version 1.2.0 --directory /opt/mooship --dry-run +MOOSHIP_CLI="$(command -v mooship)" +sudo "$MOOSHIP_CLI" install --version 1.2.0 +mooship toolkit-version +sudo "$MOOSHIP_CLI" deploy -- \ + --slug lms --domain lms.example.com --db-type postgres --ssl-email admin@example.com +sudo "$MOOSHIP_CLI" doctor -- --slug lms +sudo "$MOOSHIP_CLI" backup -- --slug lms +``` + +Default directory: `/opt/mooship`; its parent must exist. Use `--directory /absolute/path` before `--` for another installation or a complete reviewed checkout containing VERSION. The CLI never escalates privileges implicitly; underlying scripts retain their root checks, confirmation prompts and operational contracts. + +## Commands and forwarding + +```text +mooship --version +mooship install [--version X.Y.Z] [--directory PATH] [--dry-run] +mooship toolkit-version [--directory PATH] +mooship deploy|activate|deactivate|remove|list|backup|restore|schedule-backup|doctor [--directory PATH] -- [script arguments] +``` + +Arguments following `--` are forwarded unchanged as separate values. The adapter does not print them. Terminal input, signals and exit codes pass to the existing Bash process, including doctor's status 0/1/2. Argument parsing errors exit 2, installation/preflight errors exit 1, and interrupted installation exits 130. Check the [README](../README.md) and each script's supported flags before use. + +The Docker Hub images contain the Moodle application and nginx, not these host administration scripts. There is no privileged Docker-controller backend in this CLI. Run the standalone stack using the separate [Docker guide](DOCKER.md). + +Updating the Python package updates the adapter only. Install a new native toolkit into a new directory and select it explicitly; existing sites, data, backups and templates are not migrated. Uninstalling the package does not remove sites or toolkit files. + +## Build and verify + +```bash +python -m pip install -r requirements-python-build.txt +python -m unittest discover -s tests -p test_python_cli.py -v +python -m build +python -m twine check --strict dist/* +``` + +The package workflow also rebuilds the wheel from the source distribution, installs it in fresh Python environments, and runs installed CLI checks. Native toolkit archives are separate GitHub release assets generated from tracked runtime files by `.github/scripts/build_release.py`; no local secrets or AI-KIT files are included. + +## PyPI publication + +The Release workflow publishes checked distributions through [PyPI Trusted Publishing](https://docs.pypi.org/trusted-publishers/creating-a-project-through-oidc/), without a long-lived PyPI token. Configure a pending publisher in [your PyPI account](https://pypi.org/manage/account/publishing/): project **mooship**, owner **shellharbor**, repository **mooship**, workflow **release.yml**, environment **pypi**. A pending publisher does not reserve the package name. Create the matching GitHub environment and set repository variable `PYPI_PUBLISH=true` when configured. + +See [release publication](RELEASING.md) for checked release tags, native assets, Docker Hub configuration, retries and evidence. Package version and toolkit release come from the same VERSION file. diff --git a/docs/RELEASING.md b/docs/RELEASING.md new file mode 100644 index 0000000..5d3e8ce --- /dev/null +++ b/docs/RELEASING.md @@ -0,0 +1,36 @@ +# Release publication + +Mooship 1.2.0 introduces a dependency-free Python CLI and the standalone Moodle/PHP and nginx images. The host toolkit, Python distribution and image metadata use root VERSION. A new version never changes old tags, existing installations, databases or volumes. + +## Prepare and verify + +1. Update VERSION and release documentation, keeping Docker image tags and the pinned Moodle/PHP pair consistent. Build/test the package and the affected images according to [Python](PYTHON.md) and [Docker](DOCKER.md). +2. Push a review branch and require successful **CI**, **Configuration**, **Component integration**, **Security**, **CodeQL**, **Documentation**, **Moodle images**, and **Python package** workflows on the exact commit being released. Dependency Review additionally needs GitHub's repository Dependency Graph; it is separate from the release gate. +3. Create a matching `vX.Y.Z` tag on that verified commit, push it, and publish a GitHub release with that tag. The **Release** workflow checks tag/VERSION and exact-commit workflow evidence before creating release assets or publishing packages. + +The native archive contains tracked runtime files only: scripts, `lib/`, Moodle/proxy templates, VERSION, README and LICENSE. `.github/scripts/build_release.py` normalizes archive metadata and shell line endings, and generates a separate `.sha256` file. Local credentials, AI-KIT, working environments and test reports are excluded by the explicit runtime inventory. The Python installer rejects unsafe/mismatched archives, existing destinations and partial installations. + +## Docker Hub + +The Docker Hub repository is **shellharbor/mooship**. Configure repository variable `DOCKERHUB_USERNAME`, a read/write `DOCKERHUB_TOKEN` Secret, and variable `DOCKERHUB_PUBLISH=true` to enable the optional release job. The existing Moodle images workflow remains build/test-only. Local Docker Desktop login does not automatically configure GitHub Actions secrets. + +The release job checks out the verified commit, builds Linux amd64 `app` and `web`, runs real Moodle installs with PostgreSQL and MySQL, then publishes: + +```text +shellharbor/mooship:1.2.0-moodle4.5.15-php8.3 +shellharbor/mooship:1.2.0-moodle4.5.15-nginx +``` + +PHP and nginx must use the matching Moodle source. No `latest` alias or untested architecture is published. Record the resulting digests and prefer digest references in production. The [Docker guide](DOCKER.md#tests-and-later-publication) also supports a checked manual publication using an existing local Docker login; always set MOOSHIP_VERSION and VCS_REF to the release version and exact source commit. + +## PyPI + +Configure the pending publisher described in [the Python guide](PYTHON.md#pypi-publication), create GitHub environment **pypi**, and set variable `PYPI_PUBLISH=true`. The isolated publish job receives only built distributions and short-lived OIDC credentials. No long-lived PyPI token or build-source checkout is needed in that job. + +PyPI project name: **mooship**. Distributions contain the Python adapter; the separately checksummed GitHub archive supplies the native toolkit. The Docker images contain Moodle itself. These are separate distribution formats with different operational roles. + +## Retry and verification + +Use **Release → Run workflow**, select the reviewed workflow branch and enter the existing release tag. The tag must still match VERSION and have successful checks. Native asset retries compare content and refuse to replace conflicting files; PyPI skips existing distribution files. Docker publication refuses existing release image tags; disable `DOCKERHUB_PUBLISH` when retrying other formats after a successful image upload. Do not move an existing tag to publish changed code. Fixes require a new version. + +After publication, verify the Docker Hub tags/digests, pull and smoke-test the images, and install the package from PyPI in a fresh environment. Also run `mooship install` against the real GitHub assets in an unused temporary destination, then check the toolkit VERSION. A workflow definition or local build alone is not publication evidence. diff --git a/docs/adr/0004-standalone-moodle-images.md b/docs/adr/0004-standalone-moodle-images.md new file mode 100644 index 0000000..e6c2fe7 --- /dev/null +++ b/docs/adr/0004-standalone-moodle-images.md @@ -0,0 +1,27 @@ +# ADR-0004: Immutable Moodle runtime and nginx companion + +- Date: 2026-10-15 +- Status: Accepted + +## Context + +The user requested a Moodle/PHP image for later Docker Hub publication, with separate nginx and DB services. The host installer manages Docker, host directories, certificates and shared proxy state; packaging it as a privileged Docker controller would not provide the chosen application runtime. + +## Decision + +Use one multi-stage root Dockerfile: default `app` target and `web` companion. Pin Moodle 4.5.15 by commit/archive SHA-256 and compatible PHP 8.3/nginx bases by digest. Both images contain matching immutable code, avoiding a shared mutable source volume or startup downloads. Use distinct tags in the future `shellharbor/mooship` repository. Publication remains separate. + +Run as UID/GID 1000 with read-only roots, data/tmp mounts, external config, mounted secrets and graceful shutdown. Keep install/upgrade explicit; normal startup refuses an absent/incompatible schema. Installation requires an advisory DB lock and empty DB; the admin password is passed in PHP memory. Cron is an independent role. Readiness checks DB/schema availability; process liveness is independent of the DB. + +Compose overlays provide PostgreSQL 17 or MySQL 8.4. Kubernetes combines PHP/nginx in one Deployment with RWX storage, an external DB, and separate install Job/CronJob. Start with one replica. CI builds and installs real Moodle on both DB engines without registry publication. Host installer templates and lifecycle commands remain independent. + +## Alternatives and consequences + +- A containerized host installer requires host filesystem/Docker access and conflicts with the requested runtime scope. +- An all-in-one PHP/nginx/DB image couples persistent state and unrelated lifecycles. +- Startup-populated shared code can leave PHP/nginx serving inconsistent versions during updates. +- Duplicated source increases registry storage, but keeps static assets consistent. Deploy matching tags/digests. +- Operators must back up DB/data and perform explicit tested upgrades; host installer backup tools expect a different layout. +- Bundled Moodle/dependencies retain their licenses; wrapper MIT terms do not replace Moodle's GPL terms. + +See the [image operations guide](../DOCKER.md) for configuration, installation, health, storage, upgrades, tests and later publication. diff --git a/docs/adr/0005-deployment-preflight.md b/docs/adr/0005-deployment-preflight.md new file mode 100644 index 0000000..ad5224e --- /dev/null +++ b/docs/adr/0005-deployment-preflight.md @@ -0,0 +1,32 @@ +# ADR-0005: Validate deployment boundaries and probe readiness from the application + +- Date: 2026-10-15 +- Status: Accepted + +## Context + +Explicit port values reached Compose without complete validation. Native DB connections assumed a fixed default-bridge IP. A running container and a fixed startup delay did not establish that PHP, nginx, Redis or the database could serve the application. + +## Decision + +Validate port syntax before side effects; normalize decimal values, reserve shared proxy ports, prevent duplicates and exclude selected ports from automatic allocation. Check both loopback TCP bind and Docker NAT publications. Native DB listeners are expected to occupy their port. This is a preflight check, not an atomic reservation. + +Use a configurable native application endpoint, defaulting to host.docker.internal with PHP/cron host-gateway mappings. Keep administrative CREATE operations local and preserve DB ownership safeguards. Do not rewrite shared DB listeners, authentication rules or firewall settings. Probe TCP reachability from the actual PHP container before native CREATE, and authenticate/execute SQL afterward. Passwords travel through Docker client environment variables, not command arguments or error output. + +Add process health for FPM/nginx/Redis, bounded waits, and explicit DB authentication. Start cron only after installation or restoration. Use structured DB installation states instead of parsing foreign help text. Keep compatibility paths for archived configurations without health metadata or a stored native endpoint. + +Validate and reload nginx after Moodle configuration selects the document root. Preserve the configuration inode because nginx uses a single-file bind mount; replacing it leaves the container reading the old file. Health of the nginx process alone cannot prove that its loaded configuration matches a changed file. Wait for resumed service health before E2E runs doctor after backups. + +Install locked production dependencies when Moodle declares league/oauth2-server. Fresh 5.3 source archives can initialize the DB but fail HTTP API middleware with a missing ScopeTrait. Use a temporary Composer PHAR pinned by version and SHA-256 inside the existing PHP container, running as its normal user with scripts/plugins/dev packages disabled. Preserve platform checks and verify the trait before schema initialization. Older manifests retain their bundled-dependency path; existing archive vendor files are preserved. + +## Alternatives and consequences + +- A detected bridge IP alone still couples generated settings to host topology; host-gateway plus an explicit override permits controlled configuration changes. +- Automatically changing PostgreSQL/MySQL/firewall settings can affect unrelated databases and is outside deployment scope. +- A TCP connection alone is insufficient: Docker proxies may accept it after their backend stops. SQL probes are required. +- Liveness should not restart PHP/nginx for an unrelated DB outage; DB authentication is a separate deployment gate. +- Partial deployment resources remain for explicit recovery. Automatic destructive cleanup is not introduced. +- Old archives retain their original endpoint; migrating it requires an explicit change, not a silent restore rewrite. +- An unverified latest Composer download or dependency update would make initialization unpredictable. The pinned tool and existing lockfile bound this step; outbound package access is required only during installation. + +See [preflight operations](../PREFLIGHT.md) and the regression/host-gateway tests for the contract and verification limits. diff --git a/docs/adr/0006-python-cli-and-release-distribution.md b/docs/adr/0006-python-cli-and-release-distribution.md new file mode 100644 index 0000000..ed40fe7 --- /dev/null +++ b/docs/adr/0006-python-cli-and-release-distribution.md @@ -0,0 +1,24 @@ +# ADR-0006: Python CLI and release distribution + +- Date: 2026-10-28 +- Status: Accepted + +## Context + +The user requested Docker Hub publication and a Python package for PyPI, using neighboring LaraShip as the reference. Mooship already has host administration scripts and separate immutable Moodle/PHP and nginx images. Reimplementing host deployment in Python or turning the application image into a privileged Docker controller would introduce unrelated operational changes. + +## Decision + +Publish a dependency-free Python 3.8+ adapter with the `mooship` console command. Follow LaraShip's exact-release installer design: HTTPS downloads of the official GitHub archive/checksum, bounded safe extraction, matching VERSION and a new installation directory. Forward native commands through an argument-list exec, preserving terminal input, signals, existing confirmations and backend exit codes. + +Use a root VERSION shared by Python metadata, toolkit release assets and Docker OCI metadata. Generate reproducible native assets from an explicit tracked runtime inventory. Validate wheel/sdist installation and Python compatibility in CI. Gate release publication on matching tag/version and successful workflows for the exact source commit; publish checked distributions with PyPI OIDC from a separate job. Docker publication builds/tests both application targets before pushing versioned Linux amd64 tags. Account configuration and actual uploads remain separate, verified steps. + +## Alternatives and consequences + +- A full Python rewrite would duplicate mature deployment/lifecycle behavior and require its own full regression effort. +- Bundling the toolkit inside the wheel would simplify offline installation but couple server template files to site-packages. A verified native release keeps an explicit inspectable installation and existing script-relative paths. +- A Docker CLI backend like LaraShip's does not fit the chosen Mooship application images. This adapter manages native host scripts only; the standalone runtime uses Compose or Kubernetes. +- Checksums establish consistency with the official release assets rather than an independent signature. PyPI package installation itself performs no server deployment or release download. +- Existing release tags and published versions must not be rewritten to add packaging. Introduce a new release, initially 1.2.0, after checks pass. + +See [the Python guide](../PYTHON.md) and [release publication](../RELEASING.md). diff --git a/lib/check-database.php b/lib/check-database.php new file mode 100644 index 0000000..9eea36a --- /dev/null +++ b/lib/check-database.php @@ -0,0 +1,44 @@ + PDO::ERRMODE_EXCEPTION, PDO::ATTR_TIMEOUT => 5, + ]); + $db->query('SELECT 1'); + if ($mode === 'installed') { + $tables = $engine === 'pgsql' + ? $db->query('SELECT tablename FROM pg_tables WHERE schemaname = current_schema()')->fetchAll(PDO::FETCH_COLUMN) + : $db->query('SHOW TABLES')->fetchAll(PDO::FETCH_COLUMN); + if (!$tables) { + exit(3); // An empty, reachable database may be initialized. + } + if (!in_array('mdl_config', $tables, true) || + !$db->query("SELECT value FROM mdl_config WHERE name = 'version'")->fetchColumn()) { + fwrite(STDERR, "[ERROR] The database is partially installed; investigate before retrying.\n"); + exit(4); + } + } elseif ($mode !== 'connect') { + throw new RuntimeException('Unknown database check.'); + } +} catch (Throwable $error) { + // Driver exceptions can contain connection details. Report a fixed actionable message. + fwrite(STDERR, "[ERROR] Database probe failed from PHP. Check the host, port, credentials, " + . "listen_addresses/pg_hba.conf (PostgreSQL), bind-address/grants (MySQL), and firewall.\n"); + exit(1); +} diff --git a/lib/common.sh b/lib/common.sh index 62228fe..ae9a1f6 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -99,7 +99,7 @@ mooship_validate_database_identifiers() { mooship_validate_port() { [[ "$1" =~ ^[0-9]{1,5}$ ]] && ((10#$1 > 0 && 10#$1 <= 65535)) || - mooship_die "Invalid database port" + mooship_die "Invalid port: expected an integer from 1 to 65535" } mooship_validate_database_settings() { diff --git a/lib/install-dependencies.sh b/lib/install-dependencies.sh new file mode 100644 index 0000000..c3547d7 --- /dev/null +++ b/lib/install-dependencies.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# Run as the existing PHP container user; install declared production packages only. +set -eu +moodle_root=${1:-/var/www/html} + +status=0 +php -r ' + $path = $argv[1] . "/composer.json"; + if (!is_file($path)) { exit(3); } + $manifest = json_decode(file_get_contents($path), true, 512, JSON_THROW_ON_ERROR); + if (!is_array($manifest) || !is_array($manifest["require"] ?? [])) { exit(1); } + exit(isset($manifest["require"]["league/oauth2-server"]) ? 0 : 3); +' "$moodle_root" || status=$? +[ "$status" != 3 ] || exit 0 +[ "$status" = 0 ] || { echo '[ERROR] Could not read Moodle dependency requirements' >&2; exit 1; } +[ -f "$moodle_root/composer.lock" ] || { echo '[ERROR] Moodle composer.lock is missing' >&2; exit 1; } + +composer_file=$(mktemp /tmp/mooship-composer.XXXXXX) +trap 'rm -f "$composer_file"' EXIT +trap 'exit 1' HUP INT TERM +curl --fail --silent --show-error --location --proto '=https' --proto-redir '=https' \ + --connect-timeout 10 --max-time 120 \ + https://getcomposer.org/download/2.10.3/composer.phar -o "$composer_file" +printf '%s %s\n' '7a2d379d5b8ffdaa028580ef26494c36d2feef4b178d3dd1473a4dbc5e17c8d6' "$composer_file" | sha256sum -c - +php "$composer_file" install --working-dir="$moodle_root" --no-dev --no-scripts --no-plugins \ + --prefer-dist --no-interaction --no-progress --optimize-autoloader +php -r ' + require $argv[1] . "/vendor/autoload.php"; + exit(trait_exists(\League\OAuth2\Server\Entities\Traits\ScopeTrait::class) ? 0 : 1); +' "$moodle_root" diff --git a/lib/preflight.sh b/lib/preflight.sh new file mode 100644 index 0000000..d8cffec --- /dev/null +++ b/lib/preflight.sh @@ -0,0 +1,90 @@ +#!/bin/bash +# Deployment preflight, bounded readiness checks, and ordered service startup. + +mooship_port_available() { + local port="$1" + [[ " ${MOOSHIP_ASSIGNED_PORTS:-} " != *" $port "* ]] || return 1 + ! grep -Eq ":${port}->" <<< "${MOOSHIP_DOCKER_PORTS:-}" || return 1 + python3 - "$port" <<'PY' +import socket +import sys + +try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as connection: + connection.bind(("127.0.0.1", int(sys.argv[1]))) +except OSError: + sys.exit(1) +PY +} + +mooship_validate_db_host() { + [[ "$1" =~ ^[A-Za-z0-9_:][A-Za-z0-9_.:-]*$ ]] || + mooship_die "Invalid native database host; use a hostname or IP address" +} + +mooship_wait_service() { + local service="$1" state health attempt status + local attempts="${MOOSHIP_READY_ATTEMPTS:-90}" + [[ "$attempts" =~ ^[0-9]{1,4}$ ]] && ((10#$attempts > 0)) || + mooship_die "MOOSHIP_READY_ATTEMPTS must be a positive integer" + for ((attempt = 0; attempt < 10#$attempts; attempt++)); do + status=$(docker inspect --format '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "${SLUG}_${service}") || return 1 + read -r state health <<< "$status" + if [[ "$state" == exited || "$state" == dead ]]; then + printf '[ERROR] Service %s exited before becoming ready\n' "$service" >&2 + return 1 + fi + if [[ "$state" == running && "$health" == healthy ]]; then return 0; fi + if [[ "$state" == running && "$health" == none ]]; then + # Older archives have no health metadata; check the actual process/configuration. + case "$service" in + php) docker compose exec -T php php -r '$s = @fsockopen("127.0.0.1", 9000, $n, $e, 2); exit($s ? 0 : 1);' && return 0 ;; + nginx) docker compose exec -T nginx nginx -t >/dev/null 2>&1 && return 0 ;; + redis) + REDISCLI_AUTH="$(mooship_read_env "$PROJECT_DIR/.env" REDIS_PASSWORD)" \ + docker compose exec -T -e REDISCLI_AUTH redis redis-cli ping | grep -qx PONG && return 0 ;; + cron|certbot_renew) return 0 ;; + esac + fi + sleep 2 + done + printf '[ERROR] Service %s did not become ready within %s seconds\n' "$service" "$((10#$attempts * 2))" >&2 + return 1 +} + +mooship_check_database() { + local mode="${1:-connect}" host port name user password + if [[ "$DB_TYPE" == postgres ]]; then + port=5432; name="$DB_POSTGRES_NAME"; user="$DB_POSTGRES_USER"; password="$DB_POSTGRES_PASSWORD" + [[ "$DB_NATIVE" != true ]] || port="$PORT_POSTGRES" + else + port=3306; name="$DB_MYSQL_NAME"; user="$DB_MYSQL_USER"; password="$DB_MYSQL_PASSWORD" + [[ "$DB_NATIVE" != true ]] || port="$PORT_MYSQL" + fi + host="${SLUG}_db" + [[ "$DB_NATIVE" != true ]] || host="${NATIVE_DB_HOST:-host.docker.internal}" + # Export secrets only to the Docker client environment, never as command arguments. + MOOSHIP_DB_TYPE="$DB_TYPE" MOOSHIP_DB_HOST="$host" MOOSHIP_DB_PORT="$port" \ + MOOSHIP_DB_NAME="$name" MOOSHIP_DB_USER="$user" MOOSHIP_DB_PASSWORD="$password" \ + MOOSHIP_DB_CHECK="$mode" docker compose exec -T \ + -e MOOSHIP_DB_TYPE -e MOOSHIP_DB_HOST -e MOOSHIP_DB_PORT \ + -e MOOSHIP_DB_NAME -e MOOSHIP_DB_USER -e MOOSHIP_DB_PASSWORD -e MOOSHIP_DB_CHECK \ + php php < "$SCRIPT_DIR/lib/check-database.php" +} + +start_moodle_cron() { + (cd "$PROJECT_DIR" && docker compose up -d cron && + mooship_wait_service cron) || error "Moodle cron did not start" +} + +reload_project_nginx() { + (cd "$PROJECT_DIR" && docker compose exec -T nginx nginx -t && + docker compose exec -T nginx nginx -s reload) || error "Could not apply the project's nginx configuration" +} + +install_moodle_dependencies() { + (cd "$PROJECT_DIR" && docker compose exec -T \ + -e COMPOSER_HOME=/tmp/mooship-composer-home -e COMPOSER_CACHE_DIR=/tmp/mooship-composer-cache \ + php sh < "$SCRIPT_DIR/lib/install-dependencies.sh") || + error "Could not install and verify Moodle's locked PHP dependencies" +} diff --git a/lib/web.py b/lib/web.py index 12d65fa..57cebda 100644 --- a/lib/web.py +++ b/lib/web.py @@ -68,10 +68,17 @@ def http_configuration(content: str) -> str: def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("configuration", type=Path) + parser.add_argument("--public-root", action="store_true", + help="Select Moodle 5.1+ public/ without replacing the mounted file") args = parser.parse_args() try: content = args.configuration.read_text(encoding="utf-8") - result = http_configuration(content) + if args.public_root: + result = re.sub(r"(?m)^(\s*#?\s*)root /var/www/html;", + r"\1root /var/www/html/public;", content) + else: + result = http_configuration(content) + # Preserve the inode: nginx bind-mounts this file and must see its new contents. args.configuration.write_text(result, encoding="utf-8") except (OSError, ValueError) as error: print("Mooship: {}".format(error), file=sys.stderr) diff --git a/moodle/.config/nginx/_site.conf b/moodle/.config/nginx/_site.conf index efafd6b..4f9bfba 100644 --- a/moodle/.config/nginx/_site.conf +++ b/moodle/.config/nginx/_site.conf @@ -10,6 +10,11 @@ real_ip_recursive off; server { listen 80; server_name MYSITE.COM; + location = /mooship-healthz { + auth_basic off; + default_type text/plain; + return 200 "ok\n"; + } location ^~ /.well-known/acme-challenge/ { auth_basic off; allow all; @@ -33,6 +38,11 @@ server { server { listen 443 ssl; server_name MYSITE.COM; + location = /mooship-healthz { + auth_basic off; + default_type text/plain; + return 200 "ok\n"; + } charset UTF-8; server_tokens off; @@ -143,10 +153,14 @@ server { location ~ [^/]\.php(/|$) { fastcgi_split_path_info ^(.+\.php)(/.+)$; + # Preserve slash arguments before try_files resets FastCGI path info. + set $moodle_path_info $fastcgi_path_info; + # Removed PHP files can be router shims; an absent router remains a 404. + try_files $fastcgi_script_name /r.php =404; fastcgi_index index.php; fastcgi_pass {SLUG}_php:9000; include fastcgi_params; - fastcgi_param PATH_INFO $fastcgi_path_info; + fastcgi_param PATH_INFO $moodle_path_info; fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; } diff --git a/moodle/.config/php/fpm-pool.conf b/moodle/.config/php/fpm-pool.conf index b9077f5..b60d144 100644 --- a/moodle/.config/php/fpm-pool.conf +++ b/moodle/.config/php/fpm-pool.conf @@ -6,6 +6,7 @@ ; passed to the container by docker-compose.yml). A Moodle worker typically uses 50-100 MB. ; Change the values in .env, then run: docker compose up -d php [www] +ping.path = /mooship-fpm-ping pm = dynamic pm.max_children = ${PHP_FPM_MAX_CHILDREN} pm.start_servers = ${PHP_FPM_START_SERVERS} diff --git a/moodle/.docker/php/php74.Dockerfile b/moodle/.docker/php/php74.Dockerfile index 71347fd..12bac2b 100644 --- a/moodle/.docker/php/php74.Dockerfile +++ b/moodle/.docker/php/php74.Dockerfile @@ -53,7 +53,7 @@ RUN apt update && apt install -y \ libmcrypt-dev libssl-dev \ libreadline-dev libxslt1-dev zlib1g-dev \ python3-dev python3-psycopg2 \ - libpq-dev libfcgi-dev \ + libpq-dev libfcgi-dev libfcgi-bin \ gnupg ca-certificates \ htop tree jq \ supervisor procps \ diff --git a/moodle/.docker/php/php80.Dockerfile b/moodle/.docker/php/php80.Dockerfile index 0eb4ec0..6081bc9 100644 --- a/moodle/.docker/php/php80.Dockerfile +++ b/moodle/.docker/php/php80.Dockerfile @@ -52,7 +52,7 @@ RUN apt update && apt install -y \ libmcrypt-dev libssl-dev \ libreadline-dev libxslt1-dev zlib1g-dev \ python3-dev python3-psycopg2 \ - libpq-dev libfcgi-dev \ + libpq-dev libfcgi-dev libfcgi-bin \ gnupg ca-certificates \ htop tree jq \ supervisor procps \ diff --git a/moodle/.docker/php/php81.Dockerfile b/moodle/.docker/php/php81.Dockerfile index 310a0ed..de0994c 100644 --- a/moodle/.docker/php/php81.Dockerfile +++ b/moodle/.docker/php/php81.Dockerfile @@ -52,7 +52,7 @@ RUN apt update && apt install -y \ libmcrypt-dev libssl-dev \ libreadline-dev libxslt1-dev zlib1g-dev \ python3-dev python3-psycopg2 \ - libpq-dev libfcgi-dev \ + libpq-dev libfcgi-dev libfcgi-bin \ gnupg ca-certificates \ htop tree jq \ supervisor procps \ diff --git a/moodle/.docker/php/php82.Dockerfile b/moodle/.docker/php/php82.Dockerfile index c3c7a27..59e50ae 100644 --- a/moodle/.docker/php/php82.Dockerfile +++ b/moodle/.docker/php/php82.Dockerfile @@ -52,7 +52,7 @@ RUN apt update && apt install -y \ libmcrypt-dev libssl-dev \ libreadline-dev libxslt1-dev zlib1g-dev \ python3-dev python3-psycopg2 \ - libpq-dev libfcgi-dev \ + libpq-dev libfcgi-dev libfcgi-bin \ gnupg ca-certificates \ htop tree jq \ supervisor procps \ diff --git a/moodle/.docker/php/php83.Dockerfile b/moodle/.docker/php/php83.Dockerfile index 3ab64c2..9752309 100644 --- a/moodle/.docker/php/php83.Dockerfile +++ b/moodle/.docker/php/php83.Dockerfile @@ -52,7 +52,7 @@ RUN apt update && apt install -y \ libmcrypt-dev libssl-dev \ libreadline-dev libxslt1-dev zlib1g-dev \ python3-dev python3-psycopg2 \ - libpq-dev libfcgi-dev \ + libpq-dev libfcgi-dev libfcgi-bin \ gnupg ca-certificates \ htop tree jq \ supervisor procps \ diff --git a/moodle/.docker/php/php84.Dockerfile b/moodle/.docker/php/php84.Dockerfile index dee2a3e..eec2e3e 100644 --- a/moodle/.docker/php/php84.Dockerfile +++ b/moodle/.docker/php/php84.Dockerfile @@ -52,7 +52,7 @@ RUN apt update && apt install -y \ libmcrypt-dev libssl-dev \ libreadline-dev libxslt1-dev zlib1g-dev \ python3-dev python3-psycopg2 \ - libpq-dev libfcgi-dev \ + libpq-dev libfcgi-dev libfcgi-bin \ gnupg ca-certificates \ htop tree jq \ supervisor procps \ diff --git a/moodle/docker-compose.yml b/moodle/docker-compose.yml index 2714327..eb9d62b 100644 --- a/moodle/docker-compose.yml +++ b/moodle/docker-compose.yml @@ -14,6 +14,12 @@ services: restart: always logging: *default-logging # Moodle stores sessions in Redis (config.php) + healthcheck: + test: ["CMD-SHELL", "SCRIPT_NAME=/mooship-fpm-ping SCRIPT_FILENAME=/mooship-fpm-ping REQUEST_METHOD=GET cgi-fcgi -bind -connect 127.0.0.1:9000 | grep -q '^pong'"] + interval: 5s + timeout: 5s + retries: 24 + start_period: 10s depends_on: - redis environment: @@ -44,6 +50,12 @@ services: restart: always logging: *default-logging working_dir: /var/www/html + healthcheck: + test: ["CMD-SHELL", "wget -q -O - http://127.0.0.1/mooship-healthz | grep -qx ok"] + interval: 5s + timeout: 5s + retries: 24 + start_period: 10s depends_on: - php - db @@ -167,6 +179,14 @@ services: container_name: "{SLUG}_redis" restart: always logging: *default-logging + environment: + REDIS_PASSWORD: ${REDIS_PASSWORD} + healthcheck: + test: ["CMD-SHELL", "REDISCLI_AUTH=\"$${REDIS_PASSWORD}\" redis-cli ping | grep -qx PONG"] + interval: 5s + timeout: 5s + retries: 24 + start_period: 10s command: - redis-server - --requirepass diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..a779f21 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,41 @@ +[build-system] +requires = ["hatchling==1.32.4"] +build-backend = "hatchling.build" + +[project] +name = "mooship" +dynamic = ["version"] +description = "Install and run Mooship's Moodle deployment, backup and diagnostics toolkit." +readme = "docs/PYTHON.md" +requires-python = ">=3.8" +license = "MIT" +dependencies = [] +classifiers = [ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Environment :: Console", + "Operating System :: POSIX :: Linux", + "Topic :: System :: Installation/Setup", +] + +[project.scripts] +mooship = "mooship.cli:main" + +[project.urls] +Homepage = "https://github.com/shellharbor/mooship" +Documentation = "https://github.com/shellharbor/mooship/blob/main/docs/PYTHON.md" +Source = "https://github.com/shellharbor/mooship" +Issues = "https://github.com/shellharbor/mooship/issues" + +[tool.hatch.version] +path = "VERSION" +pattern = "(?P[0-9]+\\.[0-9]+\\.[0-9]+)" + +[tool.hatch.build.targets.sdist] +include = ["/src/mooship", "/VERSION", "/pyproject.toml", "/docs/PYTHON.md", "/LICENSE"] + +[tool.hatch.build.targets.wheel] +packages = ["src/mooship"] + +[tool.hatch.build.targets.wheel.force-include] +VERSION = "mooship/_release.txt" diff --git a/requirements-python-build.txt b/requirements-python-build.txt new file mode 100644 index 0000000..4c0c9f6 --- /dev/null +++ b/requirements-python-build.txt @@ -0,0 +1,3 @@ +# Development/release tools only; the installed package has no dependencies. +build==1.6.1 +twine==7.0.0 diff --git a/restore.sh b/restore.sh index fee446b..5fada30 100644 --- a/restore.sh +++ b/restore.sh @@ -52,7 +52,7 @@ restore_main() ( } trap cleanup_restore EXIT python3 "$SCRIPT_DIR/lib/state.py" extract "$archive" "$stage" "$SLUG" - for key in SITE_HOST SITE_SCHEME DB_NATIVE PHP_VERSION MOODLE_VERSION DB_PORT \ + for key in SITE_HOST SITE_SCHEME DB_NATIVE DB_HOST PHP_VERSION MOODLE_VERSION DB_PORT \ DB_POSTGRES_NAME DB_POSTGRES_USER DB_POSTGRES_PASSWORD \ DB_MYSQL_NAME DB_MYSQL_USER DB_MYSQL_PASSWORD DB_MYSQL_PASSWORD_ROOT AUTH_USER; do printf -v "$key" '%s' "$(mooship_read_env "$stage/project/.env" "$key")" @@ -71,6 +71,9 @@ restore_main() ( else DB_TYPE=mysql; PORT_MYSQL="$DB_PORT"; fi mooship_validate_database_settings if [[ "$DB_NATIVE" == true ]]; then + # Old archives contain the original bridge address in config.php. + NATIVE_DB_HOST="${DB_HOST:-172.17.0.1}" + mooship_validate_db_host "$NATIVE_DB_HOST" [[ "$DB_TYPE" != mysql || -n "$DB_ROOT_PASSWORD" ]] || error "Native MySQL requires --db-root-password" mooship_require_new_native_database fi @@ -108,6 +111,7 @@ restore_main() ( comment_ssl_blocks fi build_and_start_project + start_moodle_cron update_proxy_nginx_conf update_proxy_docker_compose if [[ "$has_ssl" == false ]]; then comment_ssl_blocks; fi diff --git a/src/mooship/__init__.py b/src/mooship/__init__.py new file mode 100644 index 0000000..347234a --- /dev/null +++ b/src/mooship/__init__.py @@ -0,0 +1,8 @@ +"""Mooship's dependency-free installation and execution adapter.""" + +from pathlib import Path + +_release_file = Path(__file__).with_name("_release.txt") +if not _release_file.is_file(): + _release_file = Path(__file__).resolve().parents[2] / "VERSION" +__version__ = _release_file.read_text(encoding="ascii").strip() diff --git a/src/mooship/__main__.py b/src/mooship/__main__.py new file mode 100644 index 0000000..021c170 --- /dev/null +++ b/src/mooship/__main__.py @@ -0,0 +1,5 @@ +"""Run the installed CLI with python -m mooship.""" + +from .cli import main + +raise SystemExit(main()) diff --git a/src/mooship/cli.py b/src/mooship/cli.py new file mode 100644 index 0000000..549b448 --- /dev/null +++ b/src/mooship/cli.py @@ -0,0 +1,92 @@ +"""Install Mooship and run its existing Moodle deployment toolkit.""" + +from __future__ import annotations + +import argparse +import os +from pathlib import Path +import shutil +import sys +import tarfile +from urllib.error import URLError + +from . import __version__ +from .install import absolute_path, install_native, installation_path, release_version + +COMMANDS = { + "deploy": "deploy-moodle.sh", "backup": "backup.sh", "restore": "restore.sh", + "schedule-backup": "schedule-backup.sh", "doctor": "doctor.sh", + "activate": "activate.sh", "deactivate": "deactivate.sh", "remove": "remove.sh", + "list": "list-projects.sh", +} + + +def parser() -> argparse.ArgumentParser: + result = argparse.ArgumentParser(description=__doc__) + result.add_argument("--version", action="version", version=f"mooship {__version__}") + commands = result.add_subparsers(dest="command", required=True) + install = commands.add_parser("install", help="Download and verify an official toolkit release") + install.add_argument("--version", dest="release", default=__version__, help="Exact toolkit version") + install.add_argument("--directory", default="/opt/mooship", help="New toolkit directory") + install.add_argument("--dry-run", action="store_true", help="Preview without downloads or writes") + for name in (*COMMANDS, "toolkit-version"): + command = commands.add_parser(name, help=f"Run the existing {name} command") + command.add_argument("--directory", default="/opt/mooship", help="Installed toolkit or reviewed checkout") + command.add_argument("arguments", nargs=argparse.REMAINDER, help="Pass backend arguments after --") + return result + + +def require_linux() -> None: + if sys.platform != "linux": + raise ValueError("Run installation and server operations on the target Linux server over SSH") + + +def toolkit_root(directory: str) -> Path: + root = absolute_path(directory) + if (root / ".mooship-installing").exists(): + raise ValueError("Toolkit installation is incomplete; inspect it before use") + if not (root / "VERSION").is_file(): + raise ValueError("Toolkit is missing; run 'mooship install' or select --directory") + release_version((root / "VERSION").read_text(encoding="ascii").strip()) + return root + + +def main(argv: list[str] | None = None) -> int: + args = parser().parse_args(argv) + try: + if args.command == "install": + version = release_version(args.release) + destination = installation_path(args.directory) + if args.dry_run: + print(f"Would download and verify Mooship {version}, then install into {destination}") + return 0 + require_linux() + install_native(version, str(destination)) + print(f"Installed Mooship {version} into {destination}") + return 0 + arguments = args.arguments[1:] if args.arguments[:1] == ["--"] else args.arguments + root = toolkit_root(args.directory) + if args.command == "toolkit-version": + if arguments: + raise ValueError("toolkit-version does not accept backend arguments") + print((root / "VERSION").read_text(encoding="ascii").strip()) + return 0 + require_linux() + script = root / COMMANDS[args.command] + if not script.is_file(): + raise ValueError("Toolkit command is missing; select a complete toolkit with --directory") + if not shutil.which("bash"): + raise ValueError("Bash is required; install it on the target Linux server") + # Fixed Bash executable and argv-only handoff preserve I/O, signals and status without printing secrets. + os.execvp("bash", ["bash", str(script), *arguments]) # nosec B606 + return 0 + except KeyboardInterrupt: + print("Mooship operation cancelled", file=sys.stderr) + return 130 + except (ValueError, OSError, URLError, tarfile.TarError, shutil.Error) as exc: + print(f"Mooship: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/mooship/install.py b/src/mooship/install.py new file mode 100644 index 0000000..0a0dc40 --- /dev/null +++ b/src/mooship/install.py @@ -0,0 +1,163 @@ +"""Download, verify and install an explicit official Mooship release.""" + +import hashlib +import gzip +import os +from pathlib import Path, PurePosixPath +import re +import shutil +import tarfile +import tempfile +import time +from urllib.parse import urlsplit +from urllib.request import HTTPRedirectHandler, build_opener + +MAX_DOWNLOAD = 64 * 1024 * 1024 +MAX_EXPANDED = 128 * 1024 * 1024 +MAX_MEMBERS = 20000 +RELEASE_PATTERN = r"[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z]+(?:\.[0-9A-Za-z]+)*)?" +REQUIRED_FILES = ("VERSION", "deploy-moodle.sh", "backup.sh", "restore.sh", "doctor.sh", + "schedule-backup.sh", "activate.sh", "deactivate.sh", "remove.sh", + "list-projects.sh", "lib/common.sh", "lib/state.py", "lib/web.py", + "lib/proxy.py", "moodle/docker-compose.yml", "nginxproxy/nginx.conf") + + +def release_version(value: str) -> str: + """Accept release filenames, without path components or mutable aliases.""" + if not re.fullmatch(RELEASE_PATTERN, value): + raise ValueError("Use an exact release version, for example 1.2.2 or 1.3.0-rc.1") + return value + + +def absolute_path(value: str) -> Path: + path = Path(value).expanduser() + if not path.is_absolute() or ".." in path.parts or any(c in str(path) for c in "\x00\r\n,"): + raise ValueError("Use an absolute path without '..', commas or control characters") + return path + + +def installation_path(value: str) -> Path: + destination = absolute_path(value) + for path in (destination, *destination.parents): + if path.is_symlink(): + raise ValueError("Installation paths must not contain symbolic links") + if destination.exists(): + raise ValueError("Installation path already exists; choose a new directory with --directory") + if not destination.parent.is_dir(): + raise ValueError("Installation parent directory must already exist") + return destination + + +class HTTPSRedirects(HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + if urlsplit(newurl).scheme != "https": + raise ValueError("Refusing a download redirect without HTTPS") + return super().redirect_request(req, fp, code, msg, headers, newurl) + + +def download(url: str, destination: Path, limit: int = MAX_DOWNLOAD) -> None: + """Bound network operations and stream into an exclusively created private file.""" + if urlsplit(url).scheme != "https": + raise ValueError("Release downloads require HTTPS") + started = time.monotonic() + opener = build_opener(HTTPSRedirects()) + with opener.open(url, timeout=30) as response: + with destination.open("xb") as output: + os.chmod(destination, 0o600) + total = 0 + while chunk := response.read(65536): + total += len(chunk) + if total > limit or time.monotonic() - started > 180: + raise ValueError("Release download exceeded its size or time limit") + output.write(chunk) + + +def verify_checksum(archive: Path, checksum: Path, filename: str) -> None: + text = checksum.read_text(encoding="ascii").strip() + match = re.fullmatch(r"([0-9a-fA-F]{64}) [ *]" + re.escape(filename), text) + if not match: + raise ValueError("Checksum file must contain exactly the requested archive name and SHA-256") + digest = hashlib.sha256() + with archive.open("rb") as source: + for chunk in iter(lambda: source.read(65536), b""): + digest.update(chunk) + if digest.hexdigest() != match[1].lower(): + raise ValueError("Release archive SHA-256 does not match; nothing was installed") + + +def unpack_release(archive: Path, staging: Path, version: str) -> Path: + """Extract regular files only; never follow archive links or traversal paths.""" + prefix = f"mooship-{release_version(version)}" + staging.mkdir(mode=0o700, parents=True) + unpacked_tar = staging / "release.tar" + expanded = 0 + # Bound the complete tar stream, including PAX/GNU headers, before parsing it. + with gzip.open(archive, "rb") as compressed, unpacked_tar.open("xb") as output: + while chunk := compressed.read(65536): + expanded += len(chunk) + if expanded > MAX_EXPANDED: + raise ValueError("Expanded release archive exceeds the size limit") + output.write(chunk) + seen: set[str] = set() + total = 0 + with tarfile.open(unpacked_tar, "r:") as source: + for count, member in enumerate(source, 1): + path = PurePosixPath(member.name) + parts = path.parts + if (count > MAX_MEMBERS or path.is_absolute() or not parts or parts[0] != prefix + or ".." in parts or "\\" in member.name + or any(ord(c) < 32 for c in member.name) + or not (member.isdir() or member.isfile()) or member.name in seen): + raise ValueError("Release archive contains an unsafe or duplicate entry") + seen.add(member.name) + total += member.size + if member.size < 0 or total > MAX_EXPANDED: + raise ValueError("Expanded release archive exceeds the size limit") + target = staging.joinpath(*parts) + if member.isdir(): + target.mkdir(parents=True, exist_ok=True) + continue + target.parent.mkdir(parents=True, exist_ok=True) + reader = source.extractfile(member) + if reader is None: + raise ValueError("Release archive contains an unreadable file") + with reader, target.open("xb") as output: + shutil.copyfileobj(reader, output) + target.chmod(0o755 if member.mode & 0o111 else 0o644) + root = staging / prefix + if any(not (root / name).is_file() for name in REQUIRED_FILES): + raise ValueError("Release archive is missing required Mooship files") + if (root / "VERSION").read_text(encoding="ascii").strip() != version: + raise ValueError("Release archive VERSION does not match the requested release") + if (root / ".mooship-installing").exists(): + raise ValueError("Release archive contains a reserved installation marker") + # A permissive caller umask must not make the installed root-run toolkit writable. + root.chmod(0o755) + for path in root.rglob("*"): + if path.is_dir(): + path.chmod(0o755) + return root + + +def install_native(version: str, directory: str) -> Path: + version = release_version(version) + destination = installation_path(directory) + filename = f"mooship-{version}.tar.gz" + base = f"https://github.com/shellharbor/mooship/releases/download/v{version}" + with tempfile.TemporaryDirectory(prefix="mooship-install-") as temporary: + private = Path(temporary) + archive = private / filename + checksum = private / f"{filename}.sha256" + download(f"{base}/{filename}.sha256", checksum, limit=4096) + download(f"{base}/{filename}", archive) + verify_checksum(archive, checksum, filename) + staged = unpack_release(archive, private / "unpacked", version) + # Reserve exclusively, including in a concurrent install. Failed copies stay + # visible and marked for inspection; they are never removed or overwritten. + destination.mkdir(mode=0o755) + marker = destination / ".mooship-installing" + marker.write_text("Installation incomplete; inspect this directory before use.\n", encoding="ascii") + marker.chmod(0o600) + shutil.copytree(staged, destination, dirs_exist_ok=True) + marker.unlink() + return destination diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py new file mode 100644 index 0000000..78e0717 --- /dev/null +++ b/tests/test_dependencies.py @@ -0,0 +1,95 @@ +"""Refuse failed or unverified dependency installation before application initialization.""" +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] + + +@unittest.skipUnless(os.name == 'posix' and shutil.which('sh'), 'Requires a Linux shell') +class DependencyTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.folder = Path(temporary.name) + self.bin = self.folder / 'bin' + self.bin.mkdir() + self.project = self.folder / 'moodle' + self.project.mkdir() + self.log = self.folder / 'trace' + self.stub('php', '''case "$1" in + -r) + if printf '%s' "$2" | grep -q json_decode; then exit "$METADATA_STATUS"; fi + echo TRAIT >> "$TRACE"; exit "$TRAIT_STATUS" ;; + *) echo "COMPOSER $*" >> "$TRACE"; exit "$INSTALL_STATUS" ;; +esac +''') + self.stub('curl', '''echo "CURL $*" >> "$TRACE" +while [ "$1" != -o ]; do shift; done +printf 'inert fixture' > "$2" +echo "TEMP $2" >> "$TRACE" +exit "$DOWNLOAD_STATUS" +''') + self.stub('sha256sum', '''read -r checksum path +echo "HASH $checksum" >> "$TRACE" +exit "$HASH_STATUS" +''') + + def stub(self, name, body): + path = self.bin / name + path.write_text('#!/bin/sh\n' + body, encoding='utf-8') + path.chmod(0o755) + + def invoke(self, lock=True, **changes): + if lock: + (self.project / 'composer.lock').write_text('{}', encoding='utf-8') + self.log.write_text('', encoding='utf-8') + environment = dict(os.environ, PATH=str(self.bin) + os.pathsep + os.environ['PATH'], + TRACE=str(self.log), METADATA_STATUS='0', TRAIT_STATUS='0', + INSTALL_STATUS='0', DOWNLOAD_STATUS='0', HASH_STATUS='0') + environment.update(changes) + result = subprocess.run(['sh', str(ROOT / 'lib/install-dependencies.sh'), str(self.project)], + env=environment, capture_output=True, text=True, timeout=15) + trace = self.log.read_text(encoding='utf-8') + for line in trace.splitlines(): + if line.startswith('TEMP '): + self.assertFalse(Path(line[5:]).exists(), 'Temporary Composer PHAR must be removed') + return result, trace + + def test_legacy_manifest_skips_download_and_execution(self): + result, trace = self.invoke(lock=False, METADATA_STATUS='3') + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(trace, '') + + def test_invalid_metadata_refuses_download(self): + result, trace = self.invoke(METADATA_STATUS='1') + self.assertNotEqual(result.returncode, 0) + self.assertEqual(trace, '') + + def test_missing_lockfile_refuses_resolution_and_download(self): + result, trace = self.invoke(lock=False) + self.assertNotEqual(result.returncode, 0) + self.assertIn('composer.lock is missing', result.stderr) + self.assertEqual(trace, '') + + def test_download_checksum_install_and_autoload_failures_are_fatal(self): + for stage in ('DOWNLOAD_STATUS', 'HASH_STATUS', 'INSTALL_STATUS', 'TRAIT_STATUS'): + with self.subTest(stage=stage): + result, trace = self.invoke(**{stage: '17'}) + self.assertNotEqual(result.returncode, 0) + if stage in ('DOWNLOAD_STATUS', 'HASH_STATUS'): + self.assertNotIn('COMPOSER ', trace) + + def test_pinned_verified_install_has_no_developer_code_or_script_execution(self): + result, trace = self.invoke() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('https://getcomposer.org/download/2.10.3/composer.phar', trace) + self.assertIn('HASH 7a2d379d5b8ffdaa028580ef26494c36d2feef4b178d3dd1473a4dbc5e17c8d6', trace) + for option in ('--no-dev', '--no-scripts', '--no-plugins', '--no-interaction'): + self.assertIn(option, trace) + self.assertNotIn('--ignore-platform', trace) + self.assertLess(trace.index('HASH '), trace.index('COMPOSER ')) + self.assertTrue(trace.rstrip().endswith('TRAIT')) diff --git a/tests/test_http_roundtrip.py b/tests/test_http_roundtrip.py index 96d51c4..53dc5ee 100644 --- a/tests/test_http_roundtrip.py +++ b/tests/test_http_roundtrip.py @@ -7,6 +7,7 @@ from pathlib import Path import shutil import subprocess +import sys import tempfile import time import unittest @@ -37,10 +38,21 @@ def setUp(self): # Recursive cleanup is limited to the fixture directory created in this workspace. self.assertEqual(self.directory.parent, ROOT.resolve()) self.assertTrue(self.directory.name.startswith(".mooship-http-test-")) - self.addCleanup(temporary.cleanup) + def cleanup_fixture(): + # Docker Desktop may release Windows bind mounts shortly after container removal. + for attempt in range(11): + try: + temporary.cleanup() + return + except PermissionError: + if os.name != 'nt' or attempt == 10: + raise + time.sleep(1) + self.addCleanup(cleanup_fixture) self.slug = "mooship_http_" + uuid.uuid4().hex[:12] self.network = self.slug self.containers = [] + self.extra_networks = [] # Containers run as the invoking user so Linux bind mounts stay readable and writable. if hasattr(os, "getuid"): self.uid, self.gid = os.getuid(), os.getgid() @@ -54,6 +66,8 @@ def cleanup_docker(self): for name in reversed(self.containers): subprocess.run(["docker", "rm", "--force", name], capture_output=True, timeout=30) subprocess.run(["docker", "network", "rm", self.network], capture_output=True, timeout=30) + for network in self.extra_networks: + subprocess.run(["docker", "network", "rm", network], capture_output=True, timeout=30) def mount(self, source, target): return ["--mount", "type=bind,source=" + str(source) + ",target=" + target + ",readonly"] @@ -89,10 +103,10 @@ def certificates(self): shutil.copyfile(keys / "privkey.pem", folder / "privkey.pem") return certificates - def prepare(self, https=False, basic_auth=False, public=False): + def prepare(self, https=False, basic_auth=False, public=False, legacy_root_at_start=False): project = (ROOT / "moodle/.config/nginx/_site.conf").read_text(encoding="utf-8") project = project.replace("MYSITE.COM", "test.example").replace("{SLUG}", self.slug) - if public: + if public and not legacy_root_at_start: project = project.replace("root /var/www/html;", "root /var/www/html/public;") if basic_auth: project = project.replace('# auth_basic ', ' auth_basic ') @@ -144,7 +158,8 @@ def prepare(self, https=False, basic_auth=False, public=False): ["php-fpm", "--fpm-config", "/fixtures/php-fpm.conf", "--nodaemonize"], common) self.start(self.slug + "_nginx", "nginx:1.29.1-alpine", ["nginx", "-g", "daemon off;"], - common + [(self.directory / "project-main.conf", "/etc/nginx/nginx.conf")]) + common + [(self.directory / "project-main.conf", "/etc/nginx/nginx.conf"), + (self.directory / "project.conf", "/fixtures/project.conf")]) self.proxy = self.slug + "_proxy" self.start(self.proxy, "nginx:1.29.1-alpine", ["nginx", "-g", "daemon off;"], @@ -205,6 +220,11 @@ def verify_site(self, https=False, basic_auth=False, public=False): self.assertEqual(self.request("/course/", basic_auth)[:2], (200, "COURSE")) self.assertEqual(self.request("/virtual/route?id=7", basic_auth)[:2], (200, "ROUTER:/virtual/route?id=7")) + # Retired PHP paths are router shims too, including their original query/path info. + self.assertEqual(self.request('/retired.php?id=7', basic_auth)[:2], + (200, 'ROUTER:/retired.php?id=7')) + self.assertEqual(self.request('/retired.php/extra?id=7', basic_auth)[:2], + (200, 'ROUTER:/retired.php/extra?id=7')) # $CFG->xsendfile: PHP hands the file to nginx through the internal /dataroot/ location. self.assertEqual(self.request("/sendfile.php", basic_auth)[:2], (200, "synthetic moodledata file")) # PHP sees the client address, not nginxproxy's, and clients cannot forge it. @@ -238,6 +258,35 @@ def test_stopped_project_does_not_stop_the_proxy(self): def test_http_legacy_document_root(self): self.verify_site() + def test_php_self_requests_resolve_the_synthetic_site_through_the_proxy(self): + self.prepare() + network = self.slug + '_self_checks' + self.extra_networks.append(network) + self.docker('network', 'create', '--internal', '--label', 'org.mooship.test=http', network) + self.docker('network', 'connect', '--alias', 'self.mooship.test', network, self.proxy) + self.docker('network', 'connect', network, self.slug + '_php') + result = self.docker('exec', self.slug + '_php', 'php', '-r', + 'echo file_get_contents("http://self.mooship.test/course/index.php");') + self.assertEqual(result, 'COURSE') + + def test_public_root_change_is_applied_to_the_running_nginx(self): + self.prepare(public=True, legacy_root_at_start=True) + self.assertEqual(self.request('/course/index.php')[0], 404) + path = self.directory / 'project.conf' + inode = path.stat().st_ino + subprocess.run([sys.executable, '-B', str(ROOT / 'lib/web.py'), '--public-root', str(path)], + check=True, capture_output=True, timeout=10) + self.assertEqual(path.stat().st_ino, inode) + self.assertEqual(self.request('/course/index.php')[0], 404) + self.docker('exec', self.slug + '_nginx', 'nginx', '-t') + self.docker('exec', self.slug + '_nginx', 'nginx', '-s', 'reload') + for _ in range(20): + result = self.request('/course/index.php') + if result[:2] == (200, 'COURSE'): + break + time.sleep(0.1) + self.assertEqual(result[:2], (200, 'COURSE')) + def test_http_public_document_root_with_basic_auth(self): self.verify_site(basic_auth=True, public=True) diff --git a/tests/test_native_preflight.py b/tests/test_native_preflight.py new file mode 100644 index 0000000..28ca676 --- /dev/null +++ b/tests/test_native_preflight.py @@ -0,0 +1,79 @@ +"""Real PHP-to-host DB probes against disposable databases published on random ports.""" + +import json +import os +from pathlib import Path +import subprocess +import time +import unittest +import uuid + +ROOT = Path(__file__).resolve().parents[1] + + +def docker(*args, timeout=120, input_text=None): + return subprocess.run(["docker", *args], capture_output=True, text=True, encoding="utf-8", + errors="replace", timeout=timeout, check=False, input=input_text) + + +@unittest.skipUnless(os.environ.get("MOOSHIP_NATIVE_PREFLIGHT_TESTS") == "1", "Opt-in native DB routing checks") +class NativePreflightTests(unittest.TestCase): + def test_php_authenticates_via_host_gateway_and_refuses_failures(self): + image = os.environ["MOOSHIP_PREFLIGHT_PHP_IMAGE"] + engine = os.environ.get("MOOSHIP_PREFLIGHT_DB", "postgres") + name = "mooship-native-probe-" + uuid.uuid4().hex[:12] + password = "Synthetic_" + uuid.uuid4().hex + db_image = "postgres:17-alpine" if engine == "postgres" else "mysql:8.4" + port = "5432" if engine == "postgres" else "3306" + settings = (["POSTGRES_USER=moodle", "POSTGRES_DB=moodle", "POSTGRES_PASSWORD=" + password] + if engine == "postgres" else + ["MYSQL_USER=moodle", "MYSQL_DATABASE=moodle", "MYSQL_PASSWORD=" + password, + "MYSQL_ROOT_PASSWORD=" + uuid.uuid4().hex]) + for reference in (image, db_image): + self.assertEqual(docker("image", "inspect", reference).returncode, 0, reference) + options = ["run", "-d", "--name", name, "--label", "io.mooship.test=" + name, + "--publish", "0.0.0.0::" + port] + for setting in settings: + options.extend(["--env", setting]) + self.addCleanup(lambda: docker("rm", "-f", "-v", name)) + result = docker(*options, db_image) + self.assertEqual(result.returncode, 0, result.stderr) + metadata = json.loads(docker("inspect", name).stdout)[0] + published = metadata["NetworkSettings"]["Ports"][port + "/tcp"][0]["HostPort"] + source = (ROOT / "lib/check-database.php").read_text(encoding="utf-8") + + def probe(mode="connect", secret=password): + args = ["run", "--rm", "-i", "--read-only", "--cap-drop=ALL", + "--security-opt=no-new-privileges:true", "--user", "1000:1000", + "--label", "io.mooship.test=" + name, + "--add-host", "host.docker.internal:host-gateway", "--entrypoint", "php"] + for setting in ("MOOSHIP_DB_TYPE=" + engine, "MOOSHIP_DB_HOST=host.docker.internal", + "MOOSHIP_DB_PORT=" + published, "MOOSHIP_DB_NAME=moodle", + "MOOSHIP_DB_USER=moodle", "MOOSHIP_DB_PASSWORD=" + secret, + "MOOSHIP_DB_CHECK=" + mode): + args.extend(["--env", setting]) + return docker(*args, image, input_text=source, timeout=30) + + deadline = time.monotonic() + 180 + while True: + result = probe() + if result.returncode == 0: + break + self.assertLess(time.monotonic(), deadline, result.stderr) + time.sleep(2) + self.assertEqual(probe("tcp").returncode, 0) + self.assertEqual(probe("installed").returncode, 3) + denied = probe(secret="Incorrect_" + uuid.uuid4().hex) + self.assertNotEqual(denied.returncode, 0) + self.assertNotIn(password, denied.stdout + denied.stderr) + for sql, expected in (("CREATE TABLE mdl_config (name VARCHAR(64), value VARCHAR(64))", 4), + ("INSERT INTO mdl_config VALUES ('version', '2024100715')", 0)): + command = ("PGPASSWORD=\"$POSTGRES_PASSWORD\" psql -U moodle -d moodle -v ON_ERROR_STOP=1 -c \"$1\"" + if engine == "postgres" else + "MYSQL_PWD=\"$MYSQL_PASSWORD\" mysql -umoodle -Dmoodle -e \"$1\"") + result = docker("exec", name, "sh", "-c", command, "sh", sql) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(probe("installed").returncode, expected) + self.assertEqual(docker("stop", "--time", "20", name).returncode, 0) + # Docker's host proxy may accept TCP even after its backend stops. + self.assertNotEqual(probe().returncode, 0) diff --git a/tests/test_operations.py b/tests/test_operations.py index d0965cf..9dc76b0 100644 --- a/tests/test_operations.py +++ b/tests/test_operations.py @@ -95,6 +95,7 @@ def test_admin_email_flag_reaches_the_installer(self): (self.project / ".env").write_text("SITE_HOST=test.example\n", encoding="utf-8") result = self.bash('''PROJECT_DIR="$WWW_DIR/test"; DOMAIN=test.example MOODLE_ADMIN_EMAIL=owner@example.org +mooship_check_database() { return 3; } docker() { [[ "$*" == *--help* ]] && return 0; printf '%s\\n' "$*" > "$TEST_DIR/install.log"; } install_moodle_database >/dev/null ''') diff --git a/tests/test_preflight.py b/tests/test_preflight.py new file mode 100644 index 0000000..e189aa1 --- /dev/null +++ b/tests/test_preflight.py @@ -0,0 +1,165 @@ +"""Port preflight, native DB routing, installation state and bounded service readiness.""" + +import shlex +import shutil +import socket + +from test_scripts import BashTestCase + + +class PreflightTests(BashTestCase): + def test_every_explicit_port_rejects_invalid_values_before_docker(self): + for name in ("PORT_HTTP", "PORT_HTTPS", "PORT_PHP", "PORT_REDIS", "PORT_POSTGRES", "PORT_MYSQL"): + for value in ("abc", "0", "-1", "65536", "123456", "12\n34"): + with self.subTest(name=name, value=value): + result = self.bash( + "DB_TYPE=postgres; " + name + "=" + shlex.quote(value) + "\n" + 'docker() { echo CALLED_DOCKER; }; assign_ports') + self.assertNotEqual(result.returncode, 0) + self.assertIn("Invalid port", result.stdout + result.stderr) + self.assertNotIn("CALLED_DOCKER", result.stdout) + + def test_proxy_ports_and_duplicate_normalized_ports_are_rejected(self): + for settings in ("PORT_HTTP=80", "PORT_PHP=443", "PORT_HTTP=08101; PORT_PHP=8101", + "DB_NATIVE=true; PORT_HTTP=5432"): + with self.subTest(settings=settings): + result = self.bash("DB_TYPE=postgres; " + settings + """ +docker() { :; }; mooship_port_available() { return 0; }; assign_ports +""") + self.assertNotEqual(result.returncode, 0) + self.assertRegex(result.stdout + result.stderr, "reserved|Duplicate") + + def test_occupied_socket_and_docker_publication_are_detected(self): + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + listener.listen() + result = self.bash("mooship_port_available " + str(listener.getsockname()[1])) + self.assertNotEqual(result.returncode, 0) + result = self.bash( + "MOOSHIP_DOCKER_PORTS='0.0.0.0:18101->80/tcp'; mooship_port_available 18101") + self.assertNotEqual(result.returncode, 0) + + def test_native_listener_is_allowed_and_defaults_are_reserved(self): + result = self.bash("""DB_TYPE=postgres; DB_NATIVE=true +PORT_HTTP=18101; PORT_HTTPS=14101; PORT_PHP=19101; PORT_REDIS=16501 +docker() { printf '0.0.0.0:5432->5432/tcp\\n'; } +mooship_port_available() { [[ "$1" != 5432 ]]; } +assign_ports; printf 'NATIVE=%s\\n' "$PORT_POSTGRES" +""") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("NATIVE=5432", result.stdout) + + def test_automatic_selection_skips_an_explicit_assignment(self): + result = self.bash("""MOOSHIP_ASSIGNED_PORTS=' 19101' +find_free_port 19101 19102 PHP +""") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), "19102") + + def test_native_host_flag_rejects_ignored_or_unsafe_values(self): + for flags in ("--db-host db.example", "--db-native --db-host 'db;echo injected'", + "--db-native --db-host 'db/host'"): + result = self.bash( + "parse_args --slug test --domain test.example --db-type postgres --no-ssl " + flags) + self.assertNotEqual(result.returncode, 0) + + def test_default_native_mapping_and_custom_host_are_used_by_php_and_cron(self): + for host in ("host.docker.internal", "192.0.2.10"): + with self.subTest(host=host): + # Each generation uses a separate temporary project directory. + result = self.bash("""DB_NATIVE=true; DB_TYPE=postgres; DOMAIN=test.example; PHP_VERSION=83 +NATIVE_DB_HOST=""" + shlex.quote(host) + """ +DB_POSTGRES_NAME=testsite; DB_POSTGRES_USER=testuser; DB_POSTGRES_PASSWORD=Synthetic_Pg +REDIS_PASSWORD=Synthetic_Redis; PORT_POSTGRES=5543 +PORT_HTTP=18101; PORT_HTTPS=14101; PORT_PHP=19101; PORT_REDIS=16501 +create_project +mkdir -p "$PROJECT_DIR/public_html"; touch "$PROJECT_DIR/public_html/version.php" +chown() { :; }; configure_moodle_config +""") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + config = (self.project / "public_html/config.php").read_text() + compose = (self.project / "docker-compose.yml").read_text() + self.assertIn("$CFG->dbhost = '" + host + "'", config) + self.assertIn("DB_HOST=" + host, (self.project / ".env").read_text()) + self.assertEqual(compose.count("host.docker.internal:host-gateway"), + 2 if host == "host.docker.internal" else 0) + shutil.rmtree(self.project) + + def test_database_probe_passes_secret_through_environment_and_selected_endpoint(self): + for native, host, port in (("true", "host.docker.internal", "5543"), ("false", "test_db", "5432")): + result = self.bash("""DB_TYPE=postgres; DB_NATIVE=""" + native + """ +PORT_POSTGRES=5543; DB_POSTGRES_NAME=testsite; DB_POSTGRES_USER=testuser +DB_POSTGRES_PASSWORD=Synthetic_Probe_Secret +docker() { + [[ "$MOOSHIP_DB_PASSWORD" == Synthetic_Probe_Secret ]] || return 9 + printf 'ENDPOINT=%s:%s ARGS=%s\\n' "$MOOSHIP_DB_HOST" "$MOOSHIP_DB_PORT" "$*" +} +mooship_check_database connect +""") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("ENDPOINT=" + host + ":" + port, result.stdout) + self.assertNotIn("Synthetic_Probe_Secret", result.stdout + result.stderr) + + def test_readiness_waits_for_health_and_is_bounded(self): + result = self.bash("""MOOSHIP_READY_ATTEMPTS=3 +docker() { + if [[ -f "$TEST_DIR/once" ]]; then echo 'running healthy' + else touch "$TEST_DIR/once"; echo 'running starting'; fi +} +sleep() { echo SLEPT; } +mooship_wait_service php +""") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.count("SLEPT"), 1) + for status in ("running unhealthy", "exited unhealthy"): + result = self.bash("""MOOSHIP_READY_ATTEMPTS=2 +docker() { echo '""" + status + """'; }; sleep() { :; } +mooship_wait_service nginx +""") + self.assertNotEqual(result.returncode, 0) + self.assertRegex(result.stderr, "exited|within 4 seconds") + + def test_unreachable_or_partial_database_never_runs_installer(self): + self.fixture() + for status in (1, 4): + result = self.bash('PROJECT_DIR="$WWW_DIR/test"\n' + "mooship_check_database() { return " + str(status) + "; }\n" + "docker() { echo INSTALLER_CALLED; }; install_moodle_database") + self.assertNotEqual(result.returncode, 0) + self.assertNotIn("INSTALLER_CALLED", result.stdout) + + def test_legacy_redis_readiness_checks_authenticated_ping(self): + result = self.bash('''PROJECT_DIR="$WWW_DIR/test"; MOOSHIP_READY_ATTEMPTS=2 +mooship_read_env() { printf Synthetic_Redis; } +docker() { + if [[ "$1" == inspect ]]; then echo 'running none'; + else [[ "$REDISCLI_AUTH" == Synthetic_Redis ]] && echo PONG; fi +} +mooship_wait_service redis +''') + self.assertEqual(result.returncode, 0, result.stderr) + + def test_installed_database_skips_the_installer(self): + self.fixture() + result = self.bash("""PROJECT_DIR="$WWW_DIR/test" +mooship_check_database() { return 0; }; docker() { echo INSTALLER_CALLED; } +install_moodle_database +""") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertNotIn("INSTALLER_CALLED", result.stdout) + + def test_nginx_configuration_is_checked_before_reload(self): + self.project.mkdir() + result = self.bash('''PROJECT_DIR="$WWW_DIR/test" +docker() { printf '%s\\n' "$*" >> "$TEST_DIR/reload.log"; } +reload_project_nginx +''') + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual((self.directory / 'reload.log').read_text().splitlines(), + ['compose exec -T nginx nginx -t', 'compose exec -T nginx nginx -s reload']) + result = self.bash('''PROJECT_DIR="$WWW_DIR/test" +docker() { [[ "$*" != *'nginx -t' ]] && echo RELOADED; return 42; } +reload_project_nginx +''') + self.assertNotEqual(result.returncode, 0) + self.assertNotIn('RELOADED', result.stdout) diff --git a/tests/test_python_cli.py b/tests/test_python_cli.py new file mode 100644 index 0000000..5dc42ed --- /dev/null +++ b/tests/test_python_cli.py @@ -0,0 +1,255 @@ +"""Exercise installer boundaries and backend forwarding without changing the server.""" + +from contextlib import redirect_stderr, redirect_stdout +import hashlib +import io +import os +from pathlib import Path +import subprocess +import sys +import tarfile +import tempfile +import unittest +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) +from mooship import cli, install + + +class InstallerTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.archive = self.root / "mooship-1.2.2.tar.gz" + self.destination = self.root / "toolkit" + + def archive_with(self, extra=(), version="1.2.2"): + with tarfile.open(self.archive, "w:gz") as archive: + for name in install.REQUIRED_FILES: + data = (version + "\n").encode() if name == "VERSION" else b"fixture\n" + member = tarfile.TarInfo(f"mooship-1.2.2/{name}") + member.size = len(data) + member.mode = 0o7777 if name.endswith(".sh") else 0o666 + archive.addfile(member, io.BytesIO(data)) + for member, data in extra: + archive.addfile(member, io.BytesIO(data) if data is not None else None) + return self.archive + + def fake_download(self, url, destination, limit=install.MAX_DOWNLOAD): + if url.endswith(".sha256"): + digest = hashlib.sha256(self.archive.read_bytes()).hexdigest() + destination.write_text(f"{digest} {self.archive.name}\n", encoding="ascii") + else: + destination.write_bytes(self.archive.read_bytes()) + + def perform(self): + with patch.object(install, "download", side_effect=self.fake_download): + return install.install_native("1.2.2", str(self.destination)) + + def test_verified_install_and_repeat_preserve_existing_files(self): + self.archive_with() + self.assertEqual(self.perform(), self.destination) + self.assertEqual((self.destination / "VERSION").read_text().strip(), "1.2.2") + if os.name == "posix": + self.assertEqual((self.destination / "deploy-moodle.sh").stat().st_mode & 0o7777, 0o755) + with patch.object(install, "download") as download: + with self.assertRaisesRegex(ValueError, "already exists"): + install.install_native("1.2.2", str(self.destination)) + download.assert_not_called() + self.assertEqual((self.destination / "VERSION").read_text().strip(), "1.2.2") + + def test_rejects_checksum_mismatch_and_other_filename_before_destination_creation(self): + self.archive_with() + checksum = self.root / "sum" + for content in (f"{'0' * 64} {self.archive.name}", + f"{'0' * 64} different.tar.gz", "malformed"): + with self.subTest(content=content): + checksum.write_text(content) + with self.assertRaises(ValueError): + install.verify_checksum(self.archive, checksum, self.archive.name) + self.assertFalse(self.destination.exists()) + + def test_checksum_failure_does_not_publish(self): + self.archive_with() + with patch.object(install, "download", side_effect=self.fake_download), \ + patch.object(install, "verify_checksum", side_effect=ValueError("bad checksum")): + with self.assertRaises(ValueError): + install.install_native("1.2.2", str(self.destination)) + self.assertFalse(self.destination.exists()) + + @unittest.skipUnless(os.name == "posix", "POSIX modes") + def test_permissive_umask_cannot_make_the_toolkit_world_writable(self): + self.archive_with() + previous = os.umask(0) + try: + self.perform() + finally: + os.umask(previous) + for path in (self.destination, self.destination / "moodle", self.destination / "lib"): + self.assertEqual(path.stat().st_mode & 0o7777, 0o755) + self.assertEqual((self.destination / "VERSION").stat().st_mode & 0o7777, 0o644) + + def test_rejects_links_traversal_absolute_duplicate_and_special_members(self): + for name, kind in (("mooship-1.2.2/../../escape", tarfile.REGTYPE), + ("/escape", tarfile.REGTYPE), + ("other/file", tarfile.REGTYPE), + ("mooship-1.2.2/link", tarfile.SYMTYPE), + ("mooship-1.2.2/hard", tarfile.LNKTYPE), + ("mooship-1.2.2/fifo", tarfile.FIFOTYPE), + ("mooship-1.2.2/VERSION", tarfile.REGTYPE)): + with self.subTest(name=name, kind=kind): + member = tarfile.TarInfo(name) + member.type = kind + member.linkname = "/tmp/escape" + self.archive_with([(member, b"" if kind == tarfile.REGTYPE else None)]) + with self.assertRaises(ValueError): + self.perform() + self.assertFalse(self.destination.exists()) + self.assertFalse((self.root / "escape").exists()) + + def test_rejects_mismatched_version_missing_files_and_expansion_limit(self): + self.archive_with(version="9.9.9") + with self.assertRaisesRegex(ValueError, "VERSION"): + self.perform() + with tarfile.open(self.archive, "w:gz"): + pass + with self.assertRaisesRegex(ValueError, "missing"): + self.perform() + self.archive_with() + with patch.object(install, "MAX_EXPANDED", 1): + with self.assertRaisesRegex(ValueError, "size limit"): + self.perform() + self.assertFalse(self.destination.exists()) + + @unittest.skipUnless(os.name == "posix", "POSIX symlinks") + def test_refuses_dangling_symlink_and_symlink_parent(self): + self.destination.symlink_to(self.root / "missing") + with self.assertRaisesRegex(ValueError, "symbolic"): + install.installation_path(str(self.destination)) + self.destination.unlink() + self.destination.symlink_to(self.root, target_is_directory=True) + with self.assertRaisesRegex(ValueError, "symbolic"): + install.installation_path(str(self.destination / "new")) + + def test_concurrent_install_cannot_replace_existing_destination(self): + self.archive_with() + original = install.unpack_release + + def concurrent(*args): + result = original(*args) + self.destination.mkdir() + (self.destination / "sentinel").write_text("keep") + return result + + with patch.object(install, "download", side_effect=self.fake_download), \ + patch.object(install, "unpack_release", side_effect=concurrent): + with self.assertRaises(FileExistsError): + install.install_native("1.2.2", str(self.destination)) + self.assertEqual((self.destination / "sentinel").read_text(), "keep") + + def test_bad_version_and_paths_are_rejected(self): + for value in ("latest", "../1.2.2", "1.2", "v1.2.2", "1.2.2;touch bad"): + with self.subTest(value=value), self.assertRaises(ValueError): + install.release_version(value) + for value in ("relative", "/tmp/../opt/mooship", "/tmp/a,b", "/tmp/a\nb"): + with self.subTest(value=value), self.assertRaises(ValueError): + install.absolute_path(value) + + def test_failed_copy_leaves_a_marker_and_existing_path_is_refused(self): + self.archive_with() + with patch.object(install, "download", side_effect=self.fake_download), \ + patch.object(install.shutil, "copytree", side_effect=OSError("disk full")): + with self.assertRaises(OSError): + install.install_native("1.2.2", str(self.destination)) + self.assertTrue((self.destination / ".mooship-installing").is_file()) + with self.assertRaisesRegex(ValueError, "already exists"): + self.perform() + + def test_http_download_and_downgrade_redirect_are_rejected(self): + with self.assertRaises(ValueError): + install.download("http://example.test/file", self.root / "download") + with self.assertRaises(ValueError): + install.HTTPSRedirects().redirect_request(None, None, 302, "", {}, "http://example.test/file") + + def test_download_size_limit_leaves_no_installation(self): + response = io.BytesIO(b"too large") + with patch.object(install, "build_opener") as opener: + opener.return_value.open.return_value = response + with self.assertRaisesRegex(ValueError, "size or time"): + install.download("https://example.test/file", self.root / "download", limit=2) + self.assertFalse(self.destination.exists()) + + +class CLITests(unittest.TestCase): + def call(self, arguments): + output, errors = io.StringIO(), io.StringIO() + with redirect_stdout(output), redirect_stderr(errors): + status = cli.main(arguments) + return status, output.getvalue(), errors.getvalue() + + def test_install_preview_never_downloads_or_writes(self): + with tempfile.TemporaryDirectory() as directory: + destination = Path(directory) / "new" + with patch.object(cli, "install_native") as install_native: + status, output, _ = self.call(["install", "--directory", str(destination), "--dry-run"]) + self.assertEqual(status, 0) + self.assertIn("Would download and verify Mooship", output) + install_native.assert_not_called() + self.assertFalse(destination.exists()) + + def test_native_forwarding_keeps_metacharacters_separate_and_private(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "VERSION").write_text("1.2.0", encoding="ascii") + for command, filename in cli.COMMANDS.items(): + (root / filename).write_text("#!/bin/bash\n", encoding="ascii") + with self.subTest(command=command), patch.object(cli.sys, "platform", "linux"), \ + patch.object(cli.shutil, "which", return_value="/bin/bash"), \ + patch.object(cli.os, "execvp") as execute: + status, output, errors = self.call([ + command, "--directory", directory, "--", "--slug", "demo", "secret;$(touch nope)", + ]) + self.assertEqual(status, 0) + execute.assert_called_once_with("bash", [ + "bash", str(root / filename), "--slug", "demo", "secret;$(touch nope)", + ]) + self.assertEqual(output + errors, "") + + def test_partial_and_missing_toolkits_are_refused(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.assertEqual(self.call(["doctor", "--directory", directory])[0], 1) + (root / "VERSION").write_text("1.2.0", encoding="ascii") + (root / ".mooship-installing").touch() + status, _, error = self.call(["doctor", "--directory", directory]) + self.assertEqual(status, 1) + self.assertIn("incomplete", error) + + def test_toolkit_version_reads_selected_release(self): + with tempfile.TemporaryDirectory() as directory: + (Path(directory) / "VERSION").write_text("1.1.2", encoding="ascii") + status, output, _ = self.call(["toolkit-version", "--directory", directory]) + self.assertEqual((status, output.strip()), (0, "1.1.2")) + self.assertEqual(self.call(["toolkit-version", "--directory", directory, "--", "unexpected"])[0], 1) + + def test_server_operations_explain_unsupported_platform(self): + with tempfile.TemporaryDirectory() as directory: + (Path(directory) / "VERSION").write_text("1.2.0", encoding="ascii") + with patch.object(cli.sys, "platform", "win32"): + status, _, error = self.call(["deploy", "--directory", directory]) + self.assertEqual(status, 1) + self.assertIn("Linux server over SSH", error) + + @unittest.skipUnless(sys.platform == "linux", "Requires Linux Bash and process signals") + def test_real_backend_exit_status_is_preserved(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "VERSION").write_text("1.2.0", encoding="ascii") + (root / "doctor.sh").write_text("exit 2\n", encoding="ascii") + process = subprocess.run( + [sys.executable, "-m", "mooship", "doctor", "--directory", directory], + env={**os.environ, "PYTHONPATH": str(Path(__file__).resolve().parents[1] / "src")}, + capture_output=True, check=False, + ) + self.assertEqual(process.returncode, 2) diff --git a/tests/test_release_publication.py b/tests/test_release_publication.py new file mode 100644 index 0000000..5f95cde --- /dev/null +++ b/tests/test_release_publication.py @@ -0,0 +1,86 @@ +"""Check release gates and native archive contents without registry credentials.""" + +import hashlib +import importlib.util +from pathlib import Path +import subprocess +import tarfile +import tempfile +import unittest +from urllib.error import HTTPError +from unittest.mock import patch + + +def load_script(name): + path = Path(__file__).resolve().parents[1] / ".github" / "scripts" / (name + ".py") + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +release = load_script("check_release") +archive = load_script("build_release") +image_tags = load_script("check_image_tags") + + +class ReleaseGateTests(unittest.TestCase): + def test_existing_tags_and_unavailable_registry_cannot_authorize_overwrite(self): + tag = "1.2.0-moodle4.5.15-php8.3" + with patch.object(image_tags, "urlopen"): + with self.assertRaisesRegex(ValueError, "already exists"): + image_tags.require_new_tag(tag) + with patch.object(image_tags, "urlopen", side_effect=HTTPError("https://example.test", 404, "missing", {}, None)): + image_tags.require_new_tag(tag) + with patch.object(image_tags, "urlopen", side_effect=HTTPError("https://example.test", 503, "unavailable", {}, None)): + with self.assertRaises(HTTPError): + image_tags.require_new_tag(tag) + + def test_tag_must_match_the_exact_version(self): + release.validate_tag("v1.2.0", "1.2.0") + for value in ("v1.1.2", "1.2.0", "v1.2", "v1.2.0/../main", "v1.2.0\n"): + with self.subTest(tag=value), self.assertRaises(ValueError): + release.validate_tag(value, "1.2.0") + + def test_ci_requires_this_commit_and_the_latest_run_to_succeed(self): + runs = [{"id": index, "name": name, "head_sha": "reviewed", "status": "completed", "conclusion": "success"} + for index, name in enumerate(sorted(release.REQUIRED))] + release.check_runs(runs, "reviewed") + with self.assertRaises(ValueError): + release.check_runs(runs, "different") + for status, conclusion in (("in_progress", None), ("completed", "failure"), ("completed", "skipped")): + latest = {**runs[0], "id": 999, "status": status, "conclusion": conclusion} + with self.subTest(status=status, conclusion=conclusion), self.assertRaises(ValueError): + release.check_runs([*runs, latest], "reviewed") + + def test_native_archive_is_reproducible_and_excludes_nonruntime_data(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + names = sorted(archive.FILES | {"lib/common.sh", "moodle/.config/php/mooship.ini", "README.secret"}) + for name in names: + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"1.2.0\n" if name == "VERSION" else b"fixture\r\n") + (root / ".env").write_text("SECRET=synthetic", encoding="ascii") + inventory = subprocess.CompletedProcess([], 0, stdout=("\0".join(names) + "\0").encode()) + with patch.object(archive.subprocess, "run", return_value=inventory): + first = archive.build(root, root / "first") + second = archive.build(root, root / "second") + self.assertEqual(first.read_bytes(), second.read_bytes()) + digest = hashlib.sha256(first.read_bytes()).hexdigest() + self.assertEqual(first.with_name(first.name + ".sha256").read_text().strip(), f"{digest} {first.name}") + with tarfile.open(first) as source: + self.assertNotIn("mooship-1.2.0/.env", source.getnames()) + self.assertNotIn("mooship-1.2.0/README.secret", source.getnames()) + script = source.getmember("mooship-1.2.0/deploy-moodle.sh") + self.assertEqual(script.mode, 0o755) + self.assertEqual(source.extractfile(script).read(), b"fixture\n") + + def test_missing_native_files_refuse_archive_creation(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "VERSION").write_text("1.2.0", encoding="ascii") + with patch.object(archive.subprocess, "run", return_value=subprocess.CompletedProcess([], 0, stdout=b"VERSION\0")): + with self.assertRaisesRegex(ValueError, "missing"): + archive.build(root, root / "output") + self.assertFalse((root / "output").exists()) diff --git a/tests/test_runtime_image.py b/tests/test_runtime_image.py new file mode 100644 index 0000000..e17c49f --- /dev/null +++ b/tests/test_runtime_image.py @@ -0,0 +1,219 @@ +"""Opt-in installation and security checks for the standalone runtime images.""" + +import hashlib +import json +import os +from pathlib import Path +import subprocess +import tempfile +import time +import unittest +import uuid + + +APP_IMAGE = os.environ.get("MOOSHIP_APP_IMAGE", "shellharbor/mooship:dev-moodle4.5.15-php8.3") +WEB_IMAGE = os.environ.get("MOOSHIP_WEB_IMAGE", "shellharbor/mooship:dev-moodle4.5.15-nginx") + + +def docker(*args, timeout=120): + return subprocess.run( + ["docker", *args], capture_output=True, text=True, encoding="utf-8", + errors="replace", timeout=timeout, check=False, + ) + + +@unittest.skipUnless(os.environ.get("MOOSHIP_IMAGE_TESTS") == "1", "Set MOOSHIP_IMAGE_TESTS=1") +class RuntimeImageTests(unittest.TestCase): + def setUp(self): + for image in (APP_IMAGE, WEB_IMAGE, "postgres:17-alpine", "mysql:8.4"): + self.assert_ok(docker("image", "inspect", image)) + self.prefix = "mooship-image-" + uuid.uuid4().hex[:12] + self.containers = [] + self.volumes = [] + self.network = self.prefix + self.label = "io.mooship.test=" + self.prefix + self.secrets = tempfile.TemporaryDirectory(prefix=self.prefix) + # The fixture directory is bind-mounted as a whole; UID 1000 must traverse it on Linux. + Path(self.secrets.name).chmod(0o755) + self.addCleanup(self.secrets.cleanup) + self.addCleanup(self.cleanup_docker) + self.assert_ok(docker("network", "create", "--label", self.label, self.network)) + # MySQL's upstream initializer interpolates this into a SQL string. + # Exercise quotes/backslashes in our secret reader and admin installer separately. + self.password = "Test_$&-" + uuid.uuid4().hex + self.admin_password = "Test_Aa9!'$\\&-" + uuid.uuid4().hex + for name, value in (("db_password", self.password), ("admin_password", self.admin_password)): + path = Path(self.secrets.name) / name + path.write_text(value + "\n", encoding="utf-8") + path.chmod(0o644) + + def cleanup_docker(self): + for name in reversed(self.containers): + docker("rm", "--force", "--volumes", name) + for name in self.volumes: + docker("volume", "rm", name) + docker("network", "rm", self.network) + + def assert_ok(self, result): + self.assertEqual(result.returncode, 0, result.stdout[-6000:] + result.stderr[-6000:]) + return result.stdout + + def run_container(self, role, image, args=(), options=(), detached=False): + name = self.prefix + "-" + role + self.containers.append(name) + command = ["run", "--name", name, "--label", self.label, "--network", self.network] + if detached: + command.append("--detach") + return name, docker(*command, *options, image, *args, timeout=600) + + def runtime_options(self, volume, engine="pgsql"): + return [ + "--read-only", "--cap-drop=ALL", "--security-opt=no-new-privileges:true", + "--tmpfs", "/tmp:uid=1000,gid=1000,mode=1770", "--volume", volume + ":/var/moodledata", + "--mount", "type=bind,source=" + self.secrets.name + ",target=/run/secrets,readonly", + "--env", "MOODLE_DB_TYPE=" + engine, "--env", "MOODLE_DB_HOST=db", + "--env", "MOODLE_DB_PASSWORD_FILE=/run/secrets/db_password", + "--env", "MOODLE_WWWROOT=http://web:8080", + ] + + def test_image_contract_and_invalid_configuration(self): + self.password = "Test_'$\\&-" + uuid.uuid4().hex + (Path(self.secrets.name) / "db_password").write_text(self.password + "\n", encoding="utf-8") + for image in (APP_IMAGE, WEB_IMAGE): + metadata = json.loads(self.assert_ok(docker("image", "inspect", image)))[0] + self.assertEqual(metadata["Config"]["User"], "1000:1000") + self.assertEqual(metadata["Config"]["StopSignal"], "SIGQUIT") + self.assertIn("Healthcheck", metadata["Config"]) + _, result = self.run_container("help", APP_IMAGE, ["--help"]) + self.assertIn("Mooship Moodle runtime", self.assert_ok(result)) + volume = self.prefix + "-data" + self.volumes.append(volume) + self.assert_ok(docker("volume", "create", "--label", self.label, volume)) + options = self.runtime_options(volume) + code = ( + 'require "/opt/mooship/environment.php"; ' + 'echo json_encode(["uid" => posix_geteuid(), "writable" => is_writable("/opt/moodle"), ' + '"password" => hash("sha256", \\Mooship\\settings()["password"]), ' + '"modules" => get_loaded_extensions()]);' + ) + _, result = self.run_container("contract", APP_IMAGE, ["php", "-r", code], options) + contract = json.loads(self.assert_ok(result)) + self.assertEqual(contract["uid"], 1000) + self.assertFalse(contract["writable"]) + self.assertEqual(contract["password"], hashlib.sha256(self.password.encode()).hexdigest()) + self.assertTrue({"sodium", "intl", "mysqli", "pgsql", "redis", "soap", "gd", "zip"} + .issubset(contract["modules"])) + _, result = self.run_container("conflict", APP_IMAGE, ["fpm"], [ + *options, "--env", "MOODLE_DB_PASSWORD=" + self.password, + ]) + self.assertNotEqual(result.returncode, 0) + self.assertIn("not both", result.stderr) + self.assertNotIn(self.password, result.stderr + result.stdout) + _, result = self.run_container("bad-url", APP_IMAGE, ["fpm"], [ + *options, "--env", "MOODLE_WWWROOT=http://web:8080/subdirectory", + ]) + self.assertNotEqual(result.returncode, 0) + self.assertIn("MOODLE_WWWROOT", result.stderr) + _, result = self.run_container("bad-workers", APP_IMAGE, ["fpm"], [ + *options, "--env", "PHP_FPM_MAX_CHILDREN=1", + ]) + self.assertNotEqual(result.returncode, 0) + self.assertIn("PHP_FPM_MAX_CHILDREN", result.stderr) + + def check_database_installation(self, engine): + volume = self.prefix + "-data" + self.volumes.append(volume) + self.assert_ok(docker("volume", "create", "--label", self.label, volume)) + if engine == "pgsql": + image = "postgres:17-alpine" + environment = ["POSTGRES_USER=moodle", "POSTGRES_DB=moodle", "POSTGRES_PASSWORD=" + self.password] + probe = ["pg_isready", "-U", "moodle", "-d", "moodle", "-h", "127.0.0.1"] + query = ["psql", "-U", "moodle", "-d", "moodle", "-c"] + else: + image = "mysql:8.4" + environment = ["MYSQL_USER=moodle", "MYSQL_DATABASE=moodle", "MYSQL_PASSWORD=" + self.password, + "MYSQL_ROOT_PASSWORD=" + uuid.uuid4().hex] + probe = ["sh", "-c", 'MYSQL_PWD="$MYSQL_PASSWORD" mysql -umoodle -h127.0.0.1 -Dmoodle -e "SELECT 1"'] + query = ["sh", "-c", 'MYSQL_PWD="$MYSQL_PASSWORD" mysql -umoodle -h127.0.0.1 -Dmoodle -e "$1"', "sh"] + db_options = ["--network-alias", "db"] + for setting in environment: + db_options.extend(["--env", setting]) + db, result = self.run_container("db", image, options=db_options, detached=True) + self.assert_ok(result) + deadline = time.monotonic() + 180 + while docker("exec", db, *probe).returncode != 0: + state = json.loads(self.assert_ok(docker("inspect", db)))[0]["State"] + if not state["Running"]: + self.fail("Database exited during startup: " + docker("logs", db).stderr[-3000:]) + if time.monotonic() > deadline: + self.fail("Database did not start: " + docker("logs", db).stdout[-3000:]) + time.sleep(2) + options = self.runtime_options(volume, engine) + _, result = self.run_container("before-install", APP_IMAGE, ["fpm"], options) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Install the database first", result.stderr) + install_options = [*options, "--env", "MOODLE_AGREE_LICENSE=true", "--env", + "MOODLE_ADMIN_EMAIL=admin@example.org", "--env", + "MOODLE_ADMIN_PASSWORD_FILE=/run/secrets/admin_password"] + install, result = self.run_container("install", APP_IMAGE, ["install"], install_options) + self.assert_ok(result) + self.assertNotIn(self.admin_password, result.stdout + result.stderr) + inspect = self.assert_ok(docker("inspect", install)) + self.assertNotIn(self.admin_password, inspect) + _, result = self.run_container("install-again", APP_IMAGE, ["install"], install_options) + self.assertNotEqual(result.returncode, 0) + self.assertIn("empty database", result.stderr) + app, result = self.run_container("app", APP_IMAGE, options=[*options, "--network-alias", "app"], detached=True) + self.assert_ok(result) + deadline = time.monotonic() + 30 + while docker("exec", app, "mooship-healthcheck").returncode != 0: + if time.monotonic() > deadline: + self.fail("FPM did not start: " + docker("logs", app).stderr[-3000:]) + time.sleep(1) + web, result = self.run_container("web", WEB_IMAGE, options=[ + "--network-alias", "web", "--read-only", "--cap-drop=ALL", + "--security-opt=no-new-privileges:true", "--tmpfs", "/tmp:uid=1000,gid=1000,mode=1770", + "--volume", volume + ":/var/moodledata:ro", + ], detached=True) + self.assert_ok(result) + deadline = time.monotonic() + 30 + while docker("exec", web, "wget", "-qO-", "http://web:8080/healthz").returncode != 0: + if time.monotonic() > deadline: + self.fail("nginx did not start: " + docker("logs", web).stderr[-3000:]) + time.sleep(1) + self.assertIn("ready", self.assert_ok(docker("exec", web, "wget", "-qO-", "http://web:8080/readyz"))) + login = self.assert_ok(docker("exec", web, "wget", "-qO-", "http://web:8080/login/index.php")) + self.assertIn('name="username"', login) + for path in ("/config.php", "/config.php/test", "/admin/cli/install_database.php", "/dataroot/test", "/.git/config"): + result = docker("exec", web, "wget", "-S", "-O", "/dev/null", "http://web:8080" + path) + self.assertNotEqual(result.returncode, 0, path) + self.assertIn("404", result.stderr, path) + _, result = self.run_container("cron", APP_IMAGE, ["cron-once"], options) + self.assert_ok(result) + self.assert_ok(docker("stop", "--time", "30", db)) + self.assert_ok(docker("exec", app, "mooship-healthcheck")) + result = docker("exec", web, "wget", "-S", "-O", "/dev/null", "http://web:8080/readyz") + self.assertIn("503", result.stderr) + self.assert_ok(docker("start", db)) + deadline = time.monotonic() + 90 + while docker("exec", app, "mooship", "check").returncode != 0: + self.assertLess(time.monotonic(), deadline, "Readiness did not recover") + time.sleep(2) + self.assert_ok(docker("exec", db, *query, "UPDATE mdl_config SET value='0' WHERE name='version'")) + _, result = self.run_container("wrong-version", APP_IMAGE, ["fpm"], options) + self.assertNotEqual(result.returncode, 0) + for name in (web, app): + self.assert_ok(docker("stop", "--time", "30", name)) + state = json.loads(self.assert_ok(docker("inspect", name)))[0]["State"] + self.assertEqual(state["ExitCode"], 0, state) + self.assertFalse(state["OOMKilled"]) + + def test_postgresql_installation(self): + self.check_database_installation("pgsql") + + def test_mysql_installation(self): + self.check_database_installation("mysqli") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_scripts.py b/tests/test_scripts.py index 2668241..603af66 100644 --- a/tests/test_scripts.py +++ b/tests/test_scripts.py @@ -92,6 +92,7 @@ def test_install_failure_is_fatal(self): self.fixture() result = self.bash('''PROJECT_DIR="$WWW_DIR/test"; DOMAIN=test.example docker() { if [[ "$*" == *--help* ]]; then echo help; return 0; fi; return 42; } +mooship_check_database() { return 3; } install_moodle_database echo SHOULD_NOT_RUN ''') @@ -217,7 +218,7 @@ def test_ssl_failure_is_fatal(self): def test_failed_application_does_not_publish_proxy_configuration(self): # Exercise main's ordering with side effects replaced by trace markers. result = self.bash(''' -for function in check_root parse_args mooship_require_new_project assign_ports resolve_moodle_version select_php_version_for_moodle select_resource_profile install_docker mooship_require_docker mooship_require_unused_resources ensure_www_dir init_nginxproxy ensure_logrotate create_dhparam create_project comment_ssl_blocks create_native_database build_and_start_project install_moodle configure_moodle_config; do +for function in check_root parse_args mooship_require_new_project assign_ports resolve_moodle_version select_php_version_for_moodle select_resource_profile install_docker mooship_require_docker mooship_require_unused_resources ensure_www_dir init_nginxproxy ensure_logrotate create_dhparam create_project comment_ssl_blocks create_native_database build_and_start_project install_moodle install_moodle_dependencies configure_moodle_config mooship_check_database reload_project_nginx; do eval "$function() { :; }" done install_moodle_database() { return 42; } @@ -453,6 +454,7 @@ def restore_protocol(self, scheme): create_native_database() { :; }; restore_database() { :; }; restart_nginxproxy() { :; } ensure_project_docker_resources() { cd "$PROJECT_DIR"; } build_and_start_project() { printf 'MODE=%s\\n' "$(site_scheme)"; } +start_moodle_cron() { :; } docker() { printf '%s\\n' "$*" >> "$TEST_DIR/docker.log"; } restore_main --slug test --archive "$TEST_DIR/site.tar.gz" <<< test ''', source="restore.sh") diff --git a/tests/test_web.py b/tests/test_web.py index 0964fdf..f35eb28 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -1,6 +1,9 @@ import importlib.util from pathlib import Path import unittest +import subprocess +import sys +import tempfile ROOT = Path(__file__).resolve().parents[1] SPEC = importlib.util.spec_from_file_location("mooship_web", ROOT / "lib/web.py") @@ -73,6 +76,19 @@ def test_commented_tls_template_can_be_restored_as_http(self): result = WEB.http_configuration("\n".join(lines)) self.assertEqual(result, WEB.http_configuration(self.template)) + def test_public_root_cli_preserves_mounted_file_inode_and_protocol(self): + for template in (self.template, WEB.http_configuration(self.template)): + with self.subTest(http_only=template != self.template), tempfile.TemporaryDirectory() as folder: + path = Path(folder) / 'site.conf' + path.write_text(template, encoding='utf-8') + inode = path.stat().st_ino + expected = template.replace('root /var/www/html;', 'root /var/www/html/public;') + for _ in range(2): + subprocess.run([sys.executable, '-B', str(ROOT / 'lib/web.py'), '--public-root', str(path)], + check=True, capture_output=True, timeout=10) + self.assertEqual(path.stat().st_ino, inode) + self.assertEqual(path.read_text(encoding='utf-8'), expected) + def test_unsupported_or_unterminated_configuration_is_rejected(self): for template in ("server {\n listen 80;\n}\n", self.template[:-2]): with self.subTest(template=template):