taskfile.dist.yml

# Reference: https://taskfile.dev
version: "3"

vars:
  GREETING: |
    \033[1;36mMLOX Taskfile (Developer)\033[0m

    MLOX helps you stand up opinionated MLOps infrastructure (MLflow, Feast,
    Milvus, telemetry) on your own servers and in hybrid setups. This Taskfile acts as a
    lightweight developer CLI to keep common actions consistent and repeatable.

    Key tasks
    - deps:lock             : compile base requirements from pyproject.toml
    - deps:lock:all|dev|gcp : compile extras to requirements-<extra>.txt
    - deps:install:tools    : install tools like Multipass for integration tests
    - tests:integration:run     : run integration tests (requires Multipass)
    - tests:integration:k8s     : run Kubernetes integration tests (requires Multipass/k3s)
    - tests:integration:cleanup : clean up test Multipass VMs
    - docker:build        : build local Docker image
    - project:new         : create an encrypted .mlox project
    - project:migrate     : copy a legacy project into SQLCipher

    Get started by following these steps:
    1) Install Task: https://taskfile.dev/installation
    2) Install Tools:  task deps:install:tools
    3) Run integration tests: task tests:integration:run
    4) Clean up test VMs: task tests:integration:cleanup

tasks:
  default:
    cmds:
      - printf "%b\n" "{{.GREETING}}"
    silent: true

  dev:lock:*:
    desc: Generate requirements.txt from extras (dev, all, gcp) in pyproject.toml
    vars:
      EXTRA: "{{index .MATCH 0}}"
    cmds:
      - '{{.PYTHON | default "python"}} -m pip install --upgrade pip-tools'
      - '{{.PYTHON | default "python"}} -m piptools compile pyproject.toml --extra {{.EXTRA}} --output-file requirements.gen.{{.EXTRA}}.txt'

  deps:install:tools:
    desc: Install tools like Multipass for integration tests
    cmds:
      - |
        printf '\n\033[1;36m▶ Installing development tools...\033[0m\n'
        if [[ "$OSTYPE" == "darwin"* ]]; then \
          task vm:install:macos; \
        elif [[ "$OSTYPE" == "linux-gnu"* ]]; then \
          task vm:install:linux; \
        else \
          printf '\n\033[1;33m! Unsupported platform: %s\033[0m\n' "$OSTYPE"; \
          printf 'Please install Multipass manually: https://multipass.run/docs/installing\n\n'; \
          exit 1; \
        fi

  dev:lint:
    silent: true
    desc: Lint Python sources with flake8
    cmds:
      - |
        if ! {{.PYTHON | default "python"}} -c "import importlib.util,sys; sys.exit(0 if importlib.util.find_spec('flake8') else 1)"; then \
          printf '\n\033[1;31m✖ flake8 is not installed.\033[0m\n'; \
          printf 'Install dev deps first, e.g.: \033[1m{{.PYTHON | default "python"}} -m pip install -e .[dev]\033[0m\n\n'; \
          exit 1; \
        fi
      - '{{.PYTHON | default "python"}} -m flake8 mlox tests --statistics --extend-ignore=E203,W503'

  dev:env:create:
    desc: Create a Conda dev environment with selected Python and install .[dev]
    vars:
      NAME: '{{.NAME | default "mlox-dev"}}'
      PY: '{{.PY | default "3.12.5"}}'
    cmds:
      - |
        if ! command -v conda >/dev/null 2>&1; then \
          printf '\n\033[1;31m✖ Conda not found.\033[0m\n'; \
          printf 'Please install Miniconda/Mamba and ensure `conda` is on PATH.\n\n'; \
          exit 1; \
        fi
      - |
        set -euo pipefail
        printf '\n\033[1;36m▶ Creating env\033[0m %s with Python %s\n' '{{.NAME}}' '{{.PY}}'
        conda create -y -n '{{.NAME}}' python='{{.PY}}'
        printf '\n\033[1;36m▶ Upgrading pip\033[0m in env %s\n' '{{.NAME}}'
        conda run -n '{{.NAME}}' python -m pip install --upgrade pip
        printf '\n\033[1;36m▶ Installing project (.[dev])\033[0m into env %s\n' '{{.NAME}}'
        conda run -n '{{.NAME}}' python -m pip install -e .[dev]
        printf '\n\033[1;36m▶ Installing multipass-sdk (needed for integration tests)\033[0from GitHub\033[0m into env %s\n' '{{.NAME}}'
        conda run -n '{{.NAME}}' python -m pip install "multipass-sdk @ git+https://github.com/okyanusoz/multipass-sdk.git@main#egg=multipass-sdk"
        printf '\n\033[1;36m▶ Verifying CLI import\033[0m\n'
        conda run -n '{{.NAME}}' python -c "import mlox.cli; print('mlox.cli OK')"
        printf '\n\033[1;32m✓ Environment ready:\033[0m activate with: \033[1mconda activate {{.NAME}}\033[0m\n'

  first:steps:
    aliases: [setup]
    desc: One-time setup — create dev env and show activation steps
    vars:
      NAME: '{{.NAME | default "mlox-dev"}}'
      PY: '{{.PY | default "3.12.5"}}'
    cmds:
      - task: dev:env:create
        vars:
          NAME: "{{.NAME}}"
          PY: "{{.PY}}"
      - |
        printf '\n\033[1;35m══════════════════════════════════════════════════════════\033[0m\n'
        printf '\033[1;35m   ✅ Next Steps\033[0m\n'
        printf '\033[1;35m══════════════════════════════════════════════════════════\033[0m\n'
        printf 'Activate your new environment:\n\n  \033[1mconda activate {{.NAME}}\033[0m\n\n'
        printf 'Run tests and linting (optional):\n\n  \033[1mtask tests:unit:run\033[0m\n  \033[1mtask dev:lint\033[0m\n\n'
        printf 'Start here anytime with:\n\n  \033[1mtask first:steps\033[0m  (alias: \033[1mtask setup\033[0m)\n\n'

  tests:unit:coverage:
    silent: true
    desc: Run unit tests with coverage (excludes integration)
    cmds:
      - |
        if ! {{.PYTHON | default "python"}} -c "import importlib.util,sys; sys.exit(0 if importlib.util.find_spec('pytest') and importlib.util.find_spec('coverage') else 1)"; then \
          printf '\n\033[1;31m✖ pytest/coverage not installed.\033[0m\n'; \
          printf 'Install dev deps, e.g.: \033[1m{{.PYTHON | default "python"}} -m pip install -e .[dev]\033[0m\n\n'; \
          exit 1; \
        fi
      - '{{.PYTHON | default "python"}} -m pytest tests/unit/ --cov=mlox --cov-report=term-missing'

  tests:unit:run:
    silent: true
    desc: Run unit tests (excludes integration)
    cmds:
      - |
        if ! {{.PYTHON | default "python"}} -c "import importlib.util,sys; sys.exit(0 if importlib.util.find_spec('pytest') else 1)"; then \
          printf '\n\033[1;31m✖ pytest not installed.\033[0m\n'; \
          printf 'Install dev deps, e.g.: \033[1m{{.PYTHON | default "python"}} -m pip install -e .[dev]\033[0m\n\n'; \
          exit 1; \
        fi
      - '{{.PYTHON | default "python"}} -m pytest tests/unit/ -v'

  tests:integration:run:
    silent: true
    desc: Run integration tests (requires Multipass)
    cmds:
      - |
        if ! command -v multipass >/dev/null 2>&1; then \
          printf '\n\033[1;31m✖ Multipass is not installed.\033[0m\n'; \
          printf 'Install it first. Run: \033[1mtask deps:install:tools\033[0m\n\n'; \
          exit 1; \
        fi
      - |
        if ! {{.PYTHON | default "python"}} -c "import importlib.util,sys; sys.exit(0 if importlib.util.find_spec('pytest') else 1)"; then \
          printf '\n\033[1;31m✖ pytest not installed.\033[0m\n'; \
          printf 'Install dev deps, e.g.: \033[1m{{.PYTHON | default "python"}} -m pip install -e .[dev]\033[0m\n\n'; \
          exit 1; \
        fi
      - '{{.PYTHON | default "python"}} -m pytest -m integration --run-integration'

  tests:integration:k8s:
    aliases: [tests:integration:kubernetes]
    silent: true
    desc: Run Kubernetes integration tests (requires Multipass/k3s)
    cmds:
      - |
        if ! command -v multipass >/dev/null 2>&1; then \
          printf '\n\033[1;31m✖ Multipass is not installed.\033[0m\n'; \
          printf 'Install it first. Run: \033[1mtask deps:install:tools\033[0m\n\n'; \
          exit 1; \
        fi
      - |
        if ! {{.PYTHON | default "python"}} -c "import importlib.util,sys; sys.exit(0 if importlib.util.find_spec('pytest') else 1)"; then \
          printf '\n\033[1;31m✖ pytest not installed.\033[0m\n'; \
          printf 'Install dev deps, e.g.: \033[1m{{.PYTHON | default "python"}} -m pip install -e .[dev]\033[0m\n\n'; \
          exit 1; \
        fi
      - '{{.PYTHON | default "python"}} -m pytest tests/integration -m "integration and kubernetes" --run-integration'

  tests:integration:service:
    silent: true
    vars:
      SERVICE: "{{.SERVICE}}"
      PDB: '{{.PDB | default "false"}}'
      VSCODE_DEBUG: '{{.VSCODE_DEBUG | default "false"}}'
    cmds:
      - |
        {{if eq .VSCODE_DEBUG "true"}}
        if ! {{.PYTHON | default "python"}} -c "import importlib.util,sys; sys.exit(0 if importlib.util.find_spec('debugpy') else 1)"; then
          printf '\n\033[1;31m✖ debugpy is not installed.\033[0m\n';

          printf 'Please install it first: \033[1m{{.PYTHON | default "python"}} -m pip install debugpy\033[0m\n\n';
          exit 1;
        fi
        printf '\n\033[1;36m▶ Starting debugger. Waiting for VS Code to attach on port 5678...\033[0m\n'
        {{.PYTHON | default "python"}} -m debugpy --listen 5678 --wait-for-client -m pytest tests/integration/test_service_{{.SERVICE}}.py -m integration --run-integration
        {{else}}
        pytest tests/integration/test_service_{{.SERVICE}}.py -m integration --run-integration{{if eq .PDB "true"}} --pdb{{end}}
        {{end}}
    desc: Run integration tests for a specific service only (e.g. mlflow, airflow, etc.)

  tests:integration:cleanup:
    silent: true
    cmds:
      - printf '\n\033[1;36m══════════ ▶ BEFORE CLEANUP (Multipass VMs) ◀ ══════════\033[0m\n'; multipass list || true
      - multipass list | grep mlox-test- | cut -d' ' -f1 | xargs multipass delete $1 --purge
      - printf '\n\033[1;32m══════════ ✓ AFTER CLEANUP (Multipass VMs) ✓ ══════════\033[0m\n'; multipass list || true
    desc: Remove all Multipass VMs used for integration testing

  docker:up:
    cmds:
      - docker compose up -d --build
      - docker compose ps
      - docker compose logs -f --tail=10
    desc: Start MLOX UI locally with Docker Compose

  docker:down:
    cmds:
      - docker compose down
    desc: Stop MLOX UI

  vm:start:
    desc: Start one or more Multipass VMs with cloud-init
    vars:
      COUNT: '{{.COUNT | default "1"}}'
      CPUS: '{{.CPUS | default "2"}}'
      MEM: '{{.MEM | default "4G"}}'
      DISK: '{{.DISK | default "10G"}}'
      PREFIX: '{{.PREFIX | default "mlox-test"}}'
      CLOUD_INIT: '{{.CLOUD_INIT | default "cloud-init.yaml"}}'
    cmds:
      - |
        if ! command -v multipass >/dev/null 2>&1; then \
          printf '\n\033[1;31m✖ Multipass is not installed.\033[0m\n'; \
          printf 'Install it first. On macOS run: \033[1mtask deps:install:tools\033[0m\n\n'; \
          exit 1; \
        fi
      - |
        if [ ! -f "{{.CLOUD_INIT}}" ]; then \
          printf '\n\033[1;31m✖ Cloud-init file not found:\033[0m %s\n' "{{.CLOUD_INIT}}"; \
          exit 1; \
        fi
      - |
        set -euo pipefail
        TS=$(date +%y%m%d%H%M%S)
        for i in $(seq 1 {{.COUNT}}); do \
          NAME="{{.PREFIX}}-${TS}-$i"; \
          printf '\n\033[1;36m▶ Launching VM:\033[0m %s (cpus=%s, mem=%s, disk=%s)\n' "$NAME" "{{.CPUS}}" "{{.MEM}}" "{{.DISK}}"; \
          multipass launch --name "$NAME" --cpus {{.CPUS}} --memory {{.MEM}} --disk {{.DISK}} --cloud-init "{{.CLOUD_INIT}}"; \
          multipass info "$NAME" || true; \
        done

  vm:install:macos:
    platforms:
      - darwin
    cmds:
      - brew install multipass
    desc: Install Multipass VM (macOS) via homebrew for local development and integration testing

  vm:install:linux:
    desc: Install Multipass on Linux using snap/apt/dnf (requires sudo)
    platforms:
      - linux
    cmds:
      - |
        if command -v multipass >/dev/null 2>&1; then \
          printf '\n\033[1;32m✓ Multipass already installed.\033[0m\n'; \
          multipass version || true; \
          exit 0; \
        fi
      - |
        set -euo pipefail
        if command -v snap >/dev/null 2>&1; then \
          printf '\n\033[1;36m▶ Installing Multipass via snap...\033[0m\n'; \
          sudo snap install multipass; \
        elif command -v apt-get >/dev/null 2>&1; then \
          printf '\n\033[1;36m▶ Installing Multipass via apt...\033[0m\n'; \
          sudo apt-get update; \
          sudo apt-get install -y multipass || { printf '\n\033[1;33m! Falling back to snap — please install snapd and rerun.\033[0m\n'; exit 1; }; \
        elif command -v dnf >/dev/null 2>&1; then \
          printf '\n\033[1;36m▶ Attempting install via dnf...\033[0m\n'; \
          sudo dnf install -y multipass || { printf '\n\033[1;31m✖ Could not install via dnf. Consider using snap.\033[0m\n'; exit 1; }; \
        else \
          printf '\n\033[1;31m✖ No supported package manager found (snap/apt/dnf).\033[0m\n'; \
          printf 'See installation docs: https://multipass.run/docs/installing-on-linux\n'; \
          exit 1; \
        fi
      - multipass version || true

  vm:purge:
    silent: true
    vars:
      PREFIX: '{{.PREFIX | default "mlox-test"}}'
    cmds:
      - printf '\n\033[1;36m══════════ ▶ BEFORE CLEANUP (Multipass VMs) ◀ ══════════\033[0m\n'; multipass list || true
      - multipass list | grep {{.PREFIX}} | cut -d' ' -f1 | xargs multipass delete $1 --purge
      - printf '\n\033[1;32m══════════ ✓ AFTER CLEANUP (Multipass VMs) ✓ ══════════\033[0m\n'; multipass list || true
    desc: Remove all Multipass VMs starting with the given prefix

  # -----------------------
  # UI helpers (local dev)
  # -----------------------
  ui:streamlit:
    desc: Start the Streamlit UI (local)
    cmds:
      - |
        if ! {{.PYTHON | default "python"}} -c "import importlib.util,sys; sys.exit(0 if importlib.util.find_spec('streamlit') else 1)"; then \
          printf '\n\033[1;31m✖ streamlit is not installed.\033[0m\n'; \
          printf 'Install dev deps first, e.g.: \033[1m{{.PYTHON | default "python"}} -m pip install -e .[dev]\033[0m\n\n'; \
          exit 1; \
        fi
      - "streamlit run mlox/app.py"

  ui:landing:
    desc: Start the astro landing page locally (dev server)
    cmds:
      - "cd website && npm install && npm run dev"

  ui:cli:
    desc: Start the Typer CLI app
    cmds:
      - "python -m mlox.cli {{.CLI_ARGS}}"

  ui:textual:terminal:
    desc: Start the Textual TUI in the terminal
    cmds:
      - |
        if ! {{.PYTHON | default "python"}} -c "import importlib.util,sys; sys.exit(0 if importlib.util.find_spec('textual') else 1)"; then \
          printf '\n\033[1;31m✖ textual is not installed.\033[0m\n'; \
          printf 'Install dev deps first, e.g.: \033[1m{{.PYTHON | default "python"}} -m pip install -e .[dev]\033[0m\n\n'; \
          exit 1; \
        fi
      - "export MLOX_TUI=true; textual run --dev mlox/tui/app.py"
      - "export MLOX_TUI=false"

  ui:textual:web:
    desc: Serve the Textual TUI over the web (dev server)
    env:
      MLOX_TUI: "true"
    cmds:
      - |
        if ! {{.PYTHON | default "python"}} -c "import importlib.util,sys; sys.exit(0 if importlib.util.find_spec('textual') else 1)"; then \
          printf '\n\033[1;31m✖ textual is not installed.\033[0m\n'; \
          printf 'Install dev deps first, e.g.: \033[1m{{.PYTHON | default "python"}} -m pip install -e .[dev]\033[0m\n\n'; \
          exit 1; \
        fi
      - "export MLOX_TUI=true; textual serve --dev mlox/tui/app.py"
      - "export MLOX_TUI=false"

  project:new:
    desc: Create an encrypted .mlox project (NAME=demo PASSWORD=...)
    requires:
      vars: [NAME, PASSWORD]
    cmds:
      - 'mlox project new "{{.NAME}}" --password "{{.PASSWORD}}"'

  project:migrate:
    desc: Copy a legacy .project into SQLCipher (SOURCE=... DESTINATION=... OLD_PASSWORD=... NEW_PASSWORD=...)
    requires:
      vars: [SOURCE, DESTINATION, OLD_PASSWORD, NEW_PASSWORD]
    cmds:
      - 'python scripts/migrate_legacy_project.py "{{.SOURCE}}" "{{.DESTINATION}}" --legacy-password "{{.OLD_PASSWORD}}" --new-password "{{.NEW_PASSWORD}}"'