Sitecustomize based auto-instrumentation (#7)

Author: piotrekno1

examples/django_gemma/demo.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ -z "${WILDEDGE_DSN:-}" ]]; then
+  echo 'Set WILDEDGE_DSN first, e.g. export WILDEDGE_DSN="https://<secret>@ingest.wildedge.dev/<key>"' >&2
+  exit 1
+fi
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "${SCRIPT_DIR}"
+
+uv sync
+
+uv run wildedge doctor --integrations gguf --hubs huggingface
+
+# wildedge run replaces this process (os.execle) so sitecustomize.py
+# auto-installs the runtime before Django loads, patching Llama.__init__
+# for automatic inference tracking.
+#
+# Server choice:
+#   macOS  — waitress (thread-pool, no fork). Metal is initialised once in the
+#             main process and shared safely across request threads.
+#   Linux  — gunicorn (multi-process fork). Requires llama-cpp-python built
+#             without Metal: CMAKE_ARGS="-DGGML_METAL=OFF" pip install llama-cpp-python
+#             Then: wildedge run ... -- gunicorn gemmaapp.wsgi:application --config gunicorn.conf.py
+if [[ "$(uname)" == "Darwin" ]]; then
+  uv run wildedge run \
+    --print-startup-report \
+    --integrations gguf \
+    --hubs huggingface \
+    -- waitress-serve --port=8100 gemmaapp.wsgi:application
+else
+  uv run wildedge run \
+    --print-startup-report \
+    --integrations gguf \
+    --hubs huggingface \
+    -- gunicorn gemmaapp.wsgi:application --config gunicorn.conf.py
+fi
+
+# Test with:
+#   curl -s -X POST http://localhost:8100/infer/ \
+#     -H "Content-Type: application/json" \
+#     -d '{"prompt": "What is on-device AI in one sentence?"}' | jq .

examples/django_gemma/gemmaapp/__init__.py

@@ -0,0 +1,6 @@
+SECRET_KEY = "dev-only-not-for-production"
+DEBUG = True
+ALLOWED_HOSTS = ["*"]
+INSTALLED_APPS = ["gemmaapp"]
+ROOT_URLCONF = "gemmaapp.urls"
+DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"

examples/django_gemma/gemmaapp/settings.py

@@ -0,0 +1,7 @@
+from django.urls import path
+
+from gemmaapp import views
+
+urlpatterns = [
+    path("infer/", views.infer),
+]

examples/django_gemma/gemmaapp/urls.py

@@ -0,0 +1,53 @@
+"""Gemma inference view.
+
+The Llama constructor is patched automatically by `wildedge run --integrations gguf`
+via sitecustomize.py — load/unload/inference events are tracked without any
+wildedge imports here.
+
+On macOS, waitress (thread-pool, no fork) is used as the WSGI server.
+Metal is initialised once at startup in the main process and shared safely
+across request threads. gunicorn (fork-based) requires llama-cpp-python built
+without Metal on macOS (CMAKE_ARGS="-DGGML_METAL=OFF").
+"""
+
+import json
+import os
+import threading
+
+from django.http import JsonResponse
+from django.views.decorators.csrf import csrf_exempt
+from django.views.decorators.http import require_POST
+from llama_cpp import Llama
+
+REPO = "bartowski/gemma-2-2b-it-GGUF"
+FILE = "gemma-2-2b-it-Q4_K_M.gguf"
+
+_llm = Llama.from_pretrained(
+    repo_id=REPO,
+    filename=FILE,
+    n_ctx=512,
+    n_gpu_layers=int(os.environ.get("GPU_LAYERS", "-1")),
+    verbose=False,
+)
+
+# Llama inference is not thread-safe on a single context — serialise requests.
+_llm_lock = threading.Lock()
+
+
+@csrf_exempt
+@require_POST
+def infer(request):
+    try:
+        body = json.loads(request.body)
+    except json.JSONDecodeError:
+        return JsonResponse({"error": "invalid JSON"}, status=400)
+
+    prompt = body.get("prompt", "").strip()
+    if not prompt:
+        return JsonResponse({"error": "prompt is required"}, status=400)
+
+    with _llm_lock:
+        result = _llm(prompt, max_tokens=256, temperature=0.7)
+
+    text = result["choices"][0]["text"].strip()
+    return JsonResponse({"response": text})

examples/django_gemma/gemmaapp/views.py

@@ -0,0 +1,7 @@
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "gemmaapp.settings")
+
+application = get_wsgi_application()

examples/django_gemma/gemmaapp/wsgi.py

@@ -0,0 +1,15 @@
+"""Gunicorn configuration — Linux only.
+
+On macOS use waitress instead (demo.sh selects automatically).
+Requires llama-cpp-python built without Metal:
+  CMAKE_ARGS="-DGGML_METAL=OFF" pip install llama-cpp-python --no-binary llama-cpp-python
+
+With CPU-only GGML, the model loaded via preload_app=True in the master is
+inherited safely by forked workers via copy-on-write.
+"""
+
+workers = 2
+bind = "0.0.0.0:8100"
+timeout = 120
+preload_app = True
+control_socket_disable = True

examples/django_gemma/gunicorn.conf.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+import os
+import sys
+
+if __name__ == "__main__":
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "gemmaapp.settings")
+    from django.core.management import execute_from_command_line
+
+    execute_from_command_line(sys.argv)

examples/django_gemma/manage.py

@@ -0,0 +1,15 @@
+[project]
+name = "wildedge-django-gemma"
+version = "0.1.0"
+requires-python = ">=3.10"
+dependencies = [
+    "wildedge-sdk",
+    "django",
+    "gunicorn",   # Linux (fork-based, CPU-only llama-cpp required on macOS)
+    "waitress",   # macOS (thread-based, no fork — works with Metal)
+    "llama-cpp-python",
+    "huggingface-hub",
+]
+
+[tool.uv.sources]
+wildedge-sdk = { path = "../..", editable = true }