Tokenizer fallback and robustness fixes

.dockerignore

@@ -0,0 +1 @@
+models/

.github/workflows/docker-publish.yml

@@ -22,7 +22,7 @@ env:
 
 jobs:
   build:
-    runs-on: [self-hosted, Linux, ARM64]
+    runs-on: [ubuntu-24.04-arm]
     permissions:
       contents: read
       packages: write
@@ -32,29 +32,36 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
+
+      - name: Normalize image name to lowercase
+        run: echo "IMAGE_NAME_LC=${IMAGE_NAME,,}" >> $GITHUB_ENV
+        env:
+          IMAGE_NAME: ${{ env.IMAGE_NAME }}
 
       # Install the cosign tool except on PR
       # https://github.com/sigstore/cosign-installer
       - name: Install cosign
         if: github.event_name != 'pull_request'
-        uses: sigstore/cosign-installer@59acb6260d9c0ba8f4a2f9d9b48431a222b68e20 #v3.5.0
-        with:
-          cosign-release: 'v2.2.4'
+        uses: sigstore/cosign-installer@v4.1.1
 
       # Set up BuildKit Docker container builder to be able to build
       # multi-platform images and export cache
       # https://github.com/docker/setup-buildx-action
+      #
+      # NOTE: Do NOT set `platforms: linux/arm64` here — on a native
+      # ubuntu-24.04-arm runner this forces QEMU-backed cross-compilation
+      # even though the runner is natively arm64, causing the docker-container
+      # builder to pull and cache amd64 base layers and contaminate subsequent
+      # arm64 builds.  Omitting the flag builds for the runner's native arch.
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-        with:
-          platforms: linux/arm64
+        uses: docker/setup-buildx-action@v4
 
       # Login against a Docker registry except on PR
       # https://github.com/docker/login-action
       - name: Log into registry ${{ env.REGISTRY }}
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
+        uses: docker/login-action@v4
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -64,23 +71,26 @@ jobs:
       # https://github.com/docker/metadata-action
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0
+        uses: docker/metadata-action@v6
         with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}
 
-      # Build and push Docker image with Buildx (don't push on PR)
+      # Build and push Docker image with Buildx
       # https://github.com/docker/build-push-action
+      #
+      # NOTE: Do NOT set `platforms: linux/arm64` here — same reason as above.
+      # The cache key uses the -arm64 suffix to avoid reusing the stale amd64
+      # cache that existed before the runner switch to ubuntu-24.04-arm.
       - name: Build and push Docker image
         id: build-and-push
-        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
+        uses: docker/build-push-action@v7
         with:
           context: .
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-          platforms: linux/arm64
+          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:buildcache-arm64
+          cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:buildcache-arm64,mode=max
 
       # Sign the resulting Docker image digest except on PRs.
       # This will only write to the public Rekor transparency log when the Docker
@@ -95,4 +105,4 @@ jobs:
           DIGEST: ${{ steps.build-and-push.outputs.digest }}
         # This step uses the identity token to provision an ephemeral certificate
         # against the sigstore community Fulcio instance.
-        run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST}
+        run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST}

.gitignore

@@ -1,6 +1,29 @@
-models/*/*.rkllm
-models/*/*
+models/**/*
 src/__pycache__
 *.log
 config/config.env
+build
 tests
+src/rkllama.egg-info/*
+
+# macOS system files
+.DS_Store
+
+# Python bytecode
+__pycache__/
+**/__pycache__/
+*.py[cod]
+*$py.class
+
+# Virtual environments
+venv/
+env/
+ENV/
+
+# IDE files
+.vscode/
+.idea/
+
+# Converter directory
+# Uncomment if you want to ignore it
+# converter/

Dockerfile

@@ -1,18 +1,48 @@
-FROM python:slim
+FROM ubuntu:24.04
+
+# RK3588 (Cortex-A76/A55) is ARMv8.2-A — it does NOT support Pointer
+# Authentication (PAC, ARMv8.3-A).  Debian Trixie ARM64 packages
+# (python:3.12-slim) are compiled with PAC guards; the kernel returns
+# ENOEXEC for those binaries on RK3588, causing "exec format error" at
+# container startup.  Ubuntu 24.04 ARM64 packages do NOT use PAC and
+# run correctly on ARMv8.2-A hardware.
+ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends libgomp1 wget curl sudo \
+    && apt-get install -y --no-install-recommends \
+       python3.12 python3.12-dev python3-pip python3.12-venv \
+       libgomp1 wget curl sudo git build-essential \
+       ffmpeg libsm6 libxext6 \
     && rm -rf /var/cache/apt/archives /var/lib/apt/lists/*
 
+# Use a virtual environment to avoid Ubuntu 24.04's PEP 668
+# "externally-managed-environment" restriction.
+RUN python3 -m venv /opt/venv
+ENV PATH="/opt/venv/bin:$PATH"
+
+RUN pip install --no-cache-dir --upgrade pip \
+    && pip install --no-cache-dir "setuptools<82.0.0"
+
 WORKDIR /opt/rkllama
 
-COPY ./lib /opt/rkllama/lib
+# Copy RKLLM runtime library explicitly
+COPY ./src/rkllama/lib/librkllmrt.so /usr/lib/
+RUN chmod 755 /usr/lib/librkllmrt.so && ldconfig
+
+# Copy RKNN runtime library explicitly
+COPY ./src/rkllama/lib/librknnrt.so /usr/lib/
+RUN chmod 755 /usr/lib/librknnrt.so && ldconfig
+
+# Copy the source and other resources of the RKllama project
 COPY ./src /opt/rkllama/src
-COPY requirements.txt README.md LICENSE *.sh *.py /opt/rkllama/
-RUN chmod +x setup.sh && ./setup.sh --no-conda
+RUN mkdir /opt/rkllama/models
+COPY README.md LICENSE pyproject.toml /opt/rkllama/
+
+# Install RKllama project
+RUN pip install --no-cache-dir .
 
 EXPOSE 8080
 
-#CMD ["/usr/local/bin/rkllama", "serve", "port"] if you want to change the port.
-#example: CMD ["/usr/local/bin/rkllama", "serve", "3000"]
-CMD ["/usr/local/bin/rkllama", "serve"]
+# If you want to change the port see the
+# documentation/configuration.md for the INI file settings.
+CMD ["rkllama_server", "--models", "/opt/rkllama/models"]