Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 58 additions & 0 deletions graphstorm-processing/docker/0.5.0/emr-serverless/Dockerfile.cpu
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
ARG ARCH=x86_64
FROM public.ecr.aws/emr-serverless/hive/emr-7.9.0:20250425-${ARCH} as base

USER root
ENV PYTHON_VERSION=3.9.18

# Python won’t try to write .pyc or .pyo files on the import of source modules
# Force stdin, stdout and stderr to be totally unbuffered. Good for logging
ENV PYTHONDONTWRITEBYTECODE=1
ENV PYTHONUNBUFFERED=1
ENV PYTHONIOENCODING=UTF-8


FROM base AS arch-x86_64

FROM base AS arch-arm64
RUN yum install -y python3-devel && \
rm -rf /var/cache/yum

FROM arch-${ARCH} AS runtime


WORKDIR /usr/lib/spark/code/

# Install GSProcessing requirements to pyenv Python
COPY requirements.txt requirements.txt
# Use --mount=type=cache,target=/root/.cache when Buildkit CI issue is fixed:
# https://github.com/moby/buildkit/issues/1512
RUN pip install -r /usr/lib/spark/code/requirements.txt \
&& rm -rf /root/.cache

# Install Huggingface model cache if it is necessary
ARG MODEL=""
ENV HF_HOME=/home/hadoop/.cache/huggingface/hub
RUN if [ -z "${MODEL}" ]; then \
echo "Skip installing model cache"; \
else \
echo "Installing model cache for $MODEL" && \
python3 -c "from transformers import AutoTokenizer; AutoTokenizer.from_pretrained('${MODEL}')"; \
python3 -c "from transformers import AutoModel; AutoModel.from_pretrained('${MODEL}')"; \
fi

# We use this file as an indicator of the execution environment
RUN touch /usr/lib/spark/code/EMR_SERVERLESS_EXECUTION

# GSProcessing codebase
COPY code/ /usr/lib/spark/code/

RUN python3 -m pip install --no-deps /usr/lib/spark/code/graphstorm-processing/

USER hadoop:hadoop
WORKDIR /home/hadoop

FROM runtime AS prod

FROM runtime AS test
RUN python3 -m pip install mock pytest && \
rm -rf /root/.cache
77 changes: 77 additions & 0 deletions graphstorm-processing/docker/0.5.0/emr/Dockerfile.cpu
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
FROM public.ecr.aws/amazoncorretto/amazoncorretto:17 as base

ENV PYTHON_VERSION=3.9.18

# Python won’t try to write .pyc or .pyo files on the import of source modules
# Force stdin, stdout and stderr to be totally unbuffered. Good for logging
ENV PYTHONDONTWRITEBYTECODE=1
ENV PYTHONUNBUFFERED=1
ENV PYTHONIOENCODING=UTF-8

ENV PYENV_ROOT="${HOME}/.pyenv"
ENV PATH="${PYENV_ROOT}/shims:${PYENV_ROOT}/bin:${PATH}"

ENV PYSPARK_DRIVER_PYTHON=${PYENV_ROOT}/shims/python
ENV PYSPARK_PYTHON=${PYENV_ROOT}/shims/python

# pyenv and Spark/YARN dependencies
RUN yum erase -y openssl-devel && \
yum install -y \
bzip2-devel\
gcc \
git \
headless \
hostname \
java-17-amazon-corretto-headless \
libffi-devel \
make \
ncurses-devel \
openssl11-devel \
readline-devel \
sqlite-devel \
sudo \
tar \
xz-devel && \
rm -rf /var/cache/yum

# Install Python through pyenv
RUN git clone https://github.com/pyenv/pyenv.git ${PYENV_ROOT} --single-branch && \
pyenv install ${PYTHON_VERSION} && \
pyenv global ${PYTHON_VERSION}

FROM base AS runtime

WORKDIR /usr/lib/spark/code/


# Install GSProcessing requirements to pyenv Python
COPY requirements.txt requirements.txt
# Use --mount=type=cache,target=/root/.cache when Buildkit CI issue is fixed:
# https://github.com/moby/buildkit/issues/1512
RUN pip3 install -r /usr/lib/spark/code/requirements.txt \
&& rm -rf /root/.cache

# Install Huggingface model cache if it is necessary
# This needs to happen after the transformers library has been installed above
ARG MODEL=""
ENV HF_HOME=/usr/lib/spark/.cache/huggingface/hub
RUN if [ -z "${MODEL}" ]; then \
echo "Skip installing model cache"; \
else \
echo "Installing model cache for $MODEL" && \
python3 -c "from transformers import AutoTokenizer; AutoTokenizer.from_pretrained('${MODEL}')"; \
python3 -c "from transformers import AutoModel; AutoModel.from_pretrained('${MODEL}')"; \
fi

# We use this file as an indicator of the execution environment
RUN touch /usr/lib/spark/code/EMR_EXECUTION

# GSProcessing codebase
COPY code/ /usr/lib/spark/code/

RUN python3 -m pip install --no-deps /usr/lib/spark/code/graphstorm-processing/
FROM runtime AS prod

FROM runtime AS test
RUN python3 -m pip install mock pytest && \
rm -rf /root/.cache
Loading