Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/_CE-Framework.yml
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,7 @@ jobs:
run: |
docker exec -t ${{ env.container_name }} /bin/bash -c '
export PATH=$(pwd)/run_env:${PATH}
export PYTHONFAULTHANDLER=1
cd /PaddleTest/framework/api
rm ./device/test_cuda_get_device_capability.py
bash run_paddle_ci.sh
Expand Down
8 changes: 6 additions & 2 deletions .github/workflows/docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ jobs:
docker_cpu_file: Dockerfile.cuda9_cudnn7_gcc48_py35_centos6
docker_sot_file: Dockerfile.gcc133_ubuntu24_cpu_sot
docker_coverage_file: Dockerfile.cuda117_cudnn8_gcc82_ubuntu18_coverage
docker_build_file: Dockerfile.cuda11.2_cudnn8_gcc82_trt8
docker_build_file: Dockerfile.cuda12.8_cudnn9_gcc11
docker_distribute_file: Dockerfile.cuda123_cudnn9_gcc122_ubuntu20
docker_xpu_file: Dockerfile.develop.xre.ubuntu22
docker_npu_file: Dockerfile.develop.npu
Expand Down Expand Up @@ -108,7 +108,11 @@ jobs:
fi
for name in "${!docker_files[@]}"
do
md5_value=`md5sum tools/dockerfile/${docker_files[$name]} | awk '{print $1}'`
if grep -q "tools/dockerfile/build_scripts" tools/dockerfile/${docker_files[$name]}; then
md5_value=`(md5sum tools/dockerfile/${docker_files[$name]}; find tools/dockerfile/build_scripts -type f -print | sort | xargs md5sum) | md5sum | awk '{print $1}'`
else
md5_value=`md5sum tools/dockerfile/${docker_files[$name]} | awk '{print $1}'`
fi
Comment on lines +111 to +115

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 优先级:P2
处理要求:请针对该评论进行回复(同意并已修改请回复 Done,不同意请说明理由)。

这里新增的 find tools/dockerfile/build_scripts ... 会让 is_merge == "true" 的路径在拉取缓存前失败:该分支只下载 ci_dockerfile.shDockerfile.ubuntu20,不会带出 tools/dockerfile/build_scripts 目录;在 Actions 的 bash -e -o pipefail 语义下,缺目录会直接中断,连已有缓存镜像也无法使用。请在目录存在时再把 build_scripts 纳入 hash,或者在 is_merge 分支同步下载该目录。一个本地化修复如下:

Suggested change
if grep -q "tools/dockerfile/build_scripts" tools/dockerfile/${docker_files[$name]}; then
md5_value=`(md5sum tools/dockerfile/${docker_files[$name]}; find tools/dockerfile/build_scripts -type f -print | sort | xargs md5sum) | md5sum | awk '{print $1}'`
else
md5_value=`md5sum tools/dockerfile/${docker_files[$name]} | awk '{print $1}'`
fi
if grep -q "tools/dockerfile/build_scripts" tools/dockerfile/${docker_files[$name]} && [ -d tools/dockerfile/build_scripts ]; then
md5_value=`(md5sum tools/dockerfile/${docker_files[$name]}; find tools/dockerfile/build_scripts -type f -print | sort | xargs md5sum) | md5sum | awk '{print $1}'`
else
md5_value=`md5sum tools/dockerfile/${docker_files[$name]} | awk '{print $1}'`
fi

if [ $name == "docker_dcu" ]; then
md5_value="76937a563116f6008c8ca4cb4f592759"
fi
Expand Down
14 changes: 14 additions & 0 deletions tools/dockerfile/build_scripts/install_gcc.sh
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,20 @@ elif [ "$1" == "gcc121" ]; then
cp ${lib_so_6} ${lib_so_6}.bak && rm -f ${lib_so_6} &&
ln -s /usr/local/gcc-12.1/lib64/libstdc++.so.6 ${lib_so_6} && \
cp /usr/local/gcc-12.1/lib64/libstdc++.so.6.0.30 ${lib_path}
elif [ "$1" == "gcc11" ]; then
GCC_VERSION=${GCC_VERSION:-11.5.0}
GCC_MAJOR_MINOR=$(echo ${GCC_VERSION} | cut -d. -f1,2)
GCC_PREFIX=/usr/local/gcc-${GCC_MAJOR_MINOR}
GCC_ARCHIVE=gcc-${GCC_VERSION}.tar.xz
wget -q https://ftp.gnu.org/gnu/gcc/gcc-${GCC_VERSION}/${GCC_ARCHIVE}
tar -xf ${GCC_ARCHIVE} && \
cd gcc-${GCC_VERSION} && \
unset LIBRARY_PATH CPATH C_INCLUDE_PATH PKG_CONFIG_PATH CPLUS_INCLUDE_PATH INCLUDE && \
./contrib/download_prerequisites && \
cd .. && mkdir temp_gcc11 && cd temp_gcc11 && \
../gcc-${GCC_VERSION}/configure --prefix=${GCC_PREFIX} --enable-checking=release --enable-languages=c,c++ --disable-multilib && \
make -j$(nproc) && make install
cd .. && rm -rf temp_gcc11 gcc-${GCC_VERSION} ${GCC_ARCHIVE}
elif [ "$1" == "gcc152" ]; then
GCC_VERSION=${GCC_VERSION:-15.2.0}
GCC_MAJOR_MINOR=$(echo ${GCC_VERSION} | cut -d. -f1,2)
Expand Down
16 changes: 11 additions & 5 deletions tools/dockerfile/ci_dockerfile.sh
Original file line number Diff line number Diff line change
Expand Up @@ -58,18 +58,24 @@ function make_sot_dockerfile(){


function make_ce_framework_dockerfile(){
dockerfile_name="Dockerfile.cuda11.2_cudnn8_gcc82_trt8"
sed "s#<baseimg>#nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04#g" ./Dockerfile.ubuntu20 >${dockerfile_name}
dockerfile_name="Dockerfile.cuda12.8_cudnn9_gcc11"
sed "s#<baseimg>#nvidia/cuda:12.8.1-cudnn-devel-ubuntu20.04#g" ./Dockerfile.ubuntu20 >${dockerfile_name}
dockerfile_line=$(wc -l ${dockerfile_name}|awk '{print $1}')
sed -i "s#<setcuda>#ENV LD_LIBRARY_PATH=/usr/local/cuda-11.8/targets/x86_64-linux/lib:\$LD_LIBRARY_PATH #g" ${dockerfile_name}
sed -i "s#<setcuda>#ENV LD_LIBRARY_PATH=/usr/local/cuda-12.8/targets/x86_64-linux/lib:\$LD_LIBRARY_PATH #g" ${dockerfile_name}
sed -i 's#<install_cpu_package>##g' ${dockerfile_name}
sed -i "7i RUN chmod 777 /tmp" ${dockerfile_name}
sed -i "${dockerfile_line}i RUN wget --no-check-certificate -q https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz \&\& \
tar -xzf hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/" ${dockerfile_name}
sed -i "${dockerfile_line}i RUN apt-get update && apt install -y zstd pigz libcurl4-openssl-dev gettext ninja-build" ${dockerfile_name}
sed -i "${dockerfile_line}i RUN pip3.10 install wheel distro jinja2 bce-python-sdk==0.8.74" ${dockerfile_name}
sed -i "${dockerfile_line}i RUN pip3.10 install nvidia-cuda-cupti-cu11==11.8.87 nvidia-cuda-runtime-cu11==11.8.89 nvidia-cudnn-cu11==8.7.0.84 nvidia-cublas-cu11==11.11.3.6 nvidia-cufft-cu11==10.9.0.58 nvidia-curand-cu11==10.3.0.86 nvidia-cusolver-cu11==11.4.1.48 nvidia-cusparse-cu11==11.7.5.86 nvidia-nccl-cu11==2.19.3" ${dockerfile_name}
sed -i 's#RUN bash /build_scripts/install_trt.sh#RUN bash /build_scripts/install_trt.sh trt8531#g' ${dockerfile_name}
sed -i "${dockerfile_line}i RUN pip3.10 install nvidia-cuda-nvrtc-cu12==12.8.61 nvidia-cuda-runtime-cu12==12.8.57 nvidia-cuda-cupti-cu12==12.8.57 nvidia-cudnn-cu12==9.7.1.26 nvidia-cublas-cu12==12.8.3.14 nvidia-cufft-cu12==11.3.3.41 nvidia-curand-cu12==10.3.9.55 nvidia-cusolver-cu12==11.7.2.55 nvidia-cusparse-cu12==12.5.7.53 nvidia-cusparselt-cu12==0.6.3 nvidia-nccl-cu12==2.28.3 nvidia-nvtx-cu12==12.8.55 nvidia-nvjitlink-cu12==12.8.61 nvidia-cufile-cu12==1.13.0.11" ${dockerfile_name}
sed -i 's#RUN bash /build_scripts/install_trt.sh##g' ${dockerfile_name}
sed -i 's#RUN bash /build_scripts/install_cudnn.sh cudnn841##g' ${dockerfile_name}
sed -i 's#ENV CUDNN_VERSION=8.4.1#ENV CUDNN_VERSION=9.7.1#g' ${dockerfile_name}
sed -i 's#RUN bash /build_scripts/install_gcc.sh gcc82#RUN bash /build_scripts/install_gcc.sh gcc11#g' ${dockerfile_name}
sed -i 's#/usr/local/gcc-8.2/bin/gcc#/usr/local/gcc-11.5/bin/gcc#g' ${dockerfile_name}
sed -i 's#/usr/local/gcc-8.2/bin/g++#/usr/local/gcc-11.5/bin/g++#g' ${dockerfile_name}
sed -i 's#ENV PATH=/usr/local/gcc-8.2/bin:$PATH#ENV PATH=/usr/local/gcc-11.5/bin:$PATH#g' ${dockerfile_name}
}


Expand Down
Loading