Skip to content

Commit 8977479

Browse files
yakumiotoyakumioto
committed
support arm64 compilation based on docker-build
1 parent 572e329 commit 8977479

File tree

8 files changed

+128
-10
lines changed

8 files changed

+128
-10
lines changed

‎docker-build/base/basic/Dockerfile‎ renamed to ‎docker-build/base/basic/Dockerfile.amd64‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,4 +33,4 @@ RUN python3 -m venv $VIRTUAL_ENV
3333
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
3434

3535
RUN pip install --upgrade pip && \
36-
pip install --no-cache-dir -r requirements.txt
36+
pip install --no-cache-dir -r requirements.txt

‎docker-build/base/basic/Dockerfile.arm64‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
FROM ubuntu:bionic-20220913
2+
3+
ARG version
4+
5+
USER root
6+
WORKDIR /data/projects/python/
7+
ENV VIRTUAL_ENV /data/projects/python/venv
8+
9+
RUN apt-get update && \
10+
apt-get install -y curl gcc g++ make openssl supervisor libgmp-dev \
11+
libmpfr-dev libmpc-dev libaio1 libaio-dev numactl autoconf automake \
12+
libtool libffi-dev libssl-dev liblz4-1 liblz4-dev liblz4-tool zlib1g \
13+
zlib1g-dev xz-utils libopenblas-dev liblapack-dev libsm-dev pkg-config \
14+
libblas-dev libatlas-base-dev libtinfo-dev gfortran && \
15+
apt-get autoremove && \
16+
apt-get clean && \
17+
rm -rf /var/lib/apt/lists/*
18+
19+
RUN curl -o Python-3.8.13.tar.xz https://www.python.org/ftp/python/3.8.13/Python-3.8.13.tar.xz && \
20+
tar -xvf Python-3.8.13.tar.xz && \
21+
cd Python-3.8.13 && \
22+
./configure --prefix=/opt/python3 && \
23+
make altinstall && \
24+
ln -s /opt/python3/bin/python3.8 /usr/local/bin/python3.8 && \
25+
ln -s /usr/local/bin/python3.8 /usr/local/bin/python3 && \
26+
ln -s /usr/local/bin/python3 /usr/local/bin/python && \
27+
ln -s /opt/python3/bin/pip3.8 /usr/bin/pip3.8 && \
28+
ln -s /usr/bin/pip3.8 /usr/bin/pip3 && \
29+
ln -s /usr/bin/pip3 /usr/bin/pip && \
30+
cd .. && \
31+
rm Python-3.8.13.tar.xz && \
32+
rm -rf Python-3.8.13
33+
34+
RUN curl -L https://github.com/llvm/llvm-project/releases/download/llvmorg-9.0.1/clang+llvm-9.0.1-aarch64-linux-gnu.tar.xz \
35+
-o /root/clang+llvm-9.0.1-aarch64-linux-gnu.tar.xz && \
36+
tar xJvf /root/clang+llvm-9.0.1-aarch64-linux-gnu.tar.xz -C /root/ && \
37+
mv /root/clang+llvm-9.0.1-aarch64-linux-gnu /root/clang-llvm-9.0.1 \
38+
&& rm -f /root/clang+llvm-9.0.1-aarch64-linux-gnu.tar.xz
39+
40+
RUN python3 -m venv $VIRTUAL_ENV
41+
ENV PATH="$VIRTUAL_ENV/bin:/root/clang-llvm-9.0.1/bin:$PATH"
42+
COPY requirements*.txt /data/projects/python/
43+
44+
RUN pip install --upgrade pip && \
45+
sed -i 's/tensorflow-cpu==2.11.1/tensorflow-aarch64==2.11.1/g' requirements.txt && \
46+
sed -i 's/torch==1.13.1+cpu/torch==1.13.1/g' requirements.txt && \
47+
sed -i 's/torchvision==0.14.1+cpu/torchvision==0.14.1/g' requirements.txt && \
48+
sed -i 's/ipcl-python==2.0.0/#ipcl-python==2.0.0/g' requirements.txt && \
49+
pip install --no-cache-dir -r requirements.txt

‎docker-build/build.sh‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ set -euxo pipefail
3030
: "${Build_GPU:=0}"
3131
: "${Build_LLM:=0}"
3232
: "${Build_LLM_VERSION:=v1.2.0}"
33+
: "${Platform:=amd64}"
3334

3435
BASE_DIR=$(dirname "$0")
3536
cd $BASE_DIR
@@ -75,11 +76,22 @@ check_fate_dir() {
7576
# echo "Built fate"
7677
# }
7778

79+
# bootstrapDockerBuildx(){
80+
# echo "BOOTSTRAP DOCKER BUILDX TOOLKIT"
81+
# echo "LOGIN IN YOUR DOCKER HUB ACCOUNT: ${PREFIX}"
82+
# docker login -u ${PREFIX}
83+
# docker buildx create --driver docker-container --platform "linux/amd64,linux/arm64" --name cross --node cross --use --bootstrap
84+
# echo "FINSH BOOTSTRAP DOCKER BUILDX TOOLKIT"
85+
# }
86+
7887
buildBase() {
88+
# bootstrapDockerBuildx
7989
echo "START BUILDING BASE IMAGE"
8090
#cd ${WORKING_DIR}
81-
docker build --build-arg version=${version} -f ${WORKING_DIR}/base/basic/Dockerfile \
91+
docker build --platform linux/${Platform} --build-arg version=${version} -f ${WORKING_DIR}/base/basic/Dockerfile.${Platform} \
8292
-t ${PREFIX}/base-image:${BASE_TAG} ${PACKAGE_DIR_CACHE}
93+
# docker build --build-arg version=${version} -f ${WORKING_DIR}/base/basic/Dockerfile \
94+
# -t ${PREFIX}/base-image:${BASE_TAG} ${PACKAGE_DIR_CACHE}
8395
echo "FINISH BUILDING BASE IMAGE"
8496
}
8597

@@ -89,19 +101,19 @@ buildEggrollBasicCPU() {
89101
echo "START BUILDING Eggroll Module IMAGE"
90102

91103
echo "### START BUILDING fateflow ###"
92-
docker build --build-arg PREFIX=${PREFIX} --build-arg BASE_IMAGE=base-image --build-arg BASE_TAG=${BASE_TAG} ${Docker_Options} -t ${PREFIX}/fateflow:${TAG} \
104+
docker build --platform linux/${Platform} --build-arg PREFIX=${PREFIX} --build-arg BASE_IMAGE=base-image --build-arg BASE_TAG=${BASE_TAG} ${Docker_Options} -t ${PREFIX}/fateflow:${TAG} \
93105
-f ${WORKING_DIR}/modules/fateflow/Dockerfile ${PACKAGE_DIR_CACHE}
94106
echo "### FINISH BUILDING fateflow ###"
95107
echo ""
96108
echo "### START BUILDING fateboard ###"
97-
docker build --build-arg PREFIX=${PREFIX} --build-arg BASE_TAG=${BASE_TAG} ${Docker_Options} -t ${PREFIX}/fateboard:${TAG} \
98-
-f ${WORKING_DIR}/modules/fateboard/Dockerfile ${PACKAGE_DIR_CACHE}
109+
docker build --platform linux/${Platform} --build-arg PREFIX=${PREFIX} --build-arg BASE_TAG=${BASE_TAG} ${Docker_Options} -t ${PREFIX}/fateboard:${TAG} \
110+
-f ${WORKING_DIR}/modules/fateboard/Dockerfile.${Platform} ${PACKAGE_DIR_CACHE}
99111
echo "### FINISH BUILDING fateboard ###"
100112
echo ""
101113

102114
echo "### START BUILDING eggroll ###"
103-
docker build --build-arg PREFIX=${PREFIX} --build-arg BASE_IMAGE=base-image --build-arg BASE_TAG=${BASE_TAG} ${Docker_Options} -t ${PREFIX}/eggroll:${TAG} \
104-
-f ${WORKING_DIR}/modules/eggroll/Dockerfile ${PACKAGE_DIR_CACHE}
115+
docker build --platform linux/${Platform} --build-arg PREFIX=${PREFIX} --build-arg BASE_IMAGE=base-image --build-arg BASE_TAG=${BASE_TAG} ${Docker_Options} -t ${PREFIX}/eggroll:${TAG} \
116+
-f ${WORKING_DIR}/modules/eggroll/Dockerfile.${Platform} ${PACKAGE_DIR_CACHE}
105117
echo "### FINISH BUILDING eggroll ###"
106118
echo ""
107119

@@ -111,7 +123,6 @@ buildEggrollBasicCPU() {
111123
buildSparkBasicCPU(){
112124
echo "START BUILDING Spark Module IMAGE"
113125

114-
115126
echo "### START BUILDING fateflow-spark ###"
116127
docker build --build-arg PREFIX=${PREFIX} --build-arg BASE_IMAGE=fateflow --build-arg BASE_TAG=${BASE_TAG} ${Docker_Options} -t ${PREFIX}/fateflow-spark:${TAG} -f ${WORKING_DIR}/modules/fateflow-spark/Dockerfile ${WORKING_DIR}/modules/fateflow-spark/
117128
echo "### FINISH BUILDING fateflow-spark ###"

‎docker-build/build_docker.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ packaging_fateboard() {
108108
fateboard_version=$(grep -E -m 1 -o "<version>(.*)</version>" ./pom.xml | tr -d '[\\-a-z<>//]' | awk -F "version" '{print $2}')
109109
echo "[INFO] fateboard version "${fateboard_version}
110110

111-
docker run --rm -u $(id -u):$(id -g) -v ${source_dir}/fateboard:/data/projects/fate/fateboard --entrypoint="" maven:3.8-jdk-8 /bin/bash -c "cd /data/projects/fate/fateboard && mvn clean package -DskipTests"
111+
docker run --rm --platform linux/amd64 -v ${source_dir}/fateboard:/data/projects/fate/fateboard --entrypoint="" maven:3.8-jdk-8 /bin/bash -c "cd /data/projects/fate/fateboard && mvn clean package -DskipTests"
112112
mkdir -p ${package_dir}/fateboard/conf
113113
mkdir -p ${package_dir}/fateboard/ssh
114114
cp ./target/fateboard-${fateboard_version}.jar ${package_dir}/fateboard/
@@ -125,7 +125,7 @@ packaging_eggroll() {
125125
pull_eggroll
126126
cd ./eggroll
127127
cd ./deploy
128-
docker run --rm -u $(id -u):$(id -g) -v ${source_dir}/eggroll:/data/projects/fate/eggroll --entrypoint="" maven:3.8-jdk-8 /bin/bash -c "cd /data/projects/fate/eggroll/deploy && bash auto-packaging.sh"
128+
docker run --rm --platform linux/amd64 -v ${source_dir}/eggroll:/data/projects/fate/eggroll --entrypoint="" maven:3.8-jdk-8 /bin/bash -c "cd /data/projects/fate/eggroll/deploy && bash auto-packaging.sh"
129129
mkdir -p ${package_dir}/eggroll
130130
mv ${source_dir}/eggroll/eggroll.tar.gz ${package_dir}/eggroll/
131131
cd ${package_dir}/eggroll/

‎docker-build/modules/eggroll/Dockerfile‎ renamed to ‎docker-build/modules/eggroll/Dockerfile.amd64‎

File renamed without changes.

‎docker-build/modules/eggroll/Dockerfile.arm64‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
ARG PREFIX=prefix
2+
ARG BASE_TAG=tag
3+
ARG BASE_IMAGE=image
4+
FROM ${PREFIX}/${BASE_IMAGE}:${BASE_TAG} as builder
5+
WORKDIR /data/projects/fate/
6+
COPY eggroll.tar.gz .
7+
COPY fate.tar.gz .
8+
COPY conf.tar.gz .
9+
COPY fateflow.tar.gz .
10+
COPY fate.env .
11+
RUN tar -xzf eggroll.tar.gz; \
12+
tar -xzf fate.tar.gz; \
13+
tar -xzf conf.tar.gz; \
14+
tar -xzf fateflow.tar.gz;
15+
16+
RUN ls -l
17+
18+
FROM ${PREFIX}/${BASE_IMAGE}:${BASE_TAG}
19+
20+
RUN apt-get update && \
21+
apt-get install -y openjdk-8-jdk && \
22+
apt-get clean && \
23+
rm -rf /var/lib/apt/lists/*
24+
25+
WORKDIR /data/projects/fate/eggroll/
26+
27+
COPY --from=builder /data/projects/fate/eggroll /data/projects/fate/eggroll
28+
COPY --from=builder /data/projects/fate/fate /data/projects/fate/fate
29+
COPY --from=builder /data/projects/fate/fateflow /data/projects/fate/fateflow
30+
COPY --from=builder /data/projects/fate/conf /data/projects/fate/conf
31+
COPY --from=builder /data/projects/fate/fate.env /data/projects/fate/
32+
33+
ENV PYTHONPATH=/data/projects/fate/fate/python:/data/projects/fate/eggroll/python:/data/projects/fate/fateflow/python
34+
ENV EGGROLL_HOME=/data/projects/fate/eggroll/
35+
36+
ENV TINI_VERSION v0.18.0
37+
ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-arm64 /tini
38+
RUN chmod +x /tini
39+
ENTRYPOINT ["/tini", "--"]

‎docker-build/modules/fateboard/Dockerfile‎ renamed to ‎docker-build/modules/fateboard/Dockerfile.amd64‎

File renamed without changes.

‎docker-build/modules/fateboard/Dockerfile.arm64‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
FROM openjdk:8u342-jre as builder
2+
WORKDIR /data/projects/fate/
3+
COPY fateboard.tar.gz .
4+
RUN tar -xzf fateboard.tar.gz
5+
6+
FROM openjdk:8u342-jre
7+
8+
RUN apt-get update && \
9+
apt-get install -y tzdata && \
10+
apt-get autoremove && \
11+
apt-get clean && \
12+
rm -rf /var/lib/apt/lists/*
13+
14+
WORKDIR /data/projects/fate/fateboard/
15+
16+
COPY --from=builder /data/projects/fate/fateboard /data/projects/fate/fateboard
17+
EXPOSE 8080
18+
19+
CMD java -Dspring.config.location=/data/projects/fate/fateboard/conf/application.properties -Dssh_config_file=/data/projects/fate/fateboard/conf -Xmx2048m -Xms2048m -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -jar fateboard.jar

0 commit comments

Comments
 (0)