Merge pull request #43 from yakumioto/main

dylan-fan · web-flow · commit c8450d4c95a9 · 2024-06-07T11:27:32.000+08:00
support arm64 compilation based on docker-build
diff --git a/.gitignore b/.gitignore
@@ -5,3 +5,5 @@
 /native-build/.env
 /native-build/cos.yaml
 *.log
+
+cache
diff --git a/docker-build/README.md b/docker-build/README.md
@@ -99,6 +99,7 @@ FATE_DIR=/root/FATE bash build.sh all
 | `Build_GPU` | Build images that supports GPU | 0 |
 | `Build_LLM` | Build images that supports FATE-LLM | 0 |
 | `Build_LLM_VERSION` | FATE-LLM version | v1.2.0 |
+| `Platform` | Architecture types | amd64 |
 
 The command creates the base images and then the component images. After the command finishes, all images of FATE should be created. Use `docker images` to check the newly generated images:
 
@@ -139,7 +140,18 @@ The above are all images that can be built using FATE-Builder, if you want to bu
 
 ```sh
 FATE_DIR=/root/FATE TAG=1.11.2-release Build_Basic=1 Build_NN=1 Build_FUM=1 Build_Spark=1 Build_OP=1 Build_IPCL=1 Build_GPU=1 Build_LLM=1 Build_LLM_VERSION=v1.2.0 IPCL_PKG_DIR=/root/pailliercryptolib_python/ IPCL_VERSION=v1.1.3 bash docker-build/build.sh all
-``
+```
+
+### Cross-compilation function (optional)
+The "Platform" field can be used to specify the processor architecture types that the built image can support, currently supporting arm64 and amd64. If this field is not specified, amd64 will be defaulted.
+
+This function supports building images that support either arm64 or amd64 architecture under the amd64 architecture, and it also supports building images that support either arm64 or amd64 architectures under the arm64 architecture.
+
+For example, if you want to build an arm64 image, you can use the following command.For example, if you want to build an arm64 image, you can use the following command. (The current version only supports building basic images when building images that support the arm64 architecture, that is, Build_Basic=1)
+
+```sh
+Platform=arm64 FATE_DIR=/root/FATE TAG=1.11.2-release Build_Basic=1 Build_NN=0 Build_FUM=0 Build_Spark=0 Build_OP=0 Build_IPCL=0 Build_GPU=0 Build_LLM=0 Build_LLM_VERSION=v1.2.0 IPCL_PKG_DIR=/root/pailliercryptolib_python/ IPCL_VERSION=v1.1.3 bash docker-build/build.sh all
+```
 
 ### Pushing images to a registry (optional)
 
diff --git a/docker-build/README_zh.md b/docker-build/README_zh.md
@@ -100,6 +100,7 @@ FATE_DIR=/root/FATE bash build.sh all
 | `Build_GPU` | 构建支持GPU的镜像 | 0 |
 | `Build_LLM` | 构建支持FATE-LLM的镜像 | 0 |
 | `Build_LLM_VERSION` | 构建支持Build_LLM_VERSION镜像的版本 | v1.2.0 |
+| `Platform` | 构建的镜像支持的架构类型 | amd64 |
 
 所有用于构建镜像的“ Dockerfile”文件都存储在“docker/“子目录下。在脚本运行完之后，用户可以通过以下命令来检查构建好的镜像：
 
@@ -149,6 +150,17 @@ federatedai/base-image               <TAG>
 FATE_DIR=/root/FATE TAG=1.11.2-release Build_Basic=1 Build_NN=1 Build_FUM=1 Build_Spark=1 Build_OP=1 Build_IPCL=1 Build_GPU=1 Build_LLM=1 Build_LLM_VERSION=v1.2.0 IPCL_PKG_DIR=/root/pailliercryptolib_python/ IPCL_VERSION=v1.1.3 bash docker-build/build.sh all
 ```
 
+### 交叉编译功能（可选）
+可通过Platform字段指定构建的镜像支持的处理器架构类型，当前支持arm64和amd64。不指定该字段，则默认amd64。
+
+本功能支持在amd64架构下构建支持arm64或amd64架构的镜像，且支持在arm64架构下构建支持arm64或amd64架构的镜像。
+
+例如，如果想要构建arm64的镜像可以使用下面的命令。（当前版本在构建支持arm64架构的镜像时只支持构建基本的镜像，即Build_Basic=1）
+
+```sh
+Platform=arm64 FATE_DIR=/root/FATE TAG=1.11.2-release Build_Basic=1 Build_NN=0 Build_FUM=0 Build_Spark=0 Build_OP=0 Build_IPCL=0 Build_GPU=0 Build_LLM=0 Build_LLM_VERSION=v1.2.0 IPCL_PKG_DIR=/root/pailliercryptolib_python/ IPCL_VERSION=v1.1.3 bash docker-build/build.sh all
+```
+
 ### 把镜像推送到镜像仓库（可选）
 
 如果用户需要把构建出来的镜像推送到镜像仓库如DockerHub去的话，需要先通过以下命令登录相应的用户:
diff --git a/docker-build/base/basic/Dockerfile.amd64 b/docker-build/base/basic/Dockerfile.amd64
@@ -33,4 +33,4 @@ RUN python3 -m venv $VIRTUAL_ENV
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
 RUN pip install --upgrade pip && \
-    pip install --no-cache-dir -r requirements.txt 
+    pip install --no-cache-dir -r requirements.txt
diff --git a/docker-build/base/basic/Dockerfile.arm64 b/docker-build/base/basic/Dockerfile.arm64
@@ -0,0 +1,52 @@
+FROM ubuntu:bionic-20220913
+
+ARG version
+
+USER root
+WORKDIR /data/projects/python/
+ENV VIRTUAL_ENV /data/projects/python/venv
+
+RUN apt-get update && \
+    apt-get install -y curl gcc g++ make openssl supervisor libgmp-dev \
+        libmpfr-dev libmpc-dev libaio1 libaio-dev numactl autoconf automake \
+        libtool libffi-dev libssl-dev liblz4-1 liblz4-dev liblz4-tool zlib1g \
+        zlib1g-dev xz-utils libopenblas-dev liblapack-dev libsm-dev pkg-config \
+        libblas-dev libatlas-base-dev libtinfo-dev gfortran libbz2-dev liblzma-dev \
+        libsnappy-dev && \
+    apt-get autoremove && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* 
+
+RUN curl -o Python-3.8.13.tar.xz https://www.python.org/ftp/python/3.8.13/Python-3.8.13.tar.xz && \
+    tar -xvf Python-3.8.13.tar.xz && \
+    cd Python-3.8.13 && \
+    ./configure --prefix=/opt/python3 && \
+    make altinstall && \
+    ln -s /opt/python3/bin/python3.8 /usr/local/bin/python3.8 && \
+    ln -s /usr/local/bin/python3.8 /usr/local/bin/python3 && \
+    ln -s /usr/local/bin/python3 /usr/local/bin/python && \
+    ln -s /opt/python3/bin/pip3.8 /usr/bin/pip3.8 && \
+    ln -s /usr/bin/pip3.8 /usr/bin/pip3 && \
+    ln -s /usr/bin/pip3 /usr/bin/pip && \
+    cd .. && \
+    rm Python-3.8.13.tar.xz && \
+    rm -rf Python-3.8.13
+
+RUN curl -L https://github.com/llvm/llvm-project/releases/download/llvmorg-9.0.1/clang+llvm-9.0.1-aarch64-linux-gnu.tar.xz \
+        -o /root/clang+llvm-9.0.1-aarch64-linux-gnu.tar.xz && \
+    tar xJvf /root/clang+llvm-9.0.1-aarch64-linux-gnu.tar.xz -C /root/ && \
+    mv /root/clang+llvm-9.0.1-aarch64-linux-gnu /root/clang-llvm-9.0.1 \
+    && rm -f /root/clang+llvm-9.0.1-aarch64-linux-gnu.tar.xz
+
+RUN python3 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:/root/clang-llvm-9.0.1/bin:$PATH"
+COPY requirements*.txt /data/projects/python/
+
+RUN pip install --upgrade pip && \
+    sed -i 's/tensorflow-cpu==2.11.1/tensorflow-aarch64==2.11.1/g' requirements.txt && \
+    sed -i 's/torch==1.13.1+cpu/torch==1.13.1/g' requirements.txt && \
+    sed -i 's/torchvision==0.14.1+cpu/torchvision==0.14.1/g' requirements.txt && \
+    sed -i 's/ipcl-python==2.0.0/#ipcl-python==2.0.0/g' requirements.txt && \
+    pip install --no-cache-dir -r requirements.txt
+
+ENV LD_PRELOAD=/data/projects/python/venv/lib/python3.8/site-packages/scikit_learn.libs/libgomp-d22c30c5.so.1.0.0
diff --git a/docker-build/build.sh b/docker-build/build.sh
@@ -30,6 +30,7 @@ set -euxo pipefail
 : "${Build_GPU:=0}"
 : "${Build_LLM:=0}"
 : "${Build_LLM_VERSION:=v1.2.0}"
+: "${Platform:=amd64}"
 
 BASE_DIR=$(dirname "$0")
 cd $BASE_DIR
@@ -75,11 +76,22 @@ check_fate_dir() {
 #     echo "Built fate"
 # }
 
+# bootstrapDockerBuildx(){
+#         echo "BOOTSTRAP DOCKER BUILDX TOOLKIT"
+#         echo "LOGIN IN YOUR DOCKER HUB ACCOUNT: ${PREFIX}"
+#         docker login -u ${PREFIX}
+#         docker buildx create --driver docker-container --platform "linux/amd64,linux/arm64" --name cross --node cross --use --bootstrap
+#         echo "FINSH BOOTSTRAP DOCKER BUILDX TOOLKIT"
+# }
+
 buildBase() {
+        # bootstrapDockerBuildx
         echo "START BUILDING BASE IMAGE"
         #cd ${WORKING_DIR}
-        docker build --build-arg version=${version} -f ${WORKING_DIR}/base/basic/Dockerfile \
+        docker build --platform linux/${Platform} --build-arg version=${version} -f ${WORKING_DIR}/base/basic/Dockerfile.${Platform} \
           -t ${PREFIX}/base-image:${BASE_TAG} ${PACKAGE_DIR_CACHE}
+        # docker build --build-arg version=${version} -f ${WORKING_DIR}/base/basic/Dockerfile \
+        #   -t ${PREFIX}/base-image:${BASE_TAG} ${PACKAGE_DIR_CACHE}
         echo "FINISH BUILDING BASE IMAGE"
 }
 
@@ -89,19 +101,19 @@ buildEggrollBasicCPU() {
         echo "START BUILDING Eggroll Module IMAGE"
 
         echo "### START BUILDING fateflow ###"
-        docker build --build-arg PREFIX=${PREFIX} --build-arg BASE_IMAGE=base-image --build-arg BASE_TAG=${BASE_TAG} ${Docker_Options} -t ${PREFIX}/fateflow:${TAG} \
+        docker build --platform linux/${Platform} --build-arg PREFIX=${PREFIX} --build-arg BASE_IMAGE=base-image --build-arg BASE_TAG=${BASE_TAG} ${Docker_Options} -t ${PREFIX}/fateflow:${TAG} \
                 -f ${WORKING_DIR}/modules/fateflow/Dockerfile ${PACKAGE_DIR_CACHE}
         echo "### FINISH BUILDING fateflow ###"
         echo ""
         echo "### START BUILDING fateboard ###"
-        docker build --build-arg PREFIX=${PREFIX} --build-arg BASE_TAG=${BASE_TAG} ${Docker_Options} -t ${PREFIX}/fateboard:${TAG} \
-                -f ${WORKING_DIR}/modules/fateboard/Dockerfile ${PACKAGE_DIR_CACHE}
+        docker build --platform linux/${Platform} --build-arg PREFIX=${PREFIX} --build-arg BASE_TAG=${BASE_TAG} ${Docker_Options} -t ${PREFIX}/fateboard:${TAG} \
+                -f ${WORKING_DIR}/modules/fateboard/Dockerfile.${Platform} ${PACKAGE_DIR_CACHE}
         echo "### FINISH BUILDING fateboard ###"
         echo ""
 
         echo "### START BUILDING eggroll ###"
-        docker build --build-arg PREFIX=${PREFIX} --build-arg BASE_IMAGE=base-image --build-arg BASE_TAG=${BASE_TAG} ${Docker_Options} -t ${PREFIX}/eggroll:${TAG} \
-                -f ${WORKING_DIR}/modules/eggroll/Dockerfile ${PACKAGE_DIR_CACHE}
+        docker build --platform linux/${Platform} --build-arg PREFIX=${PREFIX} --build-arg BASE_IMAGE=base-image --build-arg BASE_TAG=${BASE_TAG} ${Docker_Options} -t ${PREFIX}/eggroll:${TAG} \
+                -f ${WORKING_DIR}/modules/eggroll/Dockerfile.${Platform} ${PACKAGE_DIR_CACHE}
         echo "### FINISH BUILDING eggroll ###"
         echo ""
 
@@ -111,7 +123,6 @@ buildEggrollBasicCPU() {
 buildSparkBasicCPU(){
         echo "START BUILDING Spark Module IMAGE"
 
-
         echo "### START BUILDING fateflow-spark ###"
         docker build --build-arg PREFIX=${PREFIX} --build-arg BASE_IMAGE=fateflow --build-arg BASE_TAG=${BASE_TAG} ${Docker_Options} -t ${PREFIX}/fateflow-spark:${TAG} -f ${WORKING_DIR}/modules/fateflow-spark/Dockerfile ${WORKING_DIR}/modules/fateflow-spark/
         echo "### FINISH BUILDING fateflow-spark ###"
diff --git a/docker-build/build_docker.sh b/docker-build/build_docker.sh
@@ -108,7 +108,7 @@ packaging_fateboard() {
     fateboard_version=$(grep -E -m 1 -o "<version>(.*)</version>" ./pom.xml | tr -d '[\\-a-z<>//]' | awk -F "version" '{print $2}')
     echo "[INFO] fateboard version "${fateboard_version}
         
-    docker run --rm -u $(id -u):$(id -g) -v ${source_dir}/fateboard:/data/projects/fate/fateboard --entrypoint="" maven:3.8-jdk-8 /bin/bash -c "cd /data/projects/fate/fateboard && mvn clean package -DskipTests"
+    docker run --rm --platform linux/amd64 -v ${source_dir}/fateboard:/data/projects/fate/fateboard --entrypoint="" maven:3.8-jdk-8 /bin/bash -c "cd /data/projects/fate/fateboard && mvn clean package -DskipTests"
     mkdir -p ${package_dir}/fateboard/conf
     mkdir -p ${package_dir}/fateboard/ssh
     cp ./target/fateboard-${fateboard_version}.jar ${package_dir}/fateboard/
@@ -125,7 +125,7 @@ packaging_eggroll() {
     pull_eggroll
     cd ./eggroll
     cd ./deploy
-    docker run --rm -u $(id -u):$(id -g) -v ${source_dir}/eggroll:/data/projects/fate/eggroll --entrypoint="" maven:3.8-jdk-8 /bin/bash -c "cd /data/projects/fate/eggroll/deploy && bash auto-packaging.sh"
+    docker run --rm --platform linux/amd64 -v ${source_dir}/eggroll:/data/projects/fate/eggroll --entrypoint="" maven:3.8-jdk-8 /bin/bash -c "cd /data/projects/fate/eggroll/deploy && bash auto-packaging.sh"
     mkdir -p ${package_dir}/eggroll
     mv ${source_dir}/eggroll/eggroll.tar.gz ${package_dir}/eggroll/
     cd ${package_dir}/eggroll/
diff --git a/docker-build/modules/eggroll/Dockerfile.amd64 b/docker-build/modules/eggroll/Dockerfile.amd64
diff --git a/docker-build/modules/eggroll/Dockerfile.arm64 b/docker-build/modules/eggroll/Dockerfile.arm64
@@ -0,0 +1,39 @@
+ARG PREFIX=prefix
+ARG BASE_TAG=tag
+ARG BASE_IMAGE=image
+FROM ${PREFIX}/${BASE_IMAGE}:${BASE_TAG} as builder
+WORKDIR /data/projects/fate/
+COPY eggroll.tar.gz .
+COPY fate.tar.gz .
+COPY conf.tar.gz .
+COPY fateflow.tar.gz .
+COPY fate.env .
+RUN tar -xzf eggroll.tar.gz; \
+    tar -xzf fate.tar.gz; \
+    tar -xzf conf.tar.gz; \
+    tar -xzf fateflow.tar.gz;
+
+RUN ls -l
+
+FROM ${PREFIX}/${BASE_IMAGE}:${BASE_TAG}
+
+RUN apt-get update && \
+    apt-get install -y openjdk-8-jdk && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR /data/projects/fate/eggroll/
+
+COPY --from=builder /data/projects/fate/eggroll /data/projects/fate/eggroll
+COPY --from=builder /data/projects/fate/fate /data/projects/fate/fate
+COPY --from=builder /data/projects/fate/fateflow /data/projects/fate/fateflow
+COPY  --from=builder /data/projects/fate/conf /data/projects/fate/conf
+COPY --from=builder /data/projects/fate/fate.env /data/projects/fate/
+
+ENV PYTHONPATH=/data/projects/fate/fate/python:/data/projects/fate/eggroll/python:/data/projects/fate/fateflow/python
+ENV EGGROLL_HOME=/data/projects/fate/eggroll/
+
+ENV TINI_VERSION v0.18.0
+ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-arm64 /tini
+RUN chmod +x /tini
+ENTRYPOINT ["/tini", "--"]
diff --git a/docker-build/modules/fateboard/Dockerfile.amd64 b/docker-build/modules/fateboard/Dockerfile.amd64
diff --git a/docker-build/modules/fateboard/Dockerfile.arm64 b/docker-build/modules/fateboard/Dockerfile.arm64
@@ -0,0 +1,19 @@
+FROM openjdk:8u342-jre as builder
+WORKDIR /data/projects/fate/
+COPY fateboard.tar.gz .
+RUN tar -xzf fateboard.tar.gz
+
+FROM openjdk:8u342-jre
+
+RUN apt-get update && \
+    apt-get install -y tzdata && \
+    apt-get autoremove && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* 
+
+WORKDIR /data/projects/fate/fateboard/
+
+COPY --from=builder /data/projects/fate/fateboard /data/projects/fate/fateboard
+EXPOSE 8080
+
+CMD java -Dspring.config.location=/data/projects/fate/fateboard/conf/application.properties  -Dssh_config_file=/data/projects/fate/fateboard/conf  -Xmx2048m -Xms2048m -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError  -jar fateboard.jar
