Add a --prefer-binary flag to 'pip install' command (#2096)

tenzen-y · web-flow · commit a44aaea7f2c6 · 2023-01-24T14:25:25.000Z
* Pin the h5py version with 3.7.0

Signed-off-by: Yuki Iwai &lt;yuki.iwai.tz@gmail.com&gt;

* Add a --prefer-binary flag to 'pip install' command

Signed-off-by: Yuki Iwai &lt;yuki.iwai.tz@gmail.com&gt;

Signed-off-by: Yuki Iwai &lt;yuki.iwai.tz@gmail.com&gt;
diff --git a/.github/workflows/template-setup-e2e-test/action.yaml b/.github/workflows/template-setup-e2e-test/action.yaml
@@ -29,4 +29,4 @@ runs:
 
     - name: Install Katib SDK
       shell: bash
-      run: pip install -e sdk/python/v1beta1
+      run: pip install --prefer-binary -e sdk/python/v1beta1
diff --git a/Makefile b/Makefile
@@ -42,7 +42,7 @@ endif
 
 yamllint:
 ifndef HAS_YAMLLINT
-	pip install yamllint
+	pip install --prefer-binary yamllint
 	$(info "yamllint has been installed")
 endif
 	hack/verify-yamllint.sh
@@ -150,16 +150,16 @@ update-boilerplate:
 	./hack/boilerplate/update-boilerplate.sh
 
 prepare-pytest:
-	pip install -r test/unit/v1beta1/requirements.txt
-	pip install -r cmd/suggestion/hyperopt/v1beta1/requirements.txt
-	pip install -r cmd/suggestion/skopt/v1beta1/requirements.txt
-	pip install -r cmd/suggestion/optuna/v1beta1/requirements.txt
-	pip install -r cmd/suggestion/hyperband/v1beta1/requirements.txt
-	pip install -r cmd/suggestion/nas/enas/v1beta1/requirements.txt
-	pip install -r cmd/suggestion/nas/darts/v1beta1/requirements.txt
-	pip install -r cmd/suggestion/pbt/v1beta1/requirements.txt
-	pip install -r cmd/earlystopping/medianstop/v1beta1/requirements.txt
-	pip install -r cmd/metricscollector/v1beta1/tfevent-metricscollector/requirements.txt
+	pip install --prefer-binary -r test/unit/v1beta1/requirements.txt
+	pip install --prefer-binary -r cmd/suggestion/hyperopt/v1beta1/requirements.txt
+	pip install --prefer-binary -r cmd/suggestion/skopt/v1beta1/requirements.txt
+	pip install --prefer-binary -r cmd/suggestion/optuna/v1beta1/requirements.txt
+	pip install --prefer-binary -r cmd/suggestion/hyperband/v1beta1/requirements.txt
+	pip install --prefer-binary -r cmd/suggestion/nas/enas/v1beta1/requirements.txt
+	pip install --prefer-binary -r cmd/suggestion/nas/darts/v1beta1/requirements.txt
+	pip install --prefer-binary -r cmd/suggestion/pbt/v1beta1/requirements.txt
+	pip install --prefer-binary -r cmd/earlystopping/medianstop/v1beta1/requirements.txt
+	pip install --prefer-binary -r cmd/metricscollector/v1beta1/tfevent-metricscollector/requirements.txt
 
 prepare-pytest-testdata:
 ifeq ("$(wildcard $(TEST_TENSORFLOW_EVENT_FILE_PATH))", "")
diff --git a/cmd/earlystopping/medianstop/v1beta1/Dockerfile b/cmd/earlystopping/medianstop/v1beta1/Dockerfile
@@ -17,7 +17,7 @@ ADD ./${EARLY_STOPPING_DIR}/ ${TARGET_DIR}/${EARLY_STOPPING_DIR}/
 
 WORKDIR  ${TARGET_DIR}/${EARLY_STOPPING_DIR}
 
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --prefer-binary --no-cache-dir -r requirements.txt
 RUN chgrp -R 0 ${TARGET_DIR} \
   && chmod -R g+rwX ${TARGET_DIR}
 
diff --git a/cmd/metricscollector/v1beta1/tfevent-metricscollector/Dockerfile b/cmd/metricscollector/v1beta1/tfevent-metricscollector/Dockerfile
@@ -17,7 +17,7 @@ RUN if [ "${TARGETARCH}" = "arm64" ]; then \
     rm -rf /var/lib/apt/lists/*; \
     fi
 
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --prefer-binary --no-cache-dir -r requirements.txt
 RUN chgrp -R 0 ${TARGET_DIR} \
   && chmod -R g+rwX ${TARGET_DIR}
 
diff --git a/cmd/metricscollector/v1beta1/tfevent-metricscollector/Dockerfile.ppc64le b/cmd/metricscollector/v1beta1/tfevent-metricscollector/Dockerfile.ppc64le
@@ -1,6 +1,6 @@
 FROM ibmcom/tensorflow-ppc64le:2.2.0-py3
 ADD . /usr/src/app/github.com/kubeflow/katib
 WORKDIR /usr/src/app/github.com/kubeflow/katib/cmd/metricscollector/v1beta1/tfevent-metricscollector/
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --prefer-binary --no-cache-dir -r requirements.txt
 ENV PYTHONPATH /usr/src/app/github.com/kubeflow/katib:/usr/src/app/github.com/kubeflow/katib/pkg/apis/manager/v1beta1/python:/usr/src/app/github.com/kubeflow/katib/pkg/metricscollector/v1beta1/tfevent-metricscollector/:/usr/src/app/github.com/kubeflow/katib/pkg/metricscollector/v1beta1/common/
 ENTRYPOINT ["python", "main.py"]
diff --git a/cmd/suggestion/hyperband/v1beta1/Dockerfile b/cmd/suggestion/hyperband/v1beta1/Dockerfile
@@ -26,7 +26,7 @@ COPY --from=downloader /bin/grpc_health_probe /bin/grpc_health_probe
 
 WORKDIR  ${TARGET_DIR}/${SUGGESTION_DIR}
 
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --prefer-binary --no-cache-dir -r requirements.txt
 RUN chgrp -R 0 ${TARGET_DIR} \
   && chmod -R g+rwX ${TARGET_DIR}
 
diff --git a/cmd/suggestion/hyperopt/v1beta1/Dockerfile b/cmd/suggestion/hyperopt/v1beta1/Dockerfile
@@ -13,7 +13,7 @@ ENV TARGET_DIR /opt/katib
 ENV SUGGESTION_DIR cmd/suggestion/hyperopt/v1beta1
 ENV PYTHONPATH ${TARGET_DIR}:${TARGET_DIR}/pkg/apis/manager/v1beta1/python:${TARGET_DIR}/pkg/apis/manager/health/python
 
-RUN if [ "${TARGETARCH}" = "ppc64le" ] || [ "${TARGETARCH}" = "aarch64" ]; then \
+RUN if [ "${TARGETARCH}" = "ppc64le" ]; then \
     apt-get -y update && \
     apt-get -y install gfortran libopenblas-dev liblapack-dev && \
     apt-get clean && \
@@ -26,7 +26,7 @@ COPY --from=downloader /bin/grpc_health_probe /bin/grpc_health_probe
 
 WORKDIR  ${TARGET_DIR}/${SUGGESTION_DIR}
 
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --prefer-binary --no-cache-dir -r requirements.txt
 RUN chgrp -R 0 ${TARGET_DIR} \
   && chmod -R g+rwX ${TARGET_DIR}
 
diff --git a/cmd/suggestion/nas/darts/v1beta1/Dockerfile b/cmd/suggestion/nas/darts/v1beta1/Dockerfile
@@ -13,7 +13,7 @@ ENV TARGET_DIR /opt/katib
 ENV SUGGESTION_DIR cmd/suggestion/nas/darts/v1beta1
 ENV PYTHONPATH ${TARGET_DIR}:${TARGET_DIR}/pkg/apis/manager/v1beta1/python:${TARGET_DIR}/pkg/apis/manager/health/python
 
-RUN if [ "${TARGETARCH}" = "ppc64le" ] || [ "${TARGETARCH}" = "aarch64" ]; then \
+RUN if [ "${TARGETARCH}" = "ppc64le" ]; then \
     apt-get -y update && \
     apt-get -y install gfortran libopenblas-dev liblapack-dev && \
     apt-get clean && \
@@ -26,7 +26,7 @@ COPY --from=downloader /bin/grpc_health_probe /bin/grpc_health_probe
 
 WORKDIR  ${TARGET_DIR}/${SUGGESTION_DIR}
 
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --prefer-binary --no-cache-dir -r requirements.txt
 RUN chgrp -R 0 ${TARGET_DIR} \
   && chmod -R g+rwX ${TARGET_DIR}
 
diff --git a/cmd/suggestion/nas/enas/v1beta1/Dockerfile b/cmd/suggestion/nas/enas/v1beta1/Dockerfile
@@ -14,7 +14,7 @@ ENV SUGGESTION_DIR cmd/suggestion/nas/enas/v1beta1
 ENV GRPC_HEALTH_PROBE_VERSION v0.4.15
 ENV PYTHONPATH ${TARGET_DIR}:${TARGET_DIR}/pkg/apis/manager/v1beta1/python:${TARGET_DIR}/pkg/apis/manager/health/python
 
-RUN if [ "${TARGETARCH}" = "ppc64le" ] || [ "${TARGETARCH}" = "aarch64" ]; then \
+RUN if [ "${TARGETARCH}" = "ppc64le" ]; then \
     apt-get -y update && \
     apt-get -y install gfortran libopenblas-dev liblapack-dev && \
     apt-get clean && \
@@ -27,7 +27,7 @@ COPY --from=downloader /bin/grpc_health_probe /bin/grpc_health_probe
 
 WORKDIR  ${TARGET_DIR}/${SUGGESTION_DIR}
 
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --prefer-binary --no-cache-dir -r requirements.txt
 RUN chgrp -R 0 ${TARGET_DIR} \
   && chmod -R g+rwX ${TARGET_DIR}
 
diff --git a/cmd/suggestion/optuna/v1beta1/Dockerfile b/cmd/suggestion/optuna/v1beta1/Dockerfile
@@ -13,7 +13,7 @@ ENV TARGET_DIR /opt/katib
 ENV SUGGESTION_DIR cmd/suggestion/optuna/v1beta1
 ENV PYTHONPATH ${TARGET_DIR}:${TARGET_DIR}/pkg/apis/manager/v1beta1/python:${TARGET_DIR}/pkg/apis/manager/health/python
 
-RUN if [ "${TARGETARCH}" = "ppc64le" ] || [ "${TARGETARCH}" = "aarch64" ]; then \
+RUN if [ "${TARGETARCH}" = "ppc64le" ]; then \
     apt-get -y update && \
     apt-get -y install gfortran libopenblas-dev liblapack-dev && \
     apt-get clean && \
@@ -26,7 +26,7 @@ COPY --from=downloader /bin/grpc_health_probe /bin/grpc_health_probe
 
 WORKDIR  ${TARGET_DIR}/${SUGGESTION_DIR}
 
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --prefer-binary --no-cache-dir -r requirements.txt
 RUN chgrp -R 0 ${TARGET_DIR} \
   && chmod -R g+rwX ${TARGET_DIR}
 
diff --git a/cmd/suggestion/pbt/v1beta1/Dockerfile b/cmd/suggestion/pbt/v1beta1/Dockerfile
@@ -13,7 +13,7 @@ ENV TARGET_DIR /opt/katib
 ENV SUGGESTION_DIR cmd/suggestion/pbt/v1beta1
 ENV PYTHONPATH ${TARGET_DIR}:${TARGET_DIR}/pkg/apis/manager/v1beta1/python:${TARGET_DIR}/pkg/apis/manager/health/python
 
-RUN if [ "${TARGETARCH}" = "ppc64le" ] || [ "${TARGETARCH}" = "aarch64" ]; then \
+RUN if [ "${TARGETARCH}" = "ppc64le" ]; then \
     apt-get -y update && \
     apt-get -y install gfortran libopenblas-dev liblapack-dev && \
     apt-get clean && \
@@ -26,7 +26,7 @@ COPY --from=downloader /bin/grpc_health_probe /bin/grpc_health_probe
 
 WORKDIR  ${TARGET_DIR}/${SUGGESTION_DIR}
 
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --prefer-binary --no-cache-dir -r requirements.txt
 RUN chgrp -R 0 ${TARGET_DIR} \
   && chmod -R g+rwX ${TARGET_DIR}
 
diff --git a/cmd/suggestion/skopt/v1beta1/Dockerfile b/cmd/suggestion/skopt/v1beta1/Dockerfile
@@ -13,7 +13,7 @@ ENV TARGET_DIR /opt/katib
 ENV SUGGESTION_DIR cmd/suggestion/skopt/v1beta1
 ENV PYTHONPATH ${TARGET_DIR}:${TARGET_DIR}/pkg/apis/manager/v1beta1/python:${TARGET_DIR}/pkg/apis/manager/health/python
 
-RUN if [ "${TARGETARCH}" = "ppc64le" ] || [ "${TARGETARCH}" = "aarch64" ]; then \
+RUN if [ "${TARGETARCH}" = "ppc64le" ]; then \
     apt-get -y update && \
     apt-get -y install gfortran libopenblas-dev liblapack-dev && \
     apt-get clean && \
@@ -26,7 +26,7 @@ COPY --from=downloader /bin/grpc_health_probe /bin/grpc_health_probe
 
 WORKDIR  ${TARGET_DIR}/${SUGGESTION_DIR}
 
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --prefer-binary --no-cache-dir -r requirements.txt
 RUN chgrp -R 0 ${TARGET_DIR} \
   && chmod -R g+rwX ${TARGET_DIR}
 
diff --git a/examples/v1beta1/trial-images/darts-cnn-cifar10/Dockerfile.cpu b/examples/v1beta1/trial-images/darts-cnn-cifar10/Dockerfile.cpu
@@ -6,7 +6,7 @@ ADD examples/v1beta1/trial-images/darts-cnn-cifar10 ${TARGET_DIR}
 
 WORKDIR  ${TARGET_DIR}
 
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --prefer-binary --no-cache-dir -r requirements.txt
 RUN chgrp -R 0 ${TARGET_DIR} \
   && chmod -R g+rwX ${TARGET_DIR}
 
diff --git a/examples/v1beta1/trial-images/enas-cnn-cifar10/Dockerfile.cpu b/examples/v1beta1/trial-images/enas-cnn-cifar10/Dockerfile.cpu
@@ -15,7 +15,7 @@ RUN if [ "${TARGETARCH}" = "arm64" ]; then \
     rm -rf /var/lib/apt/lists/*; \
     fi
 
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --prefer-binary --no-cache-dir -r requirements.txt
 RUN chgrp -R 0 ${TARGET_DIR} \
   && chmod -R g+rwX ${TARGET_DIR}
 
diff --git a/examples/v1beta1/trial-images/enas-cnn-cifar10/Dockerfile.gpu b/examples/v1beta1/trial-images/enas-cnn-cifar10/Dockerfile.gpu
@@ -10,7 +10,7 @@ ADD examples/v1beta1/trial-images/enas-cnn-cifar10 ${TARGET_DIR}
 
 WORKDIR  ${TARGET_DIR}
 
-RUN pip install --no-cache-dir scipy==1.8.1
+RUN pip install --prefer-binary --no-cache-dir scipy==1.8.1
 RUN chgrp -R 0 ${TARGET_DIR} \
   && chmod -R g+rwX ${TARGET_DIR}
 
diff --git a/examples/v1beta1/trial-images/mxnet-mnist/Dockerfile b/examples/v1beta1/trial-images/mxnet-mnist/Dockerfile
@@ -20,7 +20,7 @@ RUN apt-get -y update \
 RUN if [ "${TARGETARCH}" = "arm64" ]; then \
       /opt/mxnet-mnist/install-arm-performance-libraries.sh; \
     fi
-RUN pip install -r requirements.txt
+RUN pip install --prefer-binary -r requirements.txt
 RUN chgrp -R 0 /opt/mxnet-mnist \
   && chmod -R g+rwX /opt/mxnet-mnist
 
diff --git a/examples/v1beta1/trial-images/pytorch-mnist/Dockerfile.cpu b/examples/v1beta1/trial-images/pytorch-mnist/Dockerfile.cpu
@@ -6,7 +6,7 @@ WORKDIR /opt/pytorch-mnist
 
 # Add folder for the logs.
 RUN mkdir /katib
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --prefer-binary --no-cache-dir -r requirements.txt
 
 RUN chgrp -R 0 /opt/pytorch-mnist \
   && chmod -R g+rwX /opt/pytorch-mnist \
diff --git a/examples/v1beta1/trial-images/pytorch-mnist/Dockerfile.gpu b/examples/v1beta1/trial-images/pytorch-mnist/Dockerfile.gpu
@@ -9,7 +9,7 @@ WORKDIR /opt/pytorch-mnist
 
 # Add folder for the logs.
 RUN mkdir /katib
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --prefer-binary --no-cache-dir -r requirements.txt
 
 RUN chgrp -R 0 /opt/pytorch-mnist \
   && chmod -R g+rwX /opt/pytorch-mnist \
diff --git a/examples/v1beta1/trial-images/simple-pbt/Dockerfile b/examples/v1beta1/trial-images/simple-pbt/Dockerfile
@@ -4,7 +4,7 @@ ADD examples/v1beta1/trial-images/simple-pbt /opt/pbt
 
 WORKDIR /opt/pbt
 
-RUN python3 -m pip install -r requirements.txt
+RUN python3 -m pip install --prefer-binary -r requirements.txt
 RUN chgrp -R 0 /opt/pbt \
   && chmod -R g+rwX /opt/pbt
 
diff --git a/examples/v1beta1/trial-images/tf-mnist-with-summaries/Dockerfile b/examples/v1beta1/trial-images/tf-mnist-with-summaries/Dockerfile
@@ -13,7 +13,7 @@ RUN if [ "${TARGETARCH}" = "arm64" ]; then \
     rm -rf /var/lib/apt/lists/*; \
     fi
 
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --prefer-binary --no-cache-dir -r requirements.txt
 RUN chgrp -R 0 /opt/tf-mnist-with-summaries \
   && chmod -R g+rwX /opt/tf-mnist-with-summaries
 
diff --git a/sdk/python/v1beta1/kubeflow/katib/utils/utils.py b/sdk/python/v1beta1/kubeflow/katib/utils/utils.py
@@ -102,7 +102,7 @@ def get_script_for_python_packages(packages_to_install, pip_index_url):
             python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
         fi
 
-        PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet \
+        PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --prefer-binary --quiet \
         --no-warn-script-location --index-url {pip_index_url} {packages_str}
         """
     )
