Skip to content

Commit 1c45521

Browse files
andreyvelichforsaken628tenzen-y
authored
Cherry pick of #2350 #2355 #2357 #2344 #2358 #2360 into release-0.17 branch (#2362)
* Fix TestReconcileBatchJob (#2350) * update Signed-off-by: forsaken628 <[email protected]> * fix Signed-off-by: forsaken628 <[email protected]> * update Signed-off-by: forsaken628 <[email protected]> * update Signed-off-by: forsaken628 <[email protected]> * update Signed-off-by: forsaken628 <[email protected]> * fix Signed-off-by: forsaken628 <[email protected]> * cleanup Signed-off-by: forsaken628 <[email protected]> * fix Signed-off-by: forsaken628 <[email protected]> * update Signed-off-by: forsaken628 <[email protected]> * use gomock Signed-off-by: forsaken628 <[email protected]> --------- Signed-off-by: forsaken628 <[email protected]> Signed-off-by: Andrey Velichkevich <[email protected]> * Use cache-dependency-path in actions/setup-go for CI workflow (#2355) Signed-off-by: forsaken628 <[email protected]> Signed-off-by: Andrey Velichkevich <[email protected]> * Replace already closed github.com/golang/mock with go.uber.org/mock (#2357) * replace gomock Signed-off-by: forsaken628 <[email protected]> * fix Signed-off-by: forsaken628 <[email protected]> * revert Signed-off-by: forsaken628 <[email protected]> * fix Signed-off-by: forsaken628 <[email protected]> * fix Signed-off-by: forsaken628 <[email protected]> --------- Signed-off-by: forsaken628 <[email protected]> Signed-off-by: Andrey Velichkevich <[email protected]> * Replace gRPC code generation tool from Znly/protoc to Buf (#2344) * Replace gRPC code generation tool from Znly/protoc to Buf Signed-off-by: forsaken628 <[email protected]> * fix Signed-off-by: forsaken628 <[email protected]> * del build.sh Signed-off-by: forsaken628 <[email protected]> * cleanup Signed-off-by: forsaken628 <[email protected]> * fix test Signed-off-by: forsaken628 <[email protected]> * fix Signed-off-by: forsaken628 <[email protected]> * fix Signed-off-by: forsaken628 <[email protected]> * refine Signed-off-by: forsaken628 <[email protected]> * fix Signed-off-by: forsaken628 <[email protected]> * rm outter yaml Signed-off-by: forsaken628 <[email protected]> * fix Signed-off-by: forsaken628 <[email protected]> --------- Signed-off-by: forsaken628 <[email protected]> Signed-off-by: Andrey Velichkevich <[email protected]> * Upgrade the protobuf version to >=4.21.12,<5 (#2358) Signed-off-by: Yuki Iwai <[email protected]> Signed-off-by: Andrey Velichkevich <[email protected]> * [SDK] Fix empty list for env variables and numpy version (#2360) * [SDK] Fix empty list for env variables Signed-off-by: Andrey Velichkevich <[email protected]> * Fix numpy version in tests Signed-off-by: Andrey Velichkevich <[email protected]> --------- Signed-off-by: Andrey Velichkevich <[email protected]> --------- Signed-off-by: forsaken628 <[email protected]> Signed-off-by: Andrey Velichkevich <[email protected]> Signed-off-by: Yuki Iwai <[email protected]> Co-authored-by: coldWater <[email protected]> Co-authored-by: Yuki Iwai <[email protected]>
1 parent 6cac704 commit 1c45521

File tree

57 files changed

+5340
-5013
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

57 files changed

+5340
-5013
lines changed

‎.github/workflows/test-go.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ jobs:
2828
uses: actions/setup-go@v5
2929
with:
3030
go-version-file: ${{ env.GOPATH }}/src/github.com/kubeflow/katib/go.mod
31+
cache-dependency-path: ${{ env.GOPATH }}/src/github.com/kubeflow/katib/go.sum
3132

3233
- name: Check Go Modules, Generated Go/Python codes, and Format
3334
run: make check
@@ -50,6 +51,7 @@ jobs:
5051
uses: actions/setup-go@v5
5152
with:
5253
go-version-file: ${{ env.GOPATH }}/src/github.com/kubeflow/katib/go.mod
54+
cache-dependency-path: ${{ env.GOPATH }}/src/github.com/kubeflow/katib/go.sum
5355

5456
- name: Run Go test
5557
run: go mod download && make test ENVTEST_K8S_VERSION=${{ matrix.kubernetes-version }}

‎Makefile‎

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,10 @@ COMMIT := v1beta1-$(shell git rev-parse --short=7 HEAD)
88
KATIB_REGISTRY := docker.io/kubeflowkatib
99
CPU_ARCH ?= linux/amd64,linux/arm64
1010
ENVTEST_K8S_VERSION ?= 1.29
11-
MOCKGEN_VERSION ?= $(shell grep 'github.com/golang/mock' go.mod | cut -d ' ' -f 2)
11+
MOCKGEN_VERSION ?= $(shell grep 'go.uber.org/mock' go.mod | cut -d ' ' -f 2)
1212
GO_VERSION=$(shell grep '^go' go.mod | cut -d ' ' -f 2)
13+
GOPATH ?= $(shell go env GOPATH)
1314

14-
# for pytest
15-
PYTHONPATH := $(PYTHONPATH):$(CURDIR)/pkg/apis/manager/v1beta1/python:$(CURDIR)/pkg/apis/manager/health/python
16-
PYTHONPATH := $(PYTHONPATH):$(CURDIR)/pkg/metricscollector/v1beta1/common:$(CURDIR)/pkg/metricscollector/v1beta1/tfevent-metricscollector
1715
TEST_TENSORFLOW_EVENT_FILE_PATH ?= $(CURDIR)/test/unit/v1beta1/metricscollector/testdata/tfevent-metricscollector/logs
1816

1917
# Run tests
@@ -93,17 +91,13 @@ controller-gen:
9391
# 4. Generate gRPC manager APIs (pkg/apis/manager/v1beta1/build.sh and pkg/apis/manager/health/build.sh)
9492
# 5. Generate Go mock codes
9593
generate: controller-gen
96-
ifndef GOPATH
97-
$(error GOPATH not defined, please define GOPATH. Run "go help gopath" to learn more about GOPATH)
98-
endif
9994
ifndef HAS_MOCKGEN
100-
go install github.com/golang/mock/mockgen@$(MOCKGEN_VERSION)
95+
go install go.uber.org/mock/mockgen@$(MOCKGEN_VERSION)
10196
$(info "mockgen has been installed")
10297
endif
10398
go generate ./pkg/... ./cmd/...
10499
hack/gen-python-sdk/gen-sdk.sh
105-
pkg/apis/manager/v1beta1/build.sh
106-
pkg/apis/manager/health/build.sh
100+
hack/update-proto.sh
107101
hack/update-mockgen.sh
108102

109103
# Build images for the Katib v1beta1 components.
@@ -175,9 +169,9 @@ ifeq ("$(wildcard $(TEST_TENSORFLOW_EVENT_FILE_PATH))", "")
175169
endif
176170

177171
pytest: prepare-pytest prepare-pytest-testdata
178-
PYTHONPATH=$(PYTHONPATH) pytest ./test/unit/v1beta1/suggestion --ignore=./test/unit/v1beta1/suggestion/test_skopt_service.py
179-
PYTHONPATH=$(PYTHONPATH) pytest ./test/unit/v1beta1/earlystopping
180-
PYTHONPATH=$(PYTHONPATH) pytest ./test/unit/v1beta1/metricscollector
172+
pytest ./test/unit/v1beta1/suggestion --ignore=./test/unit/v1beta1/suggestion/test_skopt_service.py
173+
pytest ./test/unit/v1beta1/earlystopping
174+
pytest ./test/unit/v1beta1/metricscollector
181175

182176
# The skopt service doesn't work appropriately with Python 3.11.
183177
# So, we need to run the test with Python 3.9.
@@ -187,4 +181,4 @@ pytest-skopt:
187181
pip install six
188182
pip install --prefer-binary -r test/unit/v1beta1/requirements.txt
189183
pip install --prefer-binary -r cmd/suggestion/skopt/v1beta1/requirements.txt
190-
PYTHONPATH=$(PYTHONPATH) pytest ./test/unit/v1beta1/suggestion/test_skopt_service.py
184+
pytest ./test/unit/v1beta1/suggestion/test_skopt_service.py

‎cmd/db-manager/v1beta1/main_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ import (
2020
"context"
2121
"testing"
2222

23-
"github.com/golang/mock/gomock"
23+
"go.uber.org/mock/gomock"
2424

2525
health_pb "github.com/kubeflow/katib/pkg/apis/manager/health"
2626
api_pb "github.com/kubeflow/katib/pkg/apis/manager/v1beta1"

‎cmd/earlystopping/medianstop/v1beta1/requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
grpcio>=1.41.1
2-
protobuf>=3.19.5, <=3.20.3
2+
protobuf>=4.21.12,<5
33
googleapis-common-protos==1.6.0
44
kubernetes==22.6.0
55
cython>=0.29.24

‎cmd/metricscollector/v1beta1/tfevent-metricscollector/main.py‎

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import grpc
1616
import argparse
1717
import api_pb2
18+
import api_pb2_grpc
1819
from pns import WaitMainProcesses
1920
import const
2021
from tfevent_loader import MetricsCollector
@@ -55,25 +56,28 @@ def parse_options():
5556
wait_all_processes = opt.wait_all_processes.lower() == "true"
5657
db_manager_server = opt.db_manager_server_addr.split(':')
5758
if len(db_manager_server) != 2:
58-
raise Exception("Invalid Katib DB manager service address: %s" %
59-
opt.db_manager_server_addr)
59+
raise Exception(
60+
f"Invalid Katib DB manager service address: {opt.db_manager_server_addr}"
61+
)
6062

6163
WaitMainProcesses(
6264
pool_interval=opt.poll_interval,
6365
timout=opt.timeout,
6466
wait_all=wait_all_processes,
65-
completed_marked_dir=opt.metrics_file_dir)
67+
completed_marked_dir=opt.metrics_file_dir,
68+
)
6669

67-
mc = MetricsCollector(opt.metric_names.split(';'))
70+
mc = MetricsCollector(opt.metric_names.split(";"))
6871
observation_log = mc.parse_file(opt.metrics_file_dir)
6972

70-
channel = grpc.beta.implementations.insecure_channel(
71-
db_manager_server[0], int(db_manager_server[1]))
72-
73-
with api_pb2.beta_create_DBManager_stub(channel) as client:
74-
logger.info("In " + opt.trial_name + " " +
75-
str(len(observation_log.metric_logs)) + " metrics will be reported.")
76-
client.ReportObservationLog(api_pb2.ReportObservationLogRequest(
77-
trial_name=opt.trial_name,
78-
observation_log=observation_log
79-
), timeout=timeout_in_seconds)
73+
with grpc.insecure_channel(opt.db_manager_server_addr) as channel:
74+
stub = api_pb2_grpc.DBManagerStub(channel)
75+
logger.info(
76+
f"In {opt.trial_name} {str(len(observation_log.metric_logs))} metrics will be reported."
77+
)
78+
stub.ReportObservationLog(
79+
api_pb2.ReportObservationLogRequest(
80+
trial_name=opt.trial_name, observation_log=observation_log
81+
),
82+
timeout=timeout_in_seconds,
83+
)

‎cmd/metricscollector/v1beta1/tfevent-metricscollector/requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,4 @@ rfc3339>=6.2
33
grpcio>=1.41.1
44
googleapis-common-protos==1.6.0
55
tensorflow==2.16.1
6-
protobuf<=3.20.3
6+
protobuf>=4.21.12,<5

‎cmd/suggestion/hyperband/v1beta1/requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,6 @@ numpy>=1.25.2
44
scikit-learn>=0.24.0
55
scipy>=1.5.4
66
forestci==0.3
7-
protobuf>=3.19.5, <=3.20.3
7+
protobuf>=4.21.12,<5
88
googleapis-common-protos==1.6.0
99
cython>=0.29.24

‎cmd/suggestion/hyperopt/v1beta1/requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ numpy>=1.25.2
44
scikit-learn>=0.24.0
55
scipy>=1.5.4
66
forestci==0.3
7-
protobuf>=3.19.5, <=3.20.3
7+
protobuf>=4.21.12,<5
88
googleapis-common-protos==1.6.0
99
hyperopt==0.2.5
1010
cython>=0.29.24

‎cmd/suggestion/nas/darts/v1beta1/requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
grpcio>=1.41.1
2-
protobuf>=3.19.5, <=3.20.3
2+
protobuf>=4.21.12,<5
33
googleapis-common-protos==1.6.0
44
cython>=0.29.24

‎cmd/suggestion/nas/enas/v1beta1/requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,4 @@ grpcio>=1.41.1
22
googleapis-common-protos==1.6.0
33
cython>=0.29.24
44
tensorflow==2.16.1
5-
protobuf<=3.20.3
5+
protobuf>=4.21.12,<5

0 commit comments

Comments
 (0)