Skip to content

Commit a438bf7

Browse files
forsaken628andreyvelich
forsaken628
authored and
andreyvelich
committed
Replace gRPC code generation tool from Znly/protoc to Buf (#2344)
* Replace gRPC code generation tool from Znly/protoc to Buf Signed-off-by: forsaken628 <[email protected]> * fix Signed-off-by: forsaken628 <[email protected]> * del build.sh Signed-off-by: forsaken628 <[email protected]> * cleanup Signed-off-by: forsaken628 <[email protected]> * fix test Signed-off-by: forsaken628 <[email protected]> * fix Signed-off-by: forsaken628 <[email protected]> * fix Signed-off-by: forsaken628 <[email protected]> * refine Signed-off-by: forsaken628 <[email protected]> * fix Signed-off-by: forsaken628 <[email protected]> * rm outter yaml Signed-off-by: forsaken628 <[email protected]> * fix Signed-off-by: forsaken628 <[email protected]> --------- Signed-off-by: forsaken628 <[email protected]>
1 parent e8dbf6f commit a438bf7

File tree

25 files changed

+5065
-4800
lines changed

25 files changed

+5065
-4800
lines changed

‎Makefile‎

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,8 @@ CPU_ARCH ?= linux/amd64,linux/arm64
1010
ENVTEST_K8S_VERSION ?= 1.29
1111
MOCKGEN_VERSION ?= $(shell grep 'go.uber.org/mock' go.mod | cut -d ' ' -f 2)
1212
GO_VERSION=$(shell grep '^go' go.mod | cut -d ' ' -f 2)
13+
GOPATH ?= $(shell go env GOPATH)
1314

14-
# for pytest
15-
PYTHONPATH := $(PYTHONPATH):$(CURDIR)/pkg/apis/manager/v1beta1/python:$(CURDIR)/pkg/apis/manager/health/python
16-
PYTHONPATH := $(PYTHONPATH):$(CURDIR)/pkg/metricscollector/v1beta1/common:$(CURDIR)/pkg/metricscollector/v1beta1/tfevent-metricscollector
1715
TEST_TENSORFLOW_EVENT_FILE_PATH ?= $(CURDIR)/test/unit/v1beta1/metricscollector/testdata/tfevent-metricscollector/logs
1816

1917
# Run tests
@@ -93,17 +91,13 @@ controller-gen:
9391
# 4. Generate gRPC manager APIs (pkg/apis/manager/v1beta1/build.sh and pkg/apis/manager/health/build.sh)
9492
# 5. Generate Go mock codes
9593
generate: controller-gen
96-
ifndef GOPATH
97-
$(error GOPATH not defined, please define GOPATH. Run "go help gopath" to learn more about GOPATH)
98-
endif
9994
ifndef HAS_MOCKGEN
10095
go install go.uber.org/mock/mockgen@$(MOCKGEN_VERSION)
10196
$(info "mockgen has been installed")
10297
endif
10398
go generate ./pkg/... ./cmd/...
10499
hack/gen-python-sdk/gen-sdk.sh
105-
pkg/apis/manager/v1beta1/build.sh
106-
pkg/apis/manager/health/build.sh
100+
hack/update-proto.sh
107101
hack/update-mockgen.sh
108102

109103
# Build images for the Katib v1beta1 components.
@@ -175,9 +169,9 @@ ifeq ("$(wildcard $(TEST_TENSORFLOW_EVENT_FILE_PATH))", "")
175169
endif
176170

177171
pytest: prepare-pytest prepare-pytest-testdata
178-
PYTHONPATH=$(PYTHONPATH) pytest ./test/unit/v1beta1/suggestion --ignore=./test/unit/v1beta1/suggestion/test_skopt_service.py
179-
PYTHONPATH=$(PYTHONPATH) pytest ./test/unit/v1beta1/earlystopping
180-
PYTHONPATH=$(PYTHONPATH) pytest ./test/unit/v1beta1/metricscollector
172+
pytest ./test/unit/v1beta1/suggestion --ignore=./test/unit/v1beta1/suggestion/test_skopt_service.py
173+
pytest ./test/unit/v1beta1/earlystopping
174+
pytest ./test/unit/v1beta1/metricscollector
181175

182176
# The skopt service doesn't work appropriately with Python 3.11.
183177
# So, we need to run the test with Python 3.9.
@@ -187,4 +181,4 @@ pytest-skopt:
187181
pip install six
188182
pip install --prefer-binary -r test/unit/v1beta1/requirements.txt
189183
pip install --prefer-binary -r cmd/suggestion/skopt/v1beta1/requirements.txt
190-
PYTHONPATH=$(PYTHONPATH) pytest ./test/unit/v1beta1/suggestion/test_skopt_service.py
184+
pytest ./test/unit/v1beta1/suggestion/test_skopt_service.py

‎cmd/metricscollector/v1beta1/tfevent-metricscollector/main.py‎

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import grpc
1616
import argparse
1717
import api_pb2
18+
import api_pb2_grpc
1819
from pns import WaitMainProcesses
1920
import const
2021
from tfevent_loader import MetricsCollector
@@ -55,25 +56,28 @@ def parse_options():
5556
wait_all_processes = opt.wait_all_processes.lower() == "true"
5657
db_manager_server = opt.db_manager_server_addr.split(':')
5758
if len(db_manager_server) != 2:
58-
raise Exception("Invalid Katib DB manager service address: %s" %
59-
opt.db_manager_server_addr)
59+
raise Exception(
60+
f"Invalid Katib DB manager service address: {opt.db_manager_server_addr}"
61+
)
6062

6163
WaitMainProcesses(
6264
pool_interval=opt.poll_interval,
6365
timout=opt.timeout,
6466
wait_all=wait_all_processes,
65-
completed_marked_dir=opt.metrics_file_dir)
67+
completed_marked_dir=opt.metrics_file_dir,
68+
)
6669

67-
mc = MetricsCollector(opt.metric_names.split(';'))
70+
mc = MetricsCollector(opt.metric_names.split(";"))
6871
observation_log = mc.parse_file(opt.metrics_file_dir)
6972

70-
channel = grpc.beta.implementations.insecure_channel(
71-
db_manager_server[0], int(db_manager_server[1]))
72-
73-
with api_pb2.beta_create_DBManager_stub(channel) as client:
74-
logger.info("In " + opt.trial_name + " " +
75-
str(len(observation_log.metric_logs)) + " metrics will be reported.")
76-
client.ReportObservationLog(api_pb2.ReportObservationLogRequest(
77-
trial_name=opt.trial_name,
78-
observation_log=observation_log
79-
), timeout=timeout_in_seconds)
73+
with grpc.insecure_channel(opt.db_manager_server_addr) as channel:
74+
stub = api_pb2_grpc.DBManagerStub(channel)
75+
logger.info(
76+
f"In {opt.trial_name} {str(len(observation_log.metric_logs))} metrics will be reported."
77+
)
78+
stub.ReportObservationLog(
79+
api_pb2.ReportObservationLogRequest(
80+
trial_name=opt.trial_name, observation_log=observation_log
81+
),
82+
timeout=timeout_in_seconds,
83+
)

‎go.mod‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ require (
77
github.com/awalterschulze/gographviz v2.0.3+incompatible
88
github.com/c-bata/goptuna v0.8.0
99
github.com/go-sql-driver/mysql v1.5.0
10-
github.com/golang/protobuf v1.5.4
1110
github.com/google/go-cmp v0.6.0
1211
github.com/google/go-containerregistry v0.15.2
1312
github.com/google/go-containerregistry/pkg/authn/k8schain v0.0.0-20230517160804-b7ad3f13a62c
@@ -21,8 +20,8 @@ require (
2120
github.com/spf13/viper v1.9.0
2221
github.com/tidwall/gjson v1.14.1
2322
go.uber.org/mock v0.4.0
24-
golang.org/x/net v0.23.0
2523
google.golang.org/grpc v1.58.3
24+
google.golang.org/protobuf v1.33.0
2625
k8s.io/api v0.29.3
2726
k8s.io/apimachinery v0.29.3
2827
k8s.io/client-go v0.29.3
@@ -84,6 +83,7 @@ require (
8483
github.com/gogo/protobuf v1.3.2 // indirect
8584
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
8685
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
86+
github.com/golang/protobuf v1.5.4 // indirect
8787
github.com/google/gnostic-models v0.6.8 // indirect
8888
github.com/google/go-containerregistry/pkg/authn/kubernetes v0.0.0-20230516205744-dbecb1de8cfa // indirect
8989
github.com/google/gofuzz v1.2.0 // indirect
@@ -128,6 +128,7 @@ require (
128128
golang.org/x/crypto v0.21.0 // indirect
129129
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect
130130
golang.org/x/mod v0.14.0 // indirect
131+
golang.org/x/net v0.23.0 // indirect
131132
golang.org/x/oauth2 v0.12.0 // indirect
132133
golang.org/x/sync v0.5.0 // indirect
133134
golang.org/x/sys v0.18.0 // indirect
@@ -139,7 +140,6 @@ require (
139140
gonum.org/v1/gonum v0.8.2 // indirect
140141
google.golang.org/appengine v1.6.7 // indirect
141142
google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
142-
google.golang.org/protobuf v1.33.0 // indirect
143143
gopkg.in/fsnotify/fsnotify.v1 v1.4.7 // indirect
144144
gopkg.in/inf.v0 v0.9.1 // indirect
145145
gopkg.in/ini.v1 v1.63.2 // indirect

‎hack/update-proto.sh‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#!/usr/bin/env bash
2+
3+
# Copyright 2024 The Kubeflow Authors.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -e
18+
19+
SOURCE="${BASH_SOURCE[0]}"
20+
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
21+
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
22+
SOURCE="$(readlink "$SOURCE")"
23+
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
24+
done
25+
ROOT_DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )"
26+
27+
mkdir -p "${ROOT_DIR}/bin"
28+
export GOBIN=$ROOT_DIR/bin
29+
30+
if [ ! -f "${GOBIN}/buf" ]; then
31+
go install github.com/bufbuild/buf/cmd/[email protected]
32+
fi
33+
34+
pushd "${ROOT_DIR}/pkg/apis/manager/health"
35+
"${GOBIN}/buf" generate
36+
popd
37+
38+
pushd "${ROOT_DIR}/pkg/apis/manager/v1beta1"
39+
"${GOBIN}/buf" generate
40+
popd

‎pkg/apis/manager/health/buf.gen.yaml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
version: v2
2+
plugins:
3+
- remote: buf.build/protocolbuffers/go:v1.33.0
4+
out: .
5+
opt: module=github.com/kubeflow/katib/pkg/apis/manager/health
6+
7+
- remote: buf.build/grpc/go:v1.3.0
8+
out: .
9+
opt: module=github.com/kubeflow/katib/pkg/apis/manager/health,require_unimplemented_servers=false
10+
11+
- remote: buf.build/protocolbuffers/python:v26.1
12+
out: python
13+
14+
- remote: buf.build/protocolbuffers/pyi:v26.1
15+
out: python
16+
17+
- remote: buf.build/grpc/python:v1.64.1
18+
out: python
19+
20+
inputs:
21+
- directory: .

‎pkg/apis/manager/health/build.sh‎

Lines changed: 0 additions & 24 deletions
This file was deleted.

0 commit comments

Comments
 (0)