fix: Prefer installing gopy from feast's fork as opposed to upstream (#2839)

achals · web-flow · commit 34c997d81b00 · 2022-06-22T17:32:52.000-07:00
* fix: Prefer installing gopy from feast's fork as opposed to upstream

Signed-off-by: Achal Shah &lt;achals@gmail.com&gt;

* use go mod tidy

Signed-off-by: Achal Shah &lt;achals@gmail.com&gt;

* pin and pray

Signed-off-by: Achal Shah &lt;achals@gmail.com&gt;

* fix import path

Signed-off-by: Achal Shah &lt;achals@gmail.com&gt;

* nix go install

Signed-off-by: Achal Shah &lt;achals@gmail.com&gt;

* pin go get and add back unpinned go install, and fix other tests

Signed-off-by: Achal Shah &lt;achals@gmail.com&gt;

* cr

Signed-off-by: Achal Shah &lt;achals@gmail.com&gt;
diff --git a/Makefile b/Makefile
@@ -170,10 +170,12 @@ install-go-proto-dependencies:
 	go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1.0
 
 install-go-ci-dependencies:
-	# ToDo: currently gopy installation doesn't work w/o explicit go get in the next line
-	# ToDo: there should be a better way to install gopy
-	go get github.com/go-python/gopy
+	# TODO: currently gopy installation doesn't work w/o explicit go get in the next line
+	# TODO: there should be a better way to install gopy
+	go get github.com/go-python/gopy@v0.4.0
 	go install golang.org/x/tools/cmd/goimports
+	# The `go get` command on the previous lines download the lib along with replacing the dep to `feast-dev/gopy`
+	# but the following command is needed to install it for some reason.
 	go install github.com/go-python/gopy
 	python -m pip install pybindgen==0.22.0
 
diff --git a/go.mod b/go.mod
@@ -2,6 +2,8 @@ module github.com/feast-dev/feast
 
 go 1.17
 
+replace github.com/go-python/gopy v0.4.0 => github.com/feast-dev/gopy v0.4.1-0.20220429180328-4257ac71a4d0
+
 require (
 	github.com/apache/arrow/go/v8 v8.0.0-20220408212425-58fe60f59289
 	github.com/ghodss/yaml v1.0.0
@@ -26,8 +28,6 @@ require (
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/goccy/go-json v0.9.6 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/gonuts/commander v0.1.0 // indirect
-	github.com/gonuts/flag v0.1.0 // indirect
 	github.com/google/flatbuffers v2.0.6+incompatible // indirect
 	github.com/klauspost/asmfmt v1.3.2 // indirect
 	github.com/klauspost/compress v1.15.1 // indirect
@@ -42,12 +42,10 @@ require (
 	golang.org/x/net v0.0.0-20220407224826-aac1ed45d8e3 // indirect
 	golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12 // indirect
 	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/tools v0.1.10 // indirect
+	golang.org/x/tools v0.1.11-0.20220413170336-afc6aad76eb1 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
 )
-
-replace github.com/go-python/gopy v0.4.0 => github.com/feast-dev/gopy v0.4.1-0.20220429180328-4257ac71a4d0
diff --git a/go.sum b/go.sum
@@ -146,9 +146,7 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/gonuts/commander v0.1.0 h1:EcDTiVw9oAVORFjQOEOuHQqcl6OXMyTgELocTq6zJ0I=
 github.com/gonuts/commander v0.1.0/go.mod h1:qkb5mSlcWodYgo7vs8ulLnXhfinhZsZcm6+H/z1JjgY=
-github.com/gonuts/flag v0.1.0 h1:fqMv/MZ+oNGu0i9gp0/IQ/ZaPIDoAZBOBaJoV7viCWM=
 github.com/gonuts/flag v0.1.0/go.mod h1:ZTmTGtrSPejTo/SRNhCqwLTmiAgyBdCkLYhHrAoBdz4=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -548,8 +546,8 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
 golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
-golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20=
-golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
+golang.org/x/tools v0.1.11-0.20220413170336-afc6aad76eb1 h1:Z3vE1sGlC7qiyFJkkDcZms8Y3+yV8+W7HmDSmuf71tM=
+golang.org/x/tools v0.1.11-0.20220413170336-afc6aad76eb1/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
diff --git a/sdk/python/tests/integration/online_store/test_e2e_local.py b/sdk/python/tests/integration/online_store/test_e2e_local.py
@@ -130,15 +130,15 @@ def _test_materialize_and_online_retrieval(
         cwd=Path(store.repo_path),
     )
 
-    assert r.returncode == 0
+    assert r.returncode == 0, f"stdout: {r.stdout}\n stderr: {r.stderr}"
     _assert_online_features(store, driver_df, end_date - timedelta(days=7))
 
     # Test `feast materialize-incremental` and online retrieval.
     r = runner.run(
         ["materialize-incremental", end_date.isoformat()], cwd=Path(store.repo_path),
     )
 
-    assert r.returncode == 0
+    assert r.returncode == 0, f"stdout: {r.stdout}\n stderr: {r.stderr}"
     _assert_online_features(store, driver_df, end_date)
 
 
diff --git a/sdk/python/tests/utils/cli_utils.py b/sdk/python/tests/utils/cli_utils.py
@@ -84,9 +84,13 @@ def local_repo(self, example_repo_py: str, offline_store: str):
             repo_example.write_text(example_repo_py)
 
             result = self.run(["apply"], cwd=repo_path)
-            assert result.returncode == 0
+            assert (
+                result.returncode == 0
+            ), f"stdout: {result.stdout}\n stderr: {result.stderr}"
 
             yield FeatureStore(repo_path=str(repo_path), config=None)
 
             result = self.run(["teardown"], cwd=repo_path)
-            assert result.returncode == 0
+            assert (
+                result.returncode == 0
+            ), f"stdout: {result.stdout}\n stderr: {result.stderr}"
diff --git a/setup.py b/setup.py
@@ -408,6 +408,7 @@ def _is_go_ext(self, ext: Extension):
         )
 
     def build_extension(self, ext: Extension):
+        print(f"Building extension {ext}")
         if not self._is_go_ext(ext):
             # the base class may mutate `self.compiler`
             compiler = copy.deepcopy(self.compiler)
@@ -423,8 +424,7 @@ def build_extension(self, ext: Extension):
         )
 
         destination = os.path.dirname(os.path.abspath(self.get_ext_fullpath(ext.name)))
-        subprocess.check_call(["go", "install", "golang.org/x/tools/cmd/goimports"])
-        subprocess.check_call(["go", "install", "github.com/go-python/gopy"])
+        subprocess.check_call(["go", "mod", "tidy"], env={"PATH": bin_path, **go_env})
         subprocess.check_call(
             [
                 "gopy",
@@ -456,6 +456,7 @@ def copy_extensions_to_source(self):
             src_dir = os.path.join(self.build_lib, src_dir)
 
             # copy whole directory
+            print(f"Copying from {src_dir} to {dest_dir}")
             copy_tree(src_dir, dest_dir)
 
 

Original file line number	Diff line number	Diff line change
`@@ -130,15 +130,15 @@ def _test_materialize_and_online_retrieval(`
`130`	`130`	`cwd=Path(store.repo_path),`
`131`	`131`	`)`
`132`	`132`
`133`		`- assert r.returncode == 0`
	`133`	`+ assert r.returncode == 0, f"stdout: {r.stdout}\n stderr: {r.stderr}"`
`134`	`134`	`_assert_online_features(store, driver_df, end_date - timedelta(days=7))`
`135`	`135`
`136`	`136`	# Test `feast materialize-incremental` and online retrieval.
`137`	`137`	`r = runner.run(`
`138`	`138`	`["materialize-incremental", end_date.isoformat()], cwd=Path(store.repo_path),`
`139`	`139`	`)`
`140`	`140`
`141`		`- assert r.returncode == 0`
	`141`	`+ assert r.returncode == 0, f"stdout: {r.stdout}\n stderr: {r.stderr}"`
`142`	`142`	`_assert_online_features(store, driver_df, end_date)`
`143`	`143`
`144`	`144`
Original file line number	Diff line number	Diff line change
`@@ -408,6 +408,7 @@ def _is_go_ext(self, ext: Extension):`
`408`	`408`	`)`
`409`	`409`
`410`	`410`	`def build_extension(self, ext: Extension):`
	`411`	`+ print(f"Building extension {ext}")`
`411`	`412`	`if not self._is_go_ext(ext):`
`412`	`413`	# the base class may mutate `self.compiler`
`413`	`414`	`compiler = copy.deepcopy(self.compiler)`
`@@ -423,8 +424,7 @@ def build_extension(self, ext: Extension):`
`423`	`424`	`)`
`424`	`425`
`425`	`426`	`destination = os.path.dirname(os.path.abspath(self.get_ext_fullpath(ext.name)))`
`426`		`- subprocess.check_call(["go", "install", "golang.org/x/tools/cmd/goimports"])`
`427`		`- subprocess.check_call(["go", "install", "github.com/go-python/gopy"])`
	`427`	`+ subprocess.check_call(["go", "mod", "tidy"], env={"PATH": bin_path, **go_env})`
`428`	`428`	`subprocess.check_call(`
`429`	`429`	`[`
`430`	`430`	`"gopy",`
`@@ -456,6 +456,7 @@ def copy_extensions_to_source(self):`
`456`	`456`	`src_dir = os.path.join(self.build_lib, src_dir)`
`457`	`457`
`458`	`458`	`# copy whole directory`
	`459`	`+ print(f"Copying from {src_dir} to {dest_dir}")`
`459`	`460`	`copy_tree(src_dir, dest_dir)`
`460`	`461`
`461`	`462`