feat: Add feast repo-upgrade for automated repo upgrades (#2733)

* WIP feat: Add feast repo-upgrade for automated repo upgrades

Signed-off-by: Achal Shah <[email protected]>

* pin deps

Signed-off-by: Achal Shah <[email protected]>

Author: achals

sdk/python/feast/cli.py

@@ -41,6 +41,7 @@
     registry_dump,
     teardown,
 )
+from feast.repo_upgrade import RepoUpgrader
 from feast.utils import maybe_local_tz
 
 _logger = logging.getLogger(__name__)
@@ -85,7 +86,7 @@ def cli(ctx: click.Context, chdir: Optional[str], log_level: str):
     try:
         level = getattr(logging, log_level.upper())
         logging.basicConfig(
-            format="%(asctime)s %(levelname)s:%(message)s",
+            format="%(asctime)s %(name)s %(levelname)s: %(message)s",
             datefmt="%m/%d/%Y %I:%M:%S %p",
             level=level,
         )
@@ -98,7 +99,6 @@ def cli(ctx: click.Context, chdir: Optional[str], log_level: str):
             if "feast" in logger_name:
                 logger = logging.getLogger(logger_name)
                 logger.setLevel(level)
-
     except Exception as e:
         raise e
     pass
@@ -825,5 +825,25 @@ def validate(
     exit(1)
 
 
+@cli.command("repo-upgrade", cls=NoOptionDefaultFormat)
+@click.option(
+    "--write",
+    is_flag=True,
+    default=False,
+    help="Upgrade a feature repo to use the API expected by feast 0.23.",
+)
+@click.pass_context
+def repo_upgrade(ctx: click.Context, write: bool):
+    """
+    Upgrade a feature repo in place.
+    """
+    repo = ctx.obj["CHDIR"]
+    cli_check_repo(repo)
+    try:
+        RepoUpgrader(repo, write).upgrade()
+    except FeastProviderLoginError as e:
+        print(str(e))
+
+
 if __name__ == "__main__":
     cli()

sdk/python/feast/repo_upgrade.py

@@ -0,0 +1,72 @@
+import logging
+from pathlib import Path
+from typing import Dict, List
+
+from bowler import Query
+from fissix.pgen2 import token
+from fissix.pygram import python_symbols
+from fissix.pytree import Node
+
+from feast.repo_operations import get_repo_files
+
+SOURCES = {
+    "FileSource",
+    "BigQuerySource",
+    "RedshiftSource",
+    "SnowflakeSource",
+    "KafkaSource",
+    "KinesisSource",
+}
+
+
+class RepoUpgrader:
+    def __init__(self, repo_path: str, write: bool):
+        self.repo_path = repo_path
+        self.write = write
+        self.repo_files: List[str] = [
+            str(p) for p in get_repo_files(Path(self.repo_path))
+        ]
+        logging.getLogger("RefactoringTool").setLevel(logging.WARNING)
+
+    def upgrade(self):
+        self.remove_date_partition_column()
+
+    def remove_date_partition_column(self):
+        def _remove_date_partition_column(
+            node: Node, capture: Dict[str, Node], filename: str
+        ) -> None:
+            self.remove_argument_transform(node, "date_partition_column")
+
+        for s in SOURCES:
+            Query(self.repo_files).select_class(s).is_call().modify(
+                _remove_date_partition_column
+            ).execute(write=self.write, interactive=False)
+
+    @staticmethod
+    def remove_argument_transform(node: Node, argument: str):
+        """
+        Removes the specified argument.
+        For example, if the argument is "join_key", this method transforms
+        driver = Entity(
+            name="driver_id",
+            join_key="driver_id",
+        )
+        into
+        driver = Entity(
+            name="driver_id",
+        )
+        This method assumes that node represents a class call that already has an arglist.
+        """
+        if len(node.children) < 2 or len(node.children[1].children) < 2:
+            raise ValueError(f"Expected a class call with an arglist but got {node}.")
+        class_args = node.children[1].children[1].children
+        for i, class_arg in enumerate(class_args):
+            if (
+                class_arg.type == python_symbols.argument
+                and class_arg.children[0].value == argument
+            ):
+                class_args.pop(i)
+                if i < len(class_args) and class_args[i].type == token.COMMA:
+                    class_args.pop(i)
+                if i < len(class_args) and class_args[i].type == token.NEWLINE:
+                    class_args.pop(i)

sdk/python/requirements/py3.10-ci-requirements.txt

@@ -31,13 +31,13 @@ altair==4.2.0
 anyio==3.6.1
     # via
     #   starlette
-    #   watchgod
+    #   watchfiles
 appdirs==1.4.4
-    # via black
+    # via
+    #   black
+    #   fissix
 appnope==0.1.3
     # via ipython
-asgiref==3.5.2
-    # via uvicorn
 asn1crypto==1.5.1
     # via
     #   oscrypto
@@ -54,6 +54,7 @@ attrs==21.4.0
     # via
     #   aiohttp
     #   black
+    #   bowler
     #   jsonschema
     #   pytest
 avro==1.10.0
@@ -86,6 +87,8 @@ botocore==1.23.24
     #   boto3
     #   moto
     #   s3transfer
+bowler==0.9.0
+    # via feast (setup.py)
 build==0.8.0
     # via feast (setup.py)
 cachecontrol==0.12.11
@@ -113,8 +116,10 @@ charset-normalizer==2.0.12
 click==8.0.1
     # via
     #   black
+    #   bowler
     #   feast (setup.py)
     #   great-expectations
+    #   moreorless
     #   pip-tools
     #   uvicorn
 cloudpickle==2.1.0
@@ -178,6 +183,8 @@ filelock==3.7.1
     # via virtualenv
 firebase-admin==4.5.2
     # via feast (setup.py)
+fissix==21.11.13
+    # via bowler
 flake8==4.0.1
     # via feast (setup.py)
 frozenlist==1.3.0
@@ -243,7 +250,7 @@ google-resumable-media==1.3.3
     # via
     #   google-cloud-bigquery
     #   google-cloud-storage
-googleapis-common-protos==1.56.2
+googleapis-common-protos==1.56.3
     # via
     #   feast (setup.py)
     #   google-api-core
@@ -252,15 +259,15 @@ great-expectations==0.14.13
     # via feast (setup.py)
 greenlet==1.1.2
     # via sqlalchemy
-grpcio==1.46.3
+grpcio==1.47.0
     # via
     #   feast (setup.py)
     #   google-api-core
     #   google-cloud-bigquery
     #   grpcio-reflection
     #   grpcio-testing
     #   grpcio-tools
-grpcio-reflection==1.46.3
+grpcio-reflection==1.47.0
     # via feast (setup.py)
 grpcio-testing==1.44.0
     # via feast (setup.py)
@@ -341,7 +348,9 @@ mmh3==3.0.0
     # via feast (setup.py)
 mock==2.0.0
     # via feast (setup.py)
-moto==3.1.13
+moreorless==0.4.0
+    # via bowler
+moto==3.1.14
     # via feast (setup.py)
 msal==1.18.0
     # via
@@ -369,7 +378,7 @@ mypy-extensions==0.4.3
     # via mypy
 mypy-protobuf==3.1
     # via feast (setup.py)
-mysqlclient==2.1.0
+mysqlclient==2.1.1
     # via feast (setup.py)
 nbformat==5.4.0
     # via great-expectations
@@ -399,7 +408,7 @@ packaging==21.3
     #   pytest
     #   redis
     #   sphinx
-pandas==1.4.2
+pandas==1.4.3
     # via
     #   altair
     #   feast (setup.py)
@@ -492,7 +501,7 @@ pycodestyle==2.8.0
     # via flake8
 pycparser==2.21
     # via cffi
-pycryptodomex==3.14.1
+pycryptodomex==3.15.0
     # via snowflake-connector-python
 pydantic==1.9.1
     # via
@@ -656,19 +665,19 @@ sphinxcontrib-qthelp==1.0.3
     # via sphinx
 sphinxcontrib-serializinghtml==1.1.5
     # via sphinx
-sqlalchemy[mypy]==1.4.37
+sqlalchemy[mypy]==1.4.38
     # via feast (setup.py)
 sqlalchemy2-stubs==0.0.2a24
     # via sqlalchemy
 stack-data==0.3.0
     # via ipython
 starlette==0.19.1
     # via fastapi
-tabulate==0.8.9
+tabulate==0.8.10
     # via feast (setup.py)
 tenacity==8.0.1
     # via feast (setup.py)
-tensorflow-metadata==1.8.0
+tensorflow-metadata==1.9.0
     # via feast (setup.py)
 termcolor==1.1.0
     # via great-expectations
@@ -713,19 +722,19 @@ types-protobuf==3.19.22
     # via
     #   feast (setup.py)
     #   mypy-protobuf
-types-python-dateutil==2.8.17
+types-python-dateutil==2.8.18
     # via feast (setup.py)
-types-pytz==2021.3.8
+types-pytz==2022.1.0
     # via feast (setup.py)
 types-pyyaml==6.0.8
     # via feast (setup.py)
-types-redis==4.2.7
+types-redis==4.3.2
     # via feast (setup.py)
-types-requests==2.27.30
+types-requests==2.27.31
     # via feast (setup.py)
 types-setuptools==57.4.17
     # via feast (setup.py)
-types-tabulate==0.8.9
+types-tabulate==0.8.10
     # via feast (setup.py)
 types-urllib3==1.26.15
     # via types-requests
@@ -750,17 +759,19 @@ urllib3==1.26.9
     #   minio
     #   requests
     #   responses
-uvicorn[standard]==0.17.6
+uvicorn[standard]==0.18.1
     # via feast (setup.py)
 uvloop==0.16.0
     # via uvicorn
 virtualenv==20.14.1
     # via pre-commit
-watchgod==0.8.2
+volatile==2.1.0
+    # via bowler
+watchfiles==0.15.0
     # via uvicorn
 wcwidth==0.2.5
     # via prompt-toolkit
-websocket-client==1.3.2
+websocket-client==1.3.3
     # via docker
 websockets==10.3
     # via uvicorn