feat(mlop-2269): bump versions (#355)

* fix: bump versions adjust tests

* add checklist

* chore: bump python

* bump pyspark

* chore: java version all steps modified

Author: ralphrass

.checklist.yaml

@@ -0,0 +1,30 @@
+apiVersion: quintoandar.com.br/checklist/v2
+kind: ServiceChecklist
+metadata:
+  name: butterfree
+spec:
+  description: >-
+    A solution for Feature Stores.
+
+  costCenter: C055
+  department: engineering
+  lifecycle: production
+  docs: true
+
+  ownership:
+    team: data_products_mlops
+    line: tech_platform
+    owner: [email protected]
+
+  libraries:
+    - name: butterfree
+      type: common-usage
+      path: https://quintoandar.github.io/python-package-server/
+      description: A lib to build Feature Stores.
+      registries:
+      - github-packages
+      tier: T0
+
+  channels:
+    squad: 'mlops'
+    alerts: 'data-products-reports'

.github/workflows/publish.yml

@@ -14,6 +14,19 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.9'
+
+    - uses: actions/setup-java@v4
+      with:
+        java-version: '11'
+        distribution: microsoft
+
+    - uses: vemonet/setup-spark@v1
+      with:
+        spark-version: '3.5.1'
+        hadoop-version: '3'
 
     - name: Install dependencies
       run: make ci-install

.github/workflows/staging.yml

@@ -8,11 +8,23 @@ jobs:
   Pipeline:
     if: github.ref == 'refs/heads/staging'
 
-    runs-on: ubuntu-22.04
-    container: quintoandar/python-3-7-java
+    runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v2
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
+
+      - uses: actions/setup-java@v4
+        with:
+          java-version: '11'
+          distribution: microsoft
+
+      - uses: vemonet/setup-spark@v1
+        with:
+          spark-version: '3.5.1'
+          hadoop-version: '3'
 
       - name: Install dependencies
         run: make ci-install

.github/workflows/test.yml

@@ -9,11 +9,23 @@ on:
 
 jobs:
   Pipeline:
-    runs-on: ubuntu-22.04
-    container: quintoandar/python-3-7-java
+    runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v2
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.9'
+
+    - uses: actions/setup-java@v4
+      with:
+        java-version: '11'
+        distribution: microsoft
+
+    - uses: vemonet/setup-spark@v1
+      with:
+        spark-version: '3.5.1'
+        hadoop-version: '3'
 
     - name: Install dependencies
       run: make ci-install

.gitignore

@@ -66,6 +66,7 @@ instance/
 
 # PyBuilder
 target/
+pip/
 
 # Jupyter Notebook
 .ipynb_checkpoints

Makefile

@@ -76,7 +76,7 @@ style-check:
 	@echo "Code Style"
 	@echo "=========="
 	@echo ""
-	@python -m black --check -t py36 --exclude="build/|buck-out/|dist/|_build/|pip/|\.pip/|\.git/|\.hg/|\.mypy_cache/|\.tox/|\.venv/" . && echo "\n\nSuccess" || (echo "\n\nFailure\n\nYou need to run \"make apply-style\" to apply style formatting to your code"; exit 1)
+	@python -m black --check -t py39 --exclude="build/|buck-out/|dist/|_build/|pip/|\.pip/|\.git/|\.hg/|\.mypy_cache/|\.tox/|\.venv/" . && echo "\n\nSuccess" || (echo "\n\nFailure\n\nYou need to run \"make apply-style\" to apply style formatting to your code"; exit 1)
 
 .PHONY: quality-check
 ## run code quality checks with flake8
@@ -104,8 +104,8 @@ checks: style-check quality-check type-check
 .PHONY: apply-style
 ## fix stylistic errors with black
 apply-style:
-	@python -m black -t py36 --exclude="build/|buck-out/|dist/|_build/|pip/|\.pip/|\.git/|\.hg/|\.mypy_cache/|\.tox/|\.venv/" .
-	@python -m isort -rc --atomic butterfree/ tests/
+	@python -m black -t py39 --exclude="build/|buck-out/|dist/|_build/|pip/|\.pip/|\.git/|\.hg/|\.mypy_cache/|\.tox/|\.venv/" .
+	@python -m isort --atomic butterfree/ tests/
 
 .PHONY: clean
 ## clean unused artifacts

butterfree/_cli/migrate.py

@@ -46,13 +46,13 @@ def __fs_objects(path: str) -> Set[FeatureSetPipeline]:
         logger.error(f"Path: {path} not found!")
         return set()
 
-    logger.info(f"Importing modules...")
+    logger.info("Importing modules...")
     package = ".".join(path.strip("/").split("/"))
     imported = set(
         importlib.import_module(f".{name}", package=package) for name in modules
     )
 
-    logger.info(f"Scanning modules...")
+    logger.info("Scanning modules...")
     content = {
         module: set(
             filter(
@@ -93,7 +93,8 @@ def __fs_objects(path: str) -> Set[FeatureSetPipeline]:
 
 
 PATH = typer.Argument(
-    ..., help="Full or relative path to where feature set pipelines are being defined.",
+    ...,
+    help="Full or relative path to where feature set pipelines are being defined.",
 )
 
 GENERATE_LOGS = typer.Option(
@@ -113,7 +114,10 @@ class Migrate:
         pipelines: list of Feature Set Pipelines to use to migration.
     """
 
-    def __init__(self, pipelines: Set[FeatureSetPipeline],) -> None:
+    def __init__(
+        self,
+        pipelines: Set[FeatureSetPipeline],
+    ) -> None:
         self.pipelines = pipelines
 
     def _send_logs_to_s3(self, file_local: bool, debug_mode: bool) -> None:

butterfree/clients/cassandra_client.py

@@ -129,7 +129,9 @@ def get_schema(self, table: str, database: str = None) -> List[Dict[str, str]]:
         return response
 
     def _get_create_table_query(
-        self, columns: List[CassandraColumn], table: str,
+        self,
+        columns: List[CassandraColumn],
+        table: str,
     ) -> str:
         """Creates CQL statement to create a table."""
         parsed_columns = []

butterfree/clients/spark_client.py

@@ -61,9 +61,9 @@ def read(
         if path and not isinstance(path, (str, list)):
             raise ValueError("path needs to be a string or a list of string")
 
-        df_reader: Union[
-            DataStreamReader, DataFrameReader
-        ] = self.conn.readStream if stream else self.conn.read
+        df_reader: Union[DataStreamReader, DataFrameReader] = (
+            self.conn.readStream if stream else self.conn.read
+        )
 
         df_reader = df_reader.schema(schema) if schema else df_reader
 

butterfree/extract/source.py

@@ -58,7 +58,10 @@ class Source(HookableComponent):
     """
 
     def __init__(
-        self, readers: List[Reader], query: str, eager_evaluation: bool = True,
+        self,
+        readers: List[Reader],
+        query: str,
+        eager_evaluation: bool = True,
     ) -> None:
         super().__init__()
         self.enable_pre_hooks = False